TextToAssets.h

Go to the documentation of this file.

#pragma once
 
namespace krotos
{
 
constexpr char ttfEmbeddingsFileName[] = ".ttfEmbCache";
constexpr char ttfStatusFileName[] = ".ttfStatus";
constexpr char ttfStatusDateProperty[] = "date modified";
constexpr char ttfStatusSizeProperty[] = "folder size";
constexpr char ttfNumericalMetadata[] = "-0123456789";
 
static const Identifier FilesEmbeddingsIdentifier{"FilesEmbeddings"};
static const Identifier fileNameIdentifier{"fileName"};
static const Identifier catIDNameIdentifier{"catID"};
static const Identifier ttfEmbeddingsNameIdentifier{"embeddings"};
static const Identifier customParamIdentifier{"CUSTOMPARAM"};
static const Identifier samplePathsIdentifier{"SamplePaths"};
 
 
static const Identifier contentIdentifier{"content"};
static const Identifier textIdentifier{"text"};
static const Identifier soundscapeIdentifier{"Soundscape"};
static const Identifier layer1Identifier{"Layer-1"};
static const Identifier layer2Identifier{"Layer-2"};
static const Identifier layer3Identifier{"Layer-3"};
static const Identifier layer4Identifier{"Layer-4"};
 
    enum AssetsStatus
    {
        kAssetsRemoved,
        kAssetsAdded,
        kAssetsNoChange,
        kAssetsInit
    };
 
    class TextToAssets : public Thread
    {
      public:
 
        // struct to hold asset returned by text-to-preset
        struct TTPAsset {
            String path; // path to the asset
            String label; // label for display in XYPad
            String engine; // CoreEngine mode "granular" or "sampler"
        };
 
        TextToAssets();
 
        ~TextToAssets();
 
        std::tuple<String, std::array<TTPAsset, 4> > findPresetFiles(String searchTerm);
 
        std::vector<float> rankify(const std::vector<float>& x) const;
 
        bool processFileList(Array<File>& filesToProcess);
 
        bool appendDataToEmbeddingsFile();
 
        static void searchForTagAndProperty(ValueTree& root, Identifier tag, Identifier property, String newSample,
                                            Identifier target);
 
        static void updateTabLabel(ValueTree& root, int index, const String& label);
 
        static void updateCoreEngineParamValue(ValueTree& tree, const var& coreEngineID, const var& paramID, const String& value);
 
        virtual File createFilePath(String filename);
 
        bool isUCSValid(String catID);
 
        static std::unique_ptr<XmlElement> getInitTTPData();
 
        void setClaudeApiKey(const String& key) { m_claudeApiKeyEncrypted = key; };
 
        class Listener
        {
          public:
            virtual ~Listener() = default;
 
            virtual void onTextToAssetsProcessingStatusChanged(bool isAvailable) = 0;
 
            virtual void onTextToAssetsProgressUpdate(int processedFiles, int totalFiles,
                                                      const String& estimatedTimeRemaining) = 0;
        };
 
        void addListener(Listener* listener);
 
        void removeListener(Listener* listener);
 
        bool isAvailable() { return m_ttaAvailable; };
 
        bool isStrictMode() { return m_strictMode; };
 
        void setStrictMode(bool isStrictMode) { m_strictMode = isStrictMode; };
 
      protected:
        virtual void calculateEmbeddingsForFile(const File& file);
        
        virtual bool isModelFileAvailable() const { return m_modelFileAvailable; }
 
        // model file available
        bool m_modelFileAvailable{false};
 
        // Call listeners synchronously, helper for unit testing
        bool m_synchronousUpdates{false};
 
      private:
        // struct to hold asset metadata and embeddings
        struct Data
        {
            String catID;                  // UCS CatID
            String path;                   // full file path
            std::vector<float> embeddings; // SBERT embedding
            Data(String catID = String(), String path = String(), std::vector<float> embeddings = std::vector<float>())
                : catID(catID), path(path), embeddings(embeddings)
            {
            }
        };
 
        // struct for preset layers
        struct Layer {
            bool isValid;
            float bias;
            String parentCatID;
            String collection;
            String layerName;
            String layerDescription;
            String assetCatID;
            String assetDescription;
            String engine;
            std::vector<float> layerEmbedding;
            std::vector<float> assetEmbedding;
        };
 
        void addBiasToLayers(const String& query, std::vector<Layer>& layers);
 
        StringArray parseAnthropicResponse(std::unique_ptr<InputStream>& stream);
 
        StringArray getLLMLayers(std::unique_ptr<InputStream>& stream);
 
        StringArray anthropicAPICall(String query);
 
        std::vector<float> calculateLLMLayerScores(const std::vector<Layer>& layers, const StringArray& generatedLayerNames);
 
        float dot(const std::vector<float>& a, const std::vector<float>& b) const;
 
        void normalise(std::vector<float>& x) const;
 
        std::vector<float> getVectorScores(String catID, const std::vector<float>& x) const;
 
        std::tuple<String, StringArray> preprocessQuery(String query) const;
 
        std::vector<std::pair<float, int>> topk(std::vector<std::pair<float, int>> data, int topk) const;
 
        std::vector<float> calculateReciprocalRankFusion(const std::vector<float>& scores1,
                                                         const std::vector<float>& scores2, float k = 2.0f,
                                                         float alpha = 0.5f) const;
 
        bool matchesCategory(const String& catID, StringArray tags) const;
 
        void clear();
 
        void run() override;
 
        void loadFileEmbeddings();
 
        bool readFromFile(File file);
 
        String sanitizeString(String text);
 
        String capitalize(const String& text);
 
        void loadPresetLayers();
 
        void loadQueryEmbeddings();
 
        std::tuple<String, String> findClosestAmbienceCatID(const String& query, const std::vector<float>& queryEmbedding);
 
        std::vector<int> argsort(const std::vector<float>& x) const;
 
        bool containsKeyword(const String& text, const StringArray& keywords);
 
        void maskLayersContainingTag(String tag, std::vector<Layer>& layers);
 
        void excludeLayers(const StringArray& excludeList, std::vector<Layer>& layers);
 
        std::tuple<float, int, bool> findLayer(const std::vector<float>& queryEmbedding,
                                               const std::vector<float>& presetEmbedding,
                                               const std::vector<Layer>& layers,
                                               const std::vector<float>& layerScoresLLM);
 
        std::tuple<String, float> findAsset(const Layer& layer);
 
        void notifyAvailabilityStatus(bool isAvailable);
 
        void notifyProgressUpdate(const int processedFiles, const int totalFiles, const String& estimatedTimeRemaining);
 
        // Best Match 25 (BM25) algorithm
        std::unique_ptr<BM25> m_bm25;
 
        // vector to hold corpus of data for BM25 algorithm
        std::vector<String> m_bm25Corpus;
 
        // keep assets organised by UCS CatID
        std::unordered_map<String, std::vector<Data> > m_datasetMap;
        std::unordered_map<String, std::vector<std::size_t> > m_datasetIDMap;
 
        // map to keep track of how many assets a user has for each UCS CatID
        std::map<String, int> m_assetCounter;
 
        // asset sample paths
        StringArray m_samplePaths;
 
        // asset file names
        StringArray m_sampleFilenames;
 
        // indicates should save the file
        bool m_shouldSave = true;
 
        // the sentence transformer (SBERT)
        SentenceTransformer m_sentenceTransformer;
 
        // class that holds UCS CatIDs
        SharedResourcePointer<UniversalCategorySystem> m_UCS;
 
        // critical section for thread safety
        CriticalSection m_cs;
 
        // File processing is in progress
        bool m_ttaAvailable{true};
 
        // stores SBERT embeddings and associated labels for classifying a query
        std::vector<std::pair<String, std::vector<float> > > m_queryEmbeddings;
 
        // organises preset layers by ambience subcategory
        std::unordered_map<String, std::vector<Layer> > m_layersMap;
 
        String m_claudeApiKeyEncrypted;
 
        ListenerList<Listener> m_listeners;
 
        // Async listener updaters for status and progress notifications
        // These are used to coalesce multiple updates
        // to reduce spamming the message thread during file processing
        AsyncListenerUpdater<Listener> m_statusUpdater, m_progressUpdater;
 
        // Strict mode disables bias in layer selection, potentially reducing output likelihood.
        bool m_strictMode{false};
    };
 
    class TextToAssetsShared
    {
      public:
        SharedResourcePointer<TextToAssets> textToAssets;
    };
 
    class AnthropicAPIException : public std::runtime_error {
    public:
        AnthropicAPIException(const String& message) : std::runtime_error(message.toStdString()) {}
    };
 
    class LLMJsonParseException : public std::runtime_error {
    public:
        LLMJsonParseException(const String& message) : std::runtime_error(message.toStdString()) {}
    };
 
} // namespace krotos