TextToAssets.cpp

Go to the documentation of this file.

#include "TextToAssets.h"
 
namespace krotos
{
    TextToAssets::TextToAssets() : Thread("TTAThread"), m_statusUpdater(m_listeners), m_progressUpdater(m_listeners)
    {
        File ggmlDataFile = File(krotos::utils::StringsIntoPath(AssetManager::getPluginDirectory().getFullPathName(),
                                                                "ttpResources", modelFileName));
 
        if (ggmlDataFile.existsAsFile())
        {
            m_modelFileAvailable = true;
        }
 
        startThread();
    }
 
    TextToAssets::~TextToAssets()
    {
        try
        {
            stopThread(4000);
        }
        catch (const std::exception& e)
        {
            // Handle exception a
            Logger::writeToLog("Error stopping thread: " + String(e.what()));
        }
    }
 
    StringArray TextToAssets::parseAnthropicResponse(std::unique_ptr<InputStream>& stream)
    {
        if (!stream)
        {
            throw AnthropicAPIException("Invalid stream from Anthropic API");
        }
 
        auto result = stream->readEntireStreamAsString();
        auto json = JSON::parse(result);
 
        if (json.isVoid())
        {
            throw AnthropicAPIException("Error parsing JSON returned by Anthropic API");
        }
 
        var content = json.getProperty(contentIdentifier, var());
        if (!content.isArray() || content.size() != 1)
        {
            throw AnthropicAPIException("Unexpected content structure in JSON returned by Anthropic API");
        }
 
        auto text = content[0].getProperty(textIdentifier, var()).toString();
        auto jsonText = JSON::parse(text);
 
        if (jsonText.isVoid())
        {
            throw LLMJsonParseException("Error parsing JSON returned by LLM");
        }
 
        // Optimized layer parsing for JUCE StringArray
        StringArray layers;
        layers.ensureStorageAllocated(4); // Pre-allocate space for 4 layers
 
        static const std::array<Identifier, 4> layerIdentifiers = {layer1Identifier, layer2Identifier, layer3Identifier,
                                                                   layer4Identifier};
 
        for (const auto& identifier : layerIdentifiers)
        {
            auto layerValue = jsonText.getProperty(identifier, var()).toString();
            if (layerValue.isEmpty())
            {
                throw LLMJsonParseException("Missing or empty layer in LLM response: " + identifier.toString());
            }
            layers.add(layerValue);
        }
 
        return layers;
    }
 
    StringArray TextToAssets::getLLMLayers(std::unique_ptr<InputStream>& stream)
    {
        StringArray layers;
 
        try
        {
            layers = parseAnthropicResponse(stream);
            // Process layers
            for (int i = 0; i < layers.size(); ++i)
            {
                DBG("LLM Layer " + String(i + 1) + ": " + layers[i]);
            }
        }
        catch (const AnthropicAPIException& e)
        {
            // Log to Sentry: Chris b0rked the Anthropic API
            DBG("Anthropic API Error: " + String(e.what()));
            // Report to Sentry here
        }
        catch (const LLMJsonParseException& e)
        {
            // Log to Sentry: LLM JSON parsing error from Chris
            DBG("LLM JSON Parse Error: " + String(e.what()));
            // Report to Sentry here
        }
        catch (const std::exception& e)
        {
            // Log to Sentry: Any other unexpected error
            DBG("Unexpected Error: " + String(e.what()));
            // Report to Sentry here
        }
 
        return layers;
    }
 
    StringArray TextToAssets::anthropicAPICall(String query)
    {
        URL url("https://api.anthropic.com/v1/messages");
 
        // Decrypt the Claude API key
        const int ivLength{32};
        const int claudeApiKeyLength{108};
        // Output vectors for converted decryption input arguments
        std::vector<unsigned char> ivBytes, ciphertextBytes, encryptionKeyBytes;
        // Output vector for the decrypted key plaintext
        std::vector<unsigned char> plaintextBytes;
        // The encryption key used to encrypt the Claude API key in hex format.
        const std::string encKey("23c3b67142161bb753706f4673f234906a811e10cf434e832790a4f120ab90e1");
 
        // Extract the IV and ciphertext
        auto ivHex = m_claudeApiKeyEncrypted.substring(0, ivLength).toStdString();
        auto ciphertextHex = m_claudeApiKeyEncrypted.substring(ivLength).toStdString();
 
        // Convert from hex to byte array which can be passed to EVP
        try
        {
            utils::hexStringToByteArray(ivHex, ivBytes);
            utils::hexStringToByteArray(ciphertextHex, ciphertextBytes);
            utils::hexStringToByteArray(encKey, encryptionKeyBytes);
        }
        catch (const std::invalid_argument& e)
        {
            DBG("TextToAssets: Failed to open Claude API Key\n\t" + String(e.what()));
            jassertfalse;
            return StringArray();
        }
 
        // Perform the decrypt operation
        EVP_CIPHER_CTX* cipherContext;
        int outputLen;
        int plaintextLen;
 
        plaintextBytes.resize(ciphertextBytes.size());
 
        // Create and initialise the context
        if (!(cipherContext = EVP_CIPHER_CTX_new()))
        {
            DBG("TextToAssets: Could not create new EVP context");
        }
 
        // Initialize the decryption operation
        if (EVP_DecryptInit_ex(cipherContext, EVP_aes_256_cbc(), NULL, encryptionKeyBytes.data(), ivBytes.data()) != 1)
        {
            DBG("TextToAssets: Could not initialise EVP context with the given parameters");
        }
 
        // Perform decryption
        if (EVP_DecryptUpdate(cipherContext, plaintextBytes.data(), &outputLen, ciphertextBytes.data(),
                              ciphertextBytes.size()) != 1)
        {
            DBG("TextToAssets: Could not perform decryption");
        }
        plaintextLen = outputLen;
 
        // Finalize decryption
        if (EVP_DecryptFinal_ex(cipherContext, plaintextBytes.data() + outputLen, &outputLen) != 1)
        {
            DBG("TextToAssets: Could not finalise decryption");
        }
        plaintextLen += outputLen;
 
        // Clean up
        EVP_CIPHER_CTX_free(cipherContext);
 
        // Key bytes to string
        std::string apiKey(plaintextBytes.begin(), plaintextBytes.end());
        apiKey = apiKey.substr(0, claudeApiKeyLength);
 
        // Construct the request body with the conversation format
        String requestBody = R"({
        "model": "claude-3-haiku-20240307",
        "max_tokens": 1024,
        "temperature": 0,
        "system": "Your task is to design an ambience preset for a given soundscape.\nAmbience presets should contain 4 layers.\nEach layer should contribute something unique and meaningful to the preset while being relevent to the requested soundscape.\nExample layers: Birds, Wind, Rain, Stream, Crickets, Animal calls, Waterfall, Campfire, Thunder\nHere is an example ambience preset in JSON format:\n{\"Soundscape\": \"Forest\", \"Layer-1\": \"Birds\", \"Layer-2\": \"Wind\", \"Layer-3\": \"Crickets\", \"Layer-4\": \"Stream\"}\nReturn the designed preset in JSON format and no other output.",
        "messages": [
            {"role": "user", "content": "{query}"}
        ]
        })";
 
        // swap out placeholder {query} with the actual user query
        requestBody = requestBody.replace("{query}", query);
        url = url.withPOSTData(requestBody);
 
        // create a stream for the response
        // TODO: https://krotosltd.atlassian.net/browse/KST-2608
        StringPairArray responseHeaders;
        int statusCode = 0;
        auto stream = url.createInputStream(
                URL::InputStreamOptions(URL::ParameterHandling::inAddress)
                        .withHttpRequestCmd("POST")
                        .withExtraHeaders({"x-api-key: " + apiKey +
                                           "\nanthropic-version: 2023-06-01\ncontent-type: application/json"})
                        .withConnectionTimeoutMs(10000)
                        .withResponseHeaders(&responseHeaders)
                        .withStatusCode(&statusCode));
 
        auto layers = getLLMLayers(stream);
        return layers;
    }
 
    void TextToAssets::loadPresetLayers()
    {
        // load json from binary data
        const char* jsonBinaryData = reinterpret_cast<const char*>(KrotosBinaryData::preset_layers_json);
 
        // convert to string
        String jsonString(jsonBinaryData);
 
        // and parse it
        auto parsedData = JSON::parse(jsonString);
 
        if (!parsedData.isVoid())
        {
            const auto presetLayersIdentifier = Identifier("presetLayers");
            const auto parentCatIDIdentifier = Identifier("parentCatID");
            const auto collectionIdentifier = Identifier("collection");
            const auto layerNameIdentifier = Identifier("layerName");
            const auto layerDescriptionIdentifier = Identifier("layerDescription");
            const auto assetCatIDIdentifier = Identifier("assetCatID");
            const auto assetDescriptionIdentifier = Identifier("assetDescription");
            const auto engineIdentifier = Identifier("engine");
            const auto layerEmbeddingIdentifier = Identifier("layerEmbedding");
            const auto assetEmbeddingIdentifier = Identifier("assetEmbedding");
 
            var result = parsedData.getProperty(presetLayersIdentifier, 0);
            for (int i = 0; i < result.size(); ++i)
            {
                Layer layer;
                layer.isValid = true;
                layer.bias = 0.0f;
                layer.parentCatID = result[i].getProperty(parentCatIDIdentifier, 0).toString();
                layer.collection = result[i].getProperty(collectionIdentifier, 0).toString();
                layer.layerName = result[i].getProperty(layerNameIdentifier, 0).toString();
                layer.layerDescription = result[i].getProperty(layerDescriptionIdentifier, 0).toString();
                layer.assetCatID = result[i].getProperty(assetCatIDIdentifier, 0).toString();
                layer.assetDescription = result[i].getProperty(assetDescriptionIdentifier, 0).toString();
                layer.engine = result[i].getProperty(engineIdentifier, 0).toString();
 
                // layer embedding
                auto layerEmbeddingArray = result[i].getProperty(layerEmbeddingIdentifier, 0);
                layer.layerEmbedding.resize(layerEmbeddingArray.size(), 0.0f);
                for (int j = 0; j < layerEmbeddingArray.size(); ++j)
                {
                    layer.layerEmbedding[j] = layerEmbeddingArray[j];
                }
 
                // asset embedding
                auto assetEmbeddingArray = result[i].getProperty(assetEmbeddingIdentifier, 0);
                layer.assetEmbedding.resize(assetEmbeddingArray.size(), 0.0f);
                for (int j = 0; j < assetEmbeddingArray.size(); ++j)
                {
                    layer.assetEmbedding[j] = assetEmbeddingArray[j];
                }
 
                // store the layers grouped by parent CatID (e.g. keep ambience forest layers together)
                m_layersMap[layer.parentCatID].push_back(layer);
            }
        }
        else
        {
            DBG("Failed to load KrotosBinaryData::preset_layers_json");
            jassertfalse;
        }
    }
 
    void TextToAssets::loadQueryEmbeddings()
    {
        // load json from binary data
        const char* jsonBinaryData = reinterpret_cast<const char*>(KrotosBinaryData::query_embeddings_json);
 
        // convert to string
        String jsonString(jsonBinaryData);
 
        // and parse it
        auto parsedData = JSON::parse(jsonString);
 
        if (!parsedData.isVoid())
        {
            const auto queryEmbeddingsIdentifier = Identifier("queryEmbeddings");
            const auto catIDIdentifier = Identifier("catID");
            const auto embeddingIdentifier = Identifier("embedding");
 
            var result = parsedData.getProperty(queryEmbeddingsIdentifier, 0);
            m_queryEmbeddings.reserve(result.size());
            for (int i = 0; i < result.size(); ++i)
            {
                auto catID = result[i].getProperty(catIDIdentifier, 0).toString();
                auto embeddingArray = result[i].getProperty(embeddingIdentifier, 0);
                std::vector<float> embedding(embeddingArray.size(), 0.0f);
                for (int j = 0; j < embeddingArray.size(); ++j)
                {
                    embedding[j] = embeddingArray[j];
                }
                m_queryEmbeddings.push_back(std::make_pair(catID, embedding));
            }
        }
        else
        {
            DBG("Failed to load KrotosBinaryData::query_embeddings_json");
            jassertfalse;
        }
    }
 
    std::tuple<String, String> TextToAssets::findClosestAmbienceCatID(const String& query,
                                                                      const std::vector<float>& queryEmbedding)
    {
        if (m_queryEmbeddings.empty())
        {
            return {"", ""};
        }
 
        // TODO: exclude ambience subcategories in a separate method
        // forest and tropical subcategories easily confused, so discount tropical unless keyword detected
        bool isTropical = query.contains("tropical");
 
        String CatID = "";
        auto maxScore = std::numeric_limits<float>::lowest();
        std::vector<float> maxEmbedding(queryEmbedding.size(), 0.0f);
        for (const auto& element : m_queryEmbeddings)
        {
            const auto name = element.first;
            if ((!isTropical) && (name == "AMBTrop"))
            {
                continue;
            }
 
            const auto embedding = element.second;
            const auto score = dot(embedding, queryEmbedding);
            if (score >= maxScore)
            {
                CatID = name;
                maxScore = score;
                maxEmbedding = embedding;
            }
        }
 
        // check if a combination of two ambience subcategories significantly improves the score
        String CatID2 = "";
        float maxScore2 = maxScore;
        for (const auto& element : m_queryEmbeddings)
        {
            const auto name = element.first;
            if ((!isTropical) && (name == "AMBTrop"))
            {
                continue;
            }
 
            // sum the current embedding with the maxEmbedding we found earlier.
            // normalise the combined embedding and calculate the similarity score against the query.
            auto embedding = element.second;
            std::transform(embedding.begin(), embedding.end(), maxEmbedding.begin(), embedding.begin(), std::plus<>{});
            normalise(embedding);
            const auto score = dot(embedding, queryEmbedding);
            if (score >= maxScore2)
            {
                CatID2 = name;
                maxScore2 = score;
            }
        }
 
        /* KST-2686 - it is possible for a category to be chosen where the layer which will be added contains a sample
         * not within the embeddings/on the users system, so deal with that here */
        if (m_datasetMap.count(CatID2) == 0)
        {
            return {CatID, ""};
        }
 
        constexpr float threshold = 0.1f;
        const auto isValid = maxScore2 >= (maxScore + threshold) ? true : false;
        CatID2 = isValid ? CatID2 : "";
 
        return {CatID, CatID2};
    }
 
    std::vector<int> TextToAssets::argsort(const std::vector<float>& x) const
    {
        std::vector<int> indices(x.size());
        std::iota(indices.begin(), indices.end(), 0);
        std::sort(indices.begin(), indices.end(), [&x](int left, int right) -> bool { return x[left] > x[right]; });
        return indices;
    }
 
    std::tuple<float, int, bool> TextToAssets::findLayer(const std::vector<float>& queryEmbedding,
                                                         const std::vector<float>& presetEmbedding,
                                                         const std::vector<Layer>& layers,
                                                         const std::vector<float>& layerScoresLLM)
    {
        auto layerScore = std::numeric_limits<float>::lowest();
        int layerIdx = 0;
        bool isValid = false;
        for (int i = 0; i < (int)layers.size(); ++i)
        {
            auto layer = layers[i];
            if (!layer.isValid)
            {
                continue;
            }
 
            isValid = true; // we found at least one valid layer
            auto embedding = layer.layerEmbedding;
 
            // sum preset embedding with current layer embeddings (the preset embedding)
            std::transform(embedding.begin(), embedding.end(), presetEmbedding.begin(), embedding.begin(),
                           std::plus<>{});
 
            // normalise embedding so dot product will give us cosine similarity
            normalise(embedding);
 
            // this is comparing query and combined preset embedding
            auto simScore = dot(embedding, queryEmbedding);
 
            // use LLM scores to bias the layer selection
            // e.g. the query "forest storm" gives generated names "wind", "thunder", "rain" and "rustling leaves"
            // our layers with similar names will have higher similarity scores, and become more likely to be selected
            // we use a weighted score as we still care about how the combined preset embedding matches the query
 
            // combine score with match against LLM generated layers
            simScore = 0.5f * simScore + 0.5f * layerScoresLLM[i];
 
            if (simScore >= layerScore)
            {
                layerScore = simScore;
                layerIdx = i;
            }
        }
 
        return {layerScore, layerIdx, isValid};
    }
 
    std::tuple<String, float> TextToAssets::findAsset(const Layer& layer)
    {
        const auto catID = layer.assetCatID;
        const auto description = layer.assetDescription;
        const auto embedding = layer.assetEmbedding;
 
        if (m_datasetMap.count(catID) == 0)
        {
            return {"", 0.0f}; // no assets under that CatID
        }
 
        // cosine similarity between layer embedding and file embeddings
        const auto vectorScores = getVectorScores(catID, embedding);
 
        // BM25 scores for layer label and filenames
        const auto ids = m_datasetIDMap[catID];
        const auto bm25Scores = m_bm25->getBatchScores(description, ids);
 
        // use Reciprocal Rank Fusion (RRF) to combine embedding scores and BM25 scores
        const auto rrfScores = calculateReciprocalRankFusion(vectorScores, bm25Scores);
 
        auto maxScore = std::numeric_limits<float>::lowest();
        std::size_t maxIdx = 0;
        auto cosineScore = std::numeric_limits<float>::lowest();
        for (std::size_t i = 0; i < ids.size(); ++i)
        {
            auto score = rrfScores[i];
            if (score >= maxScore)
            {
                maxScore = score;
                maxIdx = i;
                cosineScore = vectorScores[i];
            }
        }
        const auto path = m_datasetMap[catID][maxIdx].path;
        return {path, cosineScore};
    }
 
    bool TextToAssets::containsKeyword(const String& text, const StringArray& keywords)
    {
        for (const auto& keyword : keywords)
        {
            if (text.contains(keyword))
            {
                return true;
            }
        }
        return false;
    }
 
    void TextToAssets::maskLayersContainingTag(String tag, std::vector<Layer>& layers)
    {
        std::for_each(layers.begin(), layers.end(), [tag](Layer& layer) {
            layer.isValid = (layer.assetCatID.contains(tag)) ? false : layer.isValid;
        });
    }
 
    String TextToAssets::capitalize(const String& text)
    {
        return text.substring(0, 1).toUpperCase() + text.substring(1).toLowerCase();
    }
 
    std::vector<float> TextToAssets::calculateLLMLayerScores(const std::vector<Layer>& layers,
                                                             const StringArray& generatedLayerNames)
    {
        ScopedLock sl(m_cs);
 
        std::vector<std::vector<float>> embeddings;
        embeddings.reserve(generatedLayerNames.size());
        for (const auto& name : generatedLayerNames)
        {
            auto embedding = m_sentenceTransformer.encode(name.toStdString());
            embeddings.push_back(embedding);
        }
 
        std::vector<float> layerScoresLLM;
        layerScoresLLM.reserve(layers.size());
        for (const auto& layer : layers)
        {
            const auto layerEmbedding = layer.layerEmbedding;
            const auto layerBias = layer.bias;
            float maxScore = std::numeric_limits<float>::lowest();
            for (std::size_t i = 0; i < generatedLayerNames.size(); ++i)
            {
                auto score = dot(embeddings[i], layerEmbedding) + layerBias;
                maxScore = std::max(maxScore, score);
            }
            layerScoresLLM.push_back(maxScore);
        }
        return layerScoresLLM;
    }
 
    void TextToAssets::addBiasToLayers(const String& query, std::vector<Layer>& layers)
    {
        // Map of collections to their associated keywords
        const std::unordered_map<String, std::vector<String>> collectionKeywords = {
                {"aeroplane interior", {"aeroplane interior", "airplane interior", "commercial flight"}},
                {"airport", {"airport"}},
                {"apartment", {"apartment"}},
                {"basement", {"basement"}},
                {"bathroom", {"bathroom"}},
                {"beach", {"beach", "seaside"}},
                {"bus interior", {"bus interior"}},
                {"city", {"city", "urban"}},
                {"construction", {"construction", "building site"}},
                {"countryside", {"countryside", "rural"}},
                {"desert", {"desert"}},
                {"forest", {"forest", "woodland"}},
                {"garage", {"garage"}},
                {"hospital", {"hospital"}},
                {"hotel", {"hotel"}},
                {"kitchen", {"kitchen"}},
                {"office", {"office", "workplace"}},
                {"park", {"park"}},
                {"restaurant", {"restaurant", "cafe"}},
                {"sci-fi", {"starship", "spaceship"}},
                {"sewer", {"sewer"}},
                {"suburban", {"suburban", "suburbs"}},
                {"subway", {"subway"}},
                {"swamp", {"swamp", "marsh"}},
                {"town", {"town", "village"}},
                {"underwater", {"underwater"}}};
 
        // Vector to store detected collections
        std::vector<String> detectedCollections;
 
        // Check for keywords in the query that match specific KS preset keywords
        for (const auto& [collection, keywords] : collectionKeywords)
        {
            // 'collection' is a structured binding so it must be copied to a variable for lambda capture
            auto currentCollection = collection;
            bool collectionDetected =
                    std::any_of(keywords.begin(), keywords.end(), [&query, &currentCollection](const String& keyword) {
                        // Special case for "desert": check for whole word
                        bool isDetected = (currentCollection == "desert") ? query.containsWholeWord(keyword)
                                                                          : query.contains(keyword);
                        if (isDetected)
                        {
                            // Log detected collections
                            DBG("[collections] " + currentCollection);
                        }
                        return isDetected;
                    });
            if (collectionDetected)
            {
                detectedCollections.push_back(collection);
            }
        }
 
        // Use a non-zero bias for layers in any detected collections
        // The bias is added to the cosine similarity score during layer selection
        // This makes layers in a collection more likely to be selected
        // e.g. a query of "quiet office" will be more likely to select layers assigned to office
        for (auto& layer : layers)
        {
            if (std::find(detectedCollections.begin(), detectedCollections.end(), layer.collection) !=
                detectedCollections.end())
            {
                // We want the bias to be just large enough to influence layer selection
                // but not so large that relevant layers outside the collection get excluded
                layer.bias = 0.25f;
            }
        }
    }
 
    std::tuple<String, std::array<TextToAssets::TTPAsset, 4>> TextToAssets::findPresetFiles(String searchTerm)
    {
        ScopedLock sl(m_cs);
 
        // init results
        std::array<TTPAsset, 4> results;
 
        // this threshold is used to determine if we found a suitable asset
        constexpr float cosineSimilarityThreshold = 0.5f;
 
        // minimal preprocessing of the user query
        auto [query, excludeList] = preprocessQuery(searchTerm);
 
        // compute the SBERT embedding for the query
        auto queryEmbedding = m_sentenceTransformer.encode(query.toStdString());
 
        // classification using nearest neighbour
        auto [closestCatID, secondaryCatID] = findClosestAmbienceCatID(query, queryEmbedding);
        if (closestCatID.isEmpty())
        {
            return {"", results};
        }
        auto categorySubCategory = m_UCS->getCategorySubCategory(closestCatID);
 
        DBG("[closestCatID] " << closestCatID);
        DBG("[secondaryCatID] " << secondaryCatID);
 
        std::vector<Layer> layers;
 
        if (isStrictMode())
        {
            // use a subset of T2P layers based on classifying query to closest matching ambience subcategory
            auto layers = m_layersMap[closestCatID];
            if (layers.empty())
            {
                return {categorySubCategory, results};
            }
 
            // add additional layers
            if (secondaryCatID.isNotEmpty())
            {
                const auto secondaryLayers = m_layersMap[secondaryCatID];
                layers.insert(layers.end(), secondaryLayers.begin(), secondaryLayers.end());
            }
 
            // print available layers
            for (const auto& layer : layers)
            {
                DBG(layer.layerName << ": " << layer.layerDescription);
            }
        }
        else
        {
            // use all available T2P layers
            for (auto [categoryName, categoryLayers] : m_layersMap)
            {
                layers.insert(layers.end(), categoryLayers.begin(), categoryLayers.end());
            }
 
            // add non-zero bias to relevant layers if the query matches specific KS presets
            addBiasToLayers(query, layers);
        }
 
        // -----------------------------------
        // layer names generated by an LLM
        // -----------------------------------
        auto generatedLayerNames = anthropicAPICall(searchTerm);
        if (generatedLayerNames.size() != 4)
        {
            DBG("LLM generation failed");
            return {"", results};
        }
 
        // score our layers based on the generated LLM names
        auto layerScoresLLM = calculateLLMLayerScores(layers, generatedLayerNames);
        // -----------------------------------
 
        // initially all the layers should be valid
        std::for_each(layers.begin(), layers.end(), [](Layer& layer) { layer.isValid = true; });
 
        // exclude layers based on query containing -tag
        excludeLayers(excludeList, layers);
 
        // initialise the preset embedding to zeros
        std::vector<float> presetEmbedding(queryEmbedding.size(), 0.0f);
 
        // we'll store paths, labels and core engine mode for each layer in the preset
        std::set<String> uniqueAssets;
        const int maximumSearchIterations = 20;
        std::size_t resultsIndex = 0;
        for (int i = 0; i < maximumSearchIterations; ++i)
        {
            // find a layer from the valid layers
            auto [layerScore, layerIdx, layerValid] =
                    findLayer(queryEmbedding, presetEmbedding, layers, layerScoresLLM);
            if (!layerValid)
            {
                // we didn't find a single suitable layer
                break;
            }
            auto layer = layers[layerIdx];
            auto name = layer.layerName;
 
            // set layers with the same name as invalid so we don't end up selecting duplicates
            std::for_each(layers.begin(), layers.end(),
                          [name](Layer& layer) { layer.isValid = (layer.layerName == name) ? false : layer.isValid; });
 
            // find an asset for the selected layer
            auto [asset, assetScore] = findAsset(layer);
 
            // check for issues with the selected asset
            if (assetScore < cosineSimilarityThreshold)
            {
                // we didn't find a suitable local asset - continue searching
                DBG("missing local asset for layer with name: " << name);
                continue;
            }
            else if (uniqueAssets.contains(asset))
            {
                // we already have a preset layer that uses this asset
                DBG("skipping asset as already selected: " << asset);
                continue;
            }
 
            // sum the selected layer embedding with previous ones
            std::transform(presetEmbedding.begin(), presetEmbedding.end(), layer.layerEmbedding.begin(),
                           presetEmbedding.begin(), std::plus<>{});
 
            // store the layer result
            results[resultsIndex].path = asset;
            results[resultsIndex].label = capitalize(name);
            results[resultsIndex].engine = layer.engine;
 
            // insert asset into set to check for duplicates
            uniqueAssets.insert(asset);
 
            // increment index
            ++resultsIndex;
 
            // check if we're done
            if (resultsIndex == results.size())
            {
                break;
            }
        }
 
        return {categorySubCategory, results};
    }
 
    bool TextToAssets::isUCSValid(String catID)
    {
        jassert(m_UCS != nullptr);
        return m_UCS->isValid(catID);
    }
 
    void TextToAssets::run()
    {
        if (!threadShouldExit())
        {
            loadQueryEmbeddings(); // embeddings used to classify the query into an ambience subcategory
            loadPresetLayers();    // sets of named preset layers for supported ambience subcategory
            loadFileEmbeddings();
        }
    }
 
    File TextToAssets::createFilePath(String filename)
    {
        return utils::StringsIntoPath(AssetManager::getPluginDirectory().getFullPathName(), "ttpResources", filename);
    }
 
    bool TextToAssets::appendDataToEmbeddingsFile()
    {
        // clear asset counter
        m_assetCounter.clear();
 
        // clear BM25 corpus vector
        m_bm25Corpus.clear();
 
        // load existing embeddings
        auto embeddingsFile = createFilePath(ttfEmbeddingsFileName);
 
        std::unordered_map<String, std::vector<Data>> datasetMap;
 
        if (embeddingsFile.existsAsFile())
        {
            auto json = JSON::parse(embeddingsFile);
            var result = json.getProperty(FilesEmbeddingsIdentifier, 0);
            for (int i = 0; i < result.size(); ++i)
            {
                auto fileName = result[i].getProperty(fileNameIdentifier, 0).toString();
                auto catID = result[i].getProperty(catIDNameIdentifier, 0).toString();
                var embeddingsArray = result[i].getProperty(ttfEmbeddingsNameIdentifier, 0);
 
                std::vector<float> embeddings;
                for (int j = 0; j < embeddingsArray.size(); ++j)
                {
                    embeddings.push_back(embeddingsArray[j]);
                }
 
                Data data(catID, fileName, embeddings);
                datasetMap[catID].push_back(data);
            }
        }
        else
        {
            embeddingsFile.create();
        }
 
        // append new ones to the data
        for (auto const& [key, dataset] : m_datasetMap)
        {
            for (const auto& element : dataset)
            {
                datasetMap[key].push_back(element);
            }
        }
 
        // write the new dict to the embeddings file
        DynamicObject* datasetObject = new DynamicObject();
        var datasetObjectVar(datasetObject);
        Array<var> rows;
 
        for (auto const& [key, dataset] : datasetMap)
        {
            for (const auto& element : dataset)
            {
                const auto fileName = element.path;
                const auto catID = element.catID;
                const auto embeddings = element.embeddings;
 
                DynamicObject* metadata = new DynamicObject();
                var metadataVar(metadata);
                metadata->setProperty(fileNameIdentifier, var(fileName));
 
                metadata->setProperty(catIDNameIdentifier, var(catID));
 
                Array<var> embeddingsArray;
                for (auto value : embeddings)
                {
                    embeddingsArray.add(value);
                }
                metadata->setProperty(ttfEmbeddingsNameIdentifier, embeddingsArray);
 
                rows.add(metadata);
                metadata = nullptr;
            }
        }
        datasetObject->setProperty(FilesEmbeddingsIdentifier, rows);
 
        // need to have all data loaded and updated in order to perform search
        m_datasetMap = datasetMap;
 
        TemporaryFile tempEmbeddingsFile(embeddingsFile, TemporaryFile::useHiddenFile);
        if (auto stream = tempEmbeddingsFile.getFile().createOutputStream())
        {
            // overwrite an existing file
            stream->setPosition(0);
            stream->truncate();
            JSON::writeToStream(*stream, datasetObject);
            stream->flush();
 
            // close the output stream, so we can switch out the temp file.
            stream.reset();
 
            const bool success = tempEmbeddingsFile.overwriteTargetFileWithTemporary();
 
            if (!success)
            {
                // Failed to overwrite!
                jassertfalse;
                return false;
            }
        }
        else
        {
            return false;
        }
 
        readFromFile(embeddingsFile);
 
        return true;
    }
 
    void TextToAssets::loadFileEmbeddings()
    {
        ScopedLock sl(m_cs);
        // load tags/embeddings if available
        auto path = createFilePath(ttfEmbeddingsFileName);
        if (path.existsAsFile())
        {
            // clear previously stored data
            readFromFile(path);
        }
    }
 
    String TextToAssets::sanitizeString(String text)
    {
        text = text.replace("_", " ");
        text = text.replace("RXd", "");
        if (text.contains("SNDBTS"))
        {
            text = text.upToFirstOccurrenceOf("SNDBTS", false, false);
        }
        text = text.removeCharacters("0123456789");
        text = text.trim();
        text = text.toLowerCase();
        return text;
    }
 
    bool TextToAssets::readFromFile(File file)
    {
        if (file.existsAsFile() == false)
        {
            return false;
        }
 
        clear();
 
        auto json = JSON::parse(file);
        var result = json.getProperty(FilesEmbeddingsIdentifier, 0);
        auto resultSize = result.size();
        for (int i = 0; i < resultSize; ++i)
        {
            auto fileName = result[i].getProperty(fileNameIdentifier, 0).toString();
            auto catID = result[i].getProperty(catIDNameIdentifier, 0).toString();
            var embeddingsArray = result[i].getProperty(ttfEmbeddingsNameIdentifier, 0);
 
            std::vector<float> embeddings;
            for (int j = 0; j < embeddingsArray.size(); ++j)
            {
                embeddings.push_back(embeddingsArray[j]);
            }
 
            auto name = File(fileName).getFileNameWithoutExtension();
            name = name.replace(catID, "");
            name = sanitizeString(name);
            m_datasetIDMap[catID].push_back(m_bm25Corpus.size());
            m_bm25Corpus.push_back(name);
            ++m_assetCounter[catID];
 
            Data data(catID, fileName, embeddings);
            m_datasetMap[catID].push_back(data);
        }
 
        // init BM25 class so we can rank files
        if (!m_bm25Corpus.empty())
        {
            m_bm25 = std::make_unique<BM25>(m_bm25Corpus);
        }
 
        return true;
    }
 
    void TextToAssets::clear()
    {
        m_datasetMap.clear();
        m_datasetIDMap.clear();
        m_assetCounter.clear();
        m_bm25Corpus.clear();
        m_samplePaths.clear();
        m_sampleFilenames.clear();
    }
 
    bool TextToAssets::processFileList(Array<File>& filesToProcess)
    {
        if (!isModelFileAvailable())
        {
            return false;
        }
        // lock
        ScopedLock sl(m_cs);
 
        notifyAvailabilityStatus(false);
 
        // clear previously stored data
        clear();
 
        // Time estimation variables
        const int totalFiles = filesToProcess.size();
        int processedFiles = 0;
        const Time startTime = Time::getCurrentTime();
        Time lastUpdateTime = startTime;
 
        for (File& file : filesToProcess)
        {
            calculateEmbeddingsForFile(file);
            if (Thread::currentThreadShouldExit())
            {
                clear();
 
                // Notify processing status update change
                notifyAvailabilityStatus(true);
                return false;
            }
 
            // Update time estimation
            ++processedFiles;
            const Time currentTime = Time::getCurrentTime();
            const RelativeTime elapsedTime = currentTime - startTime;
            const RelativeTime timeSinceLastUpdate = currentTime - lastUpdateTime;
 
            // Check if 500ms have passed since the last update
            if (timeSinceLastUpdate.inMilliseconds() >= 500)
            {
                const double averageTimePerFile = elapsedTime.inSeconds() / processedFiles;
                const int remainingFiles = totalFiles - processedFiles;
                const double estimatedRemainingSeconds = averageTimePerFile * remainingFiles;
 
                // Convert estimated remaining time to a readable string
                const String estimatedTimeRemaining = RelativeTime(estimatedRemainingSeconds).getDescription();
 
                // Notify listeners of progress and estimated time
                // Todo: KST-2723 This has been disabled due to memory corruption crashing issues. Reinstate once fixed.
                // notifyProgressUpdate(processedFiles, totalFiles, estimatedTimeRemaining);
 
                // Update the last update time
                lastUpdateTime = currentTime;
            }
        }
 
        // init BM25 class so we can rank files
        if (!m_bm25Corpus.empty())
        {
            m_bm25 = std::make_unique<BM25>(m_bm25Corpus);
        }
 
        notifyAvailabilityStatus(true);
 
        // append - save embeddings to file
        auto appendSuccessful = appendDataToEmbeddingsFile();
        return appendSuccessful;
    }
 
    void TextToAssets::calculateEmbeddingsForFile(const File& file)
    {
        ScopedLock sl(m_cs);
 
        if (m_UCS == nullptr)
        {
            return;
        }
 
        auto name = file.getFileNameWithoutExtension();
        auto catID = name.upToFirstOccurrenceOf("_", false, false);
        catID = (catID == "AMBROOM") ? "AMBRoom" : catID; // a fix for basement preset assets
        catID = (catID == "AMBUrb") ? "AMBUrbn" : catID;  // a fix for skatepark preset assets
        const auto valid = isUCSValid(catID);
        jassert(valid); // we should already have filtered out invalid files
 
        // TODO: ideally kaf files would include metadata
        const auto sampleRate = 48000;
        const auto bitDepth = 16;
        const auto channelCount = 2;
        const auto durationSeconds = 5; // TODO: determine suitable minimum file duration
        const auto bytesThreshold = (bitDepth / 8) * channelCount * (durationSeconds * sampleRate);
        auto sizeInBytes = file.getSize(); // proxy for duration
        if (sizeInBytes >= bytesThreshold)
        {
            name = name.replace(catID, ""); // remove leading UCS CatID tag
            name = sanitizeString(name);
            auto embedding = m_sentenceTransformer.encode(name.toStdString());
 
            // update dataset
            Data data(catID, file.getFullPathName().toStdString(), embedding);
            m_datasetMap[catID].push_back(data);
 
            // update asset counter
            ++m_assetCounter[catID];
 
            // update BM25 docs
            m_datasetIDMap[catID].push_back(m_bm25Corpus.size());
            m_bm25Corpus.push_back(name);
        }
    }
 
    bool TextToAssets::matchesCategory(const String& catID, StringArray tags) const
    {
        for (auto tag : tags)
        {
            if (catID.startsWith(tag))
                return true;
        }
        return false;
    }
 
    std::vector<float> TextToAssets::rankify(const std::vector<float>& x) const
    {
        const auto n = (int)x.size();
 
        // rank vector
        std::vector<float> R(n, 0);
 
        std::vector<std::pair<float, int>> T(n);
        for (int i = 0; i < n; i++)
        {
            T[i] = std::make_pair(x[i], i);
        }
        // Sort T descending according to first element
        std::sort(T.begin(), T.end(),
                  [](const std::pair<float, int>& a, const std::pair<float, int>& b) { return a.first > b.first; });
 
        float rank = 1.0f, m = 1.0f, i = 0.0f;
        while (i < n)
        {
            float j = i;
            // Get no of elements with equal rank
            while (j < n - 1 && T[j].first == T[j + 1].first)
            {
                j += 1;
            }
 
            m = j - i + 1;
 
            for (int k = 0; k < m; ++k)
            {
                // For each equal element use formula
                // obtain index of T[i+j][0] in A
                int idx = T[i + k].second;
                R[idx] = (double)(rank + (m - 1) * 0.5);
            }
 
            // Increment rank and i
            rank += m;
            i += m;
        }
        return R;
    }
 
    std::vector<std::pair<float, int>> TextToAssets::topk(std::vector<std::pair<float, int>> data, int topk) const
    {
        // compute the top-k
        topk = std::min(topk, (int)data.size());
        std::nth_element(
                data.begin(), data.begin() + topk, data.end(),
                [](const std::pair<float, int>& a, const std::pair<float, int>& b) { return a.first > b.first; });
        data.resize(topk);
        return data;
    }
 
    std::vector<float> TextToAssets::calculateReciprocalRankFusion(const std::vector<float>& scores1,
                                                                   const std::vector<float>& scores2, float k,
                                                                   float alpha) const
    {
        const auto ranks1 = rankify(scores1);
        const auto ranks2 = rankify(scores2);
        std::vector<float> scores;
        for (int i = 0; i < (int)ranks1.size(); ++i)
        {
            const auto score1 = 1.0f / (ranks1[i] + k);
            const auto score2 = 1.0f / (ranks2[i] + k);
            const auto score = alpha * score1 + (1.0f - alpha) * score2;
            scores.push_back(score);
        }
        return scores;
    }
 
    std::tuple<String, StringArray> TextToAssets::preprocessQuery(String query) const
    {
        StringArray queryTokens;
        queryTokens.addTokens(query.toLowerCase(), " ", "\"");
 
        // update exclude tags so we can filter out files that contain those keywords
        StringArray excludeList; // {"designed", "misc"}; // initial exclude tags
        query = "";
        for (const auto& token : queryTokens)
        {
            if (token.startsWithChar('-'))
            {
                excludeList.addIfNotAlreadyThere(token.replaceSection(0, 1, ""));
            }
            else
            {
                query += " ";
                query += token;
            }
        }
        return {query, excludeList};
    }
 
    void TextToAssets::excludeLayers(const StringArray& excludeList, std::vector<Layer>& layers)
    {
        for (auto& layer : layers)
        {
            const auto name = layer.layerName;
            for (const auto& word : excludeList)
            {
                if (name.containsWholeWord(word))
                {
                    layer.isValid = false;
                    break;
                }
            }
        }
    }
 
    float TextToAssets::dot(const std::vector<float>& a, const std::vector<float>& b) const
    {
        return std::inner_product(a.begin(), a.end(), b.begin(), 0.0f);
    }
 
    void TextToAssets::normalise(std::vector<float>& x) const
    {
        const auto scale = 1.0f / std::max(std::sqrt(dot(x, x)), 1e-12f);
        for (int i = 0; i < x.size(); ++i)
        {
            x[i] *= scale;
        }
    }
 
    std::vector<float> TextToAssets::getVectorScores(String catID, const std::vector<float>& x) const
    {
        std::vector<float> vectorScores;
        if (m_datasetMap.count(catID) > 0)
        {
            const auto dataset = m_datasetMap.at(catID);
            vectorScores.reserve(dataset.size());
            for (const auto& element : dataset)
            {
                auto emb = element.embeddings;
                auto score = dot(x, emb);
                vectorScores.push_back(score);
            }
        }
        return vectorScores;
    }
 
    void TextToAssets::searchForTagAndProperty(ValueTree& root, Identifier tag, Identifier property, String newSample,
                                               Identifier target)
    {
        int index = 0;
        ValueTree child;
        int numChildrean = root.getNumChildren();
 
        while ((child = root.getChild(index)).isValid())
        {
            if (child.hasType(tag) && child.getParent().getParent().getType().toString() == "KWIDGET" &&
                child.getParent().getParent().getProperty("id").toString() == target.toString())
            {
                if (child.getProperty(Identifier("id")).toString() == property.toString())
                {
                    // replace with new sample paths
                    child.setProperty("value", newSample, nullptr);
 
                    return;
                }
                else
                {
                    searchForTagAndProperty(child, tag, property, newSample, target);
                }
            }
            else
            {
                searchForTagAndProperty(child, tag, property, newSample, target);
            }
            index++;
        }
        return;
    }
 
    void TextToAssets::updateTabLabel(ValueTree& root, int index, const String& label)
    {
        // Find CUSTOMDATA node
        ValueTree customDataNode = root.getChildWithName(Identifier("CUSTOMDATA"));
 
        if (customDataNode.isValid())
        {
            // Find C_E_LABELS node within CUSTOMDATA
            ValueTree labelsNode = customDataNode.getChildWithName(Identifier("C_E_LABELS"));
            if (labelsNode.isValid())
            {
                // Update the specific TABLABEL element by index
                for (int i = 0; i < labelsNode.getNumChildren(); ++i)
                {
                    ValueTree tabLabelNode = labelsNode.getChild(i);
                    if (tabLabelNode.hasType(Identifier("TABLABEL")) && tabLabelNode.getProperty("index") == var(index))
                    {
                        tabLabelNode.setProperty("text", label, nullptr);
                        return;
                    }
                }
            }
        }
 
        // If the TABLABEL with the given index was not found
        jassertfalse;
        DBG("Could not update the TABLABEL: specified index not found.");
    }
 
    void TextToAssets::updateCoreEngineParamValue(ValueTree& tree, const var& coreEngineID, const var& paramID,
                                                  const String& value)
    {
        // [1] update CoreEngine param value
        auto kwidgets = tree.getChildWithName(Identifier("KWIDGETS"));
        jassert(kwidgets.isValid());
 
        auto kwidget = kwidgets.getChildWithProperty(Identifier("id"), coreEngineID);
        jassert(kwidget.isValid());
 
        auto params = kwidget.getChildWithName(Identifier("PARAMS"));
        jassert(params.isValid());
 
        auto param = params.getChildWithProperty(Identifier("id"), paramID);
        jassert(param.isValid() && param.hasProperty(Identifier("value")));
 
        param.setProperty("value", value, nullptr);
 
        // [2] update param with matching linkIndex
        auto linkIndex = param.getPropertyAsValue(Identifier("linkIndex"), nullptr).toString();
        linkIndex = "#" + linkIndex.paddedLeft('0', 3); // e.g. "#052"
 
        params = tree.getChildWithName(Identifier("PARAMS"));
        jassert(params.isValid());
 
        param = params.getChildWithProperty(Identifier("id"), linkIndex);
        jassert(param.isValid() && param.hasProperty(Identifier("value")));
 
        param.setProperty("value", value, nullptr);
    }
 
    std::unique_ptr<XmlElement> TextToAssets::getInitTTPData()
    {
        auto xml = parseXML(KrotosBinaryData::init_ttp_preset_xml);
        return xml;
    }
 
    void TextToAssets::notifyAvailabilityStatus(bool isAvailable)
    {
        m_ttaAvailable = isAvailable;
        if (m_synchronousUpdates)
        {
            m_listeners.call([this](Listener& l) { l.onTextToAssetsProcessingStatusChanged(m_ttaAvailable); });
        }
        else
        {
            m_statusUpdater.triggerAsyncUpdate(
                    [this](Listener& l) { l.onTextToAssetsProcessingStatusChanged(m_ttaAvailable); });
        }
    }
 
    void TextToAssets::notifyProgressUpdate(const int processedFiles, const int totalFiles,
                                            const String& estimatedTimeRemaining)
    {
        if (m_synchronousUpdates)
        {
            m_listeners.call([processedFiles, totalFiles, estimatedTimeRemaining](Listener& l) {
                l.onTextToAssetsProgressUpdate(processedFiles, totalFiles, estimatedTimeRemaining);
            });
        }
        else
        {
            m_progressUpdater.triggerAsyncUpdate([processedFiles, totalFiles, estimatedTimeRemaining](Listener& l) {
                l.onTextToAssetsProgressUpdate(processedFiles, totalFiles, estimatedTimeRemaining);
            });
        }
    }
 
    //==============================================================================
    void TextToAssets::addListener(Listener* listener) { m_listeners.add(listener); }
 
    void TextToAssets::removeListener(Listener* listener) { m_listeners.remove(listener); }
 
} // namespace krotos