FactoryAssetsWatcher.cpp

Go to the documentation of this file.

#include "FactoryAssetsWatcher.h"
 
#include <openssl/evp.h>
#include <openssl/err.h>
 
namespace krotos
{
 
FactoryAssetsWatcher::FactoryAssetsWatcher(const File& dirToWatch) : m_directory(dirToWatch)
{
    thread.addTimeSliceClient(this);
    thread.startThread();
}
 
FactoryAssetsWatcher::~FactoryAssetsWatcher()
{
    // Signal thread to exit first so removeTimeSliceClient can return asap.
    thread.signalThreadShouldExit();
    thread.notify();
 
    // Calling stopThread before removeTimeSliceClient can result in a deadlock if the timeout expires and the thread
    // is killed whilst it's still executing. This is because removeTimeSliceClient doesn't return until the thread
    // finishes whatever it's doing. If it's killed, the lock is never released and the call will never return.  By
    // signalling the thread to exit first, it should return at the next possible opportunity.
    thread.removeTimeSliceClient(this);
 
    bool threadExitedSafely = thread.stopThread(2000);
    if (!threadExitedSafely)
    {
        jassertfalse;
    }
}
 
bool FactoryAssetsWatcher::handleContentUpdate()
{
    // scan and load all valid UCS files to a file array
    m_filesArray.clear();
    getAllFiles(m_directory, m_filesArray);
 
    // load embeddings file and load all file names into a StringArray
    auto file = m_textToAssetsSharedInstance.textToAssets->createFilePath(ttfEmbeddingsFileName);
    switch (m_assetsStatus)
    {
    case AssetsStatus::kAssetsRemoved: {
        return removeFileFromEmbeddings(file);
    }
    case AssetsStatus::kAssetsAdded: {
        return addFileToEmbeddings(file);
    }
    case AssetsStatus::kAssetsInit: {
        return initEmbeddings(file);
    }
    default: {
        m_assetsStatus = AssetsStatus::kAssetsNoChange;
        return false;
    }
    }
}
 
void FactoryAssetsWatcher::getAllFiles(const File& directory, Array<File>& fileArray)
{
    std::set<String> fileNames;
    for (DirectoryEntry entry : RangedDirectoryIterator(directory, true, "*.kaf;*.wav;*.KAF;*.WAV"))
    {
        auto file = entry.getFile();
        auto name = file.getFileNameWithoutExtension();
        auto catID = name.upToFirstOccurrenceOf("_", false, false);
        catID = (catID == "AMBROOM") ? "AMBRoom" : catID; // a fix for basement preset assets
        catID = (catID == "AMBUrb") ? "AMBUrbn" : catID; // a fix for skatepark preset assets
        auto valid = m_textToAssetsSharedInstance.textToAssets->isUCSValid(catID);
        // only process valid UCS files
        if (valid)
        {
            // only process unique files
            if (!fileNames.count(name))
            {
                fileNames.insert(name);
                fileArray.add(file);
            }
        }
    }
}
 
int FactoryAssetsWatcher::useTimeSlice()
{
    String dirModificationDate = m_directory.getLastModificationTime().toString(true, true);
 
    String lastRecordedModificationDate;
 
    File textEmbeddingsStatusFile = m_textToAssetsSharedInstance.textToAssets->createFilePath(ttfStatusFileName);
 
    File ggmlDataFile = File(utils::StringsIntoPath(AssetManager::getPluginDirectory().getFullPathName(),
                                                    "ttpResources", modelFileName));
 
    if (ggmlDataFile.existsAsFile() == false)
    {
        return 5000;
    }
 
    // trigger ttp update if there is no status file
    if (textEmbeddingsStatusFile.exists() == false)
    {
        textEmbeddingsStatusFile.create();
 
        // set process to init
        m_assetsStatus = AssetsStatus::kAssetsInit;
 
        // dir mod date
        m_dirModificationDate = dirModificationDate;
 
        // dir size init
        m_dirSize = getTotalFolderSize(m_directory);
 
        // process
        bool success = handleContentUpdate();
 
        if (success)
        {
            DynamicObject::Ptr ttfStatus(new DynamicObject());
 
            ttfStatus->setProperty(ttfStatusDateProperty, var(m_dirModificationDate));
            ttfStatus->setProperty(ttfStatusSizeProperty, var(String(m_dirSize)));
 
            FileOutputStream stream(textEmbeddingsStatusFile);
            if (stream.openedOk())
            {
                stream.setPosition(0);
                stream.truncate();
                JSON::writeToStream(stream, ttfStatus.get());
            }
        }
        else
        {
            DBG("ttf embeddings update aborted");
            textEmbeddingsStatusFile.deleteFile();
        }
    }
    else
    {
        // parse ttp status file
        auto ttfStatusJSON = JSON::parse(textEmbeddingsStatusFile);
 
        // get date modified
        m_dirModificationDate = ttfStatusJSON.getProperty(Identifier(ttfStatusDateProperty), String()).toString();
 
        // get previous dir size
        m_dirSize = static_cast<int64>(ttfStatusJSON.getProperty(Identifier(ttfStatusSizeProperty), String(0)));
 
        // get embeddings file
        File embeddingsFile = m_textToAssetsSharedInstance.textToAssets->createFilePath(ttfEmbeddingsFileName);
 
        // if it differs or if embeddings file does not exist (deleted probably) trigger ttp update
        if ((dirModificationDate != m_dirModificationDate || embeddingsFile.exists() == false) &&
            !thread.threadShouldExit())
        {
            m_dirModificationDate = dirModificationDate;
            int64 dirSize = getTotalFolderSize(m_directory);
 
            if (dirSize < m_dirSize)
            {
                m_assetsStatus = AssetsStatus::kAssetsRemoved;
            }
            else if (dirSize > m_dirSize)
            {
                m_assetsStatus = AssetsStatus::kAssetsAdded;
            }
            else
            {
                m_assetsStatus = AssetsStatus::kAssetsNoChange;
            }
            m_dirSize = dirSize;
 
            bool success = handleContentUpdate();
 
            if (success)
            {
                DynamicObject::Ptr ttfStatus(new DynamicObject());
 
                ttfStatus->setProperty(ttfStatusDateProperty, var(m_dirModificationDate));
                ttfStatus->setProperty(ttfStatusSizeProperty, var(String(m_dirSize)));
 
                FileOutputStream stream(textEmbeddingsStatusFile);
                if (stream.openedOk())
                {
                    stream.setPosition(0);
                    stream.truncate();
                    JSON::writeToStream(stream, ttfStatus.get());
                }
            }
        }
    }
    return 5000;
}
 
int64 FactoryAssetsWatcher::getTotalFolderSize(const juce::File& folder)
{
    jassert(folder.isDirectory());
    int64 totalSize = 0;
    for (const auto& entry : juce::RangedDirectoryIterator(folder, true))
    {
        totalSize += entry.getFileSize();
    }
    return totalSize;
}
 
bool FactoryAssetsWatcher::addFileToEmbeddings(const File& file)
{
    m_filesToProcess.clear();
    m_filesWithEmbeddings.clear();
 
    if (file.existsAsFile())
    {
        auto json = JSON::parse(file);
        var result = json.getProperty(Identifier(fileEmbeddingsIdentifier), 0);
        for (int i = 0; i < result.size(); ++i)
        {
            if (thread.threadShouldExit())
            {
                return false;
            }
            auto fileName = File(result[i].getProperty(fileNameIdentifier, 0)).getFileName();
            m_filesWithEmbeddings.add(fileName);
        }
    }
 
    checkAddFileInEmbeddings();
 
    m_assetsStatus = AssetsStatus::kAssetsNoChange;
 
    // calculate embeddings for new presets
    return m_textToAssetsSharedInstance.textToAssets->processFileList(m_filesToProcess);
}
 
bool FactoryAssetsWatcher::removeFileFromEmbeddings(const File& file)
{
    if (file.existsAsFile())
    {
        Array<int> indicesToRemove;
        auto json = JSON::parse(file);
        var result = json.getProperty(Identifier(fileEmbeddingsIdentifier), 0);
 
        Array<String> onlyFileNamesArrayFromFileSystem;
        for (const auto& f : m_filesArray)
        {
            onlyFileNamesArrayFromFileSystem.add(f.getFileName());
        }
 
        auto embeddingsArray = *result.getArray();
 
        for (int i = 0; i < embeddingsArray.size(); ++i)
        {
            auto fileFullPath =
                    File(embeddingsArray[i].getProperty(fileNameIdentifier, 0).toString()).getFullPathName();
            auto fileName = File(fileFullPath).getFileName();
 
            // remove from embeddings
            if (onlyFileNamesArrayFromFileSystem.contains(fileName) == false)
            {
                indicesToRemove.add(i);
            }
        }
 
        // process only if there is files to remove from embeddings
        if (indicesToRemove.size() > 0)
        {
            // sort indices to descending order to avoid shifts
            std::sort(indicesToRemove.begin(), indicesToRemove.end(), std::greater<int>());
 
            for (auto& i : indicesToRemove)
            {
                embeddingsArray.remove(i);
            }
 
            // Update the JSON object with the modified embeddings array
            DynamicObject::Ptr updatedJson(json.getDynamicObject());
 
            updatedJson->removeProperty(Identifier(fileEmbeddingsIdentifier));
            updatedJson->setProperty(Identifier(fileEmbeddingsIdentifier), var(embeddingsArray));
 
            FileOutputStream jsonOutStream(file);
            jsonOutStream.setPosition(0);
            jsonOutStream.truncate();
            JSON::writeToStream(jsonOutStream, updatedJson.get());
 
            m_assetsStatus = AssetsStatus::kAssetsNoChange;
            return true;
        }
        m_assetsStatus = AssetsStatus::kAssetsNoChange;
        return false;
    }
    m_assetsStatus = AssetsStatus::kAssetsNoChange;
    return false;
}
 
bool FactoryAssetsWatcher::initEmbeddings(const File& file)
{
    m_filesToProcess.clear();
    m_filesWithEmbeddings.clear();
 
    if (file.existsAsFile())
    {
        auto json = JSON::parse(file);
        var result = json.getProperty(Identifier(fileEmbeddingsIdentifier), 0);
        for (int i = 0; i < result.size(); ++i)
        {
            if (thread.threadShouldExit())
            {
                return false;
            }
            auto fileName = File(result[i].getProperty(fileNameIdentifier, 0)).getFileName();
            m_filesWithEmbeddings.add(fileName);
        }
    }
 
    checkAddFileInEmbeddings();
 
    m_assetsStatus = AssetsStatus::kAssetsNoChange;
 
    // calculate embeddings for new presets
    return m_textToAssetsSharedInstance.textToAssets->processFileList(m_filesToProcess);
}
 
void FactoryAssetsWatcher::checkAddFileInEmbeddings()
{
    for (File& file : m_filesArray)
    {
        if (thread.threadShouldExit())
        {
            return;
        }
 
        // file is new and has no embeddings, so ..
        String filenameToCheck = file.getFileName();
        if (m_filesWithEmbeddings.contains(filenameToCheck) == false)
        {
            m_filesToProcess.add(file);
        }
    }
}
} // namespace krotos