TextToFile.h

Go to the documentation of this file.

#pragma once
 
namespace krotos
{
 
// text to file embeddings file name
constexpr char textToFileEmbeddingsFilename[] = ".ttfEmbCache.json";
 
class TextToFile : public Thread
{
  public:
    TextToFile();
    ~TextToFile();
    std::vector<File> search(String query, std::size_t k = 25);
 
  private:
    void run() override;
    File embeddingsFilePath(String filename) const;
 
    bool saveToFile(File file, const std::map<String, std::vector<float>>& data)
    {
        DynamicObject::Ptr dataset(new DynamicObject());
        Array<var> rows;
 
        for (const auto& element : data)
        {
            const auto fileName = element.first;
            const auto embeddings = element.second;
 
            DynamicObject::Ptr metadata(new DynamicObject());
 
            metadata->setProperty("fileName", var(fileName));
 
            Array<var> embeddingsArray;
            for (auto value : embeddings)
                embeddingsArray.add(value);
            metadata->setProperty("embeddings", embeddingsArray);
 
            rows.add(metadata.get());
        }
        dataset->setProperty("FileNameEmbeddings", rows);
 
        FileOutputStream stream(file);
        if (stream.openedOk())
        {
            // overwrite an existing file
            stream.setPosition(0);
            stream.truncate();
            JSON::writeToStream(stream, dataset.get());
            return true;
        }
 
        return false;
    }
 
    std::map<String, std::vector<float>> readFromFile(File file)
    {
        assert(file.existsAsFile());
 
        std::map<String, std::vector<float>> data;
 
        auto json = JSON::parse(file);
        var result = json.getProperty(Identifier("FileNameEmbeddings"), 0);
        for (int i = 0; i < result.size(); ++i)
        {
            auto fileName = result[i].getProperty(Identifier("fileName"), 0).toString();
            var embeddingsArray = result[i].getProperty(Identifier("embeddings"), 0);
 
            std::vector<float> embeddings;
            for (int j = 0; j < embeddingsArray.size(); ++j)
                embeddings.push_back(embeddingsArray[j]);
 
            data[fileName] = embeddings;
        }
 
        return data;
    }
 
    SentenceTransformer m_sentenceTransformer;
    KDTreeND m_tree;
    std::vector<File> m_files;
    SharedResourcePointer<UniversalCategorySystem> m_UCS;
};
 
} // namespace krotos