take a user text query and return 4 assets that can be loaded as an ambience preset More...

#include <TextToAssets.h>

Inheritance diagram for krotos::TextToAssets:

Classes
struct	Data

struct	Layer

class	Listener
	Listener for the TextToAssets module. More...

struct	TTPAsset

Public Member Functions
	TextToAssets ()

	~TextToAssets ()

std::tuple< String, std::array< TTPAsset, 4 > >	findPresetFiles (String searchTerm)
	find 4 assets suitable for the given query

std::vector< float >	rankify (const std::vector< float > &x) const
	output the rank of each vector element

bool	processFileList (Array< File > &filesToProcess)
	processes a file list to create text embeddings

bool	appendDataToEmbeddingsFile ()
	appends data to the main embeddings file

virtual File	createFilePath (String filename)
	creates the path for the embeddings file

bool	isUCSValid (String catID)
	check UCS CatID is valid

void	setClaudeApiKey (const String &key)
	Set the encrypted API key string for the Claude API.

void	addListener (Listener *listener)

void	removeListener (Listener *listener)

bool	isAvailable ()
	Returns the status of the file processing.

bool	isStrictMode ()
	Returns if "Strict Mode" is enabled (default: disabled).

void	setStrictMode (bool isStrictMode)
	Sets whether "Strict Mode" is enabled or disabled (default: disabled).

Static Public Member Functions
static void	searchForTagAndProperty (ValueTree &root, Identifier tag, Identifier property, String newSample, Identifier target)
	this will add a sample that comes from ttp to the corresponding core egine but it also update the xypad labels with the assets proposed,

static void	updateTabLabel (ValueTree &root, int index, const String &label)
	This function searches for a specific TABLABEL by index within the CUSTOMDATA node and updates its text.

static void	updateCoreEngineParamValue (ValueTree &tree, const var &coreEngineID, const var &paramID, const String &value)
	update a CoreEngine PARAM value

static std::unique_ptr< XmlElement >	getInitTTPData ()
	xml for automated preset generation

Protected Member Functions
virtual void	calculateEmbeddingsForFile (const File &file)
	create text embeddings file for the given file

virtual bool	isModelFileAvailable () const
	Returns whether there is a model file available for use, in order to generate embeddings.

Protected Attributes
bool	m_modelFileAvailable {false}

bool	m_synchronousUpdates {false}

Private Member Functions
void	addBiasToLayers (const String &query, std::vector< Layer > &layers)
	add a non-zero bias to layers that match certain keywords

StringArray	parseAnthropicResponse (std::unique_ptr< InputStream > &stream)
	parse Anthropic API response and return results in a StringArray

StringArray	getLLMLayers (std::unique_ptr< InputStream > &stream)
	returns a StringArray of preset layer names generated by an LLM

StringArray	anthropicAPICall (String query)
	send query to an LLM using the anthropic API

std::vector< float >	calculateLLMLayerScores (const std::vector< Layer > &layers, const StringArray &generatedLayerNames)
	return similarity scores for layers based on the LLM generated layer names

float	dot (const std::vector< float > &a, const std::vector< float > &b) const
	return dot product between a and b

void	normalise (std::vector< float > &x) const
	L2 normalise the input vector.

std::vector< float >	getVectorScores (String catID, const std::vector< float > &x) const
	return vector scores between query embedding and dataset embeddings

std::tuple< String, StringArray >	preprocessQuery (String query) const
	return preprocessed query and exclude tags

std::vector< std::pair< float, int > >	topk (std::vector< std::pair< float, int > > data, int topk) const
	return topk elements

std::vector< float >	calculateReciprocalRankFusion (const std::vector< float > &scores1, const std::vector< float > &scores2, float k=2.0f, float alpha=0.5f) const
	calculate the reciprocal rank fusion for scores1 and scores2

bool	matchesCategory (const String &catID, StringArray tags) const
	check if UCS catID matches one of our target tags

void	clear ()
	clears internal data

void	run () override
	run thread

void	loadFileEmbeddings ()
	will load embeddings data from file to all modules that need it

bool	readFromFile (File file)
	load embeddings from file

String	sanitizeString (String text)
	removes digits and underscores from a string

String	capitalize (const String &text)
	make first letter of string upper case and the rest lower case

void	loadPresetLayers ()
	load preset layers from json stored as binary data

void	loadQueryEmbeddings ()
	load quer embeddings from json stored as binary data

std::tuple< String, String >	findClosestAmbienceCatID (const String &query, const std::vector< float > &queryEmbedding)
	search for closest match to query

std::vector< int >	argsort (const std::vector< float > &x) const
	returns the indices that would sort the vector.

bool	containsKeyword (const String &text, const StringArray &keywords)
	returns true if the text contains any keyword in keywords.

void	maskLayersContainingTag (String tag, std::vector< Layer > &layers)
	set layer.isValid to false for layers containing tag.

void	excludeLayers (const StringArray &excludeList, std::vector< Layer > &layers)
	uses the contents of exclude list to set some preset layers as invalid if certain keywords are detected.

std::tuple< float, int, bool >	findLayer (const std::vector< float > &queryEmbedding, const std::vector< float > &presetEmbedding, const std::vector< Layer > &layers, const std::vector< float > &layerScoresLLM)
	find a relevant preset layer

std::tuple< String, float >	findAsset (const Layer &layer)
	find a relevant asset for a given preset layer

void	notifyAvailabilityStatus (bool isAvailable)
	Update TTA availability status and notify listeners. TTA should not available whilst file processing is taking place.

void	notifyProgressUpdate (const int processedFiles, const int totalFiles, const String &estimatedTimeRemaining)
	Notifies listeners of the progress of the current file processing operation.

Private Attributes
std::unique_ptr< BM25 >	m_bm25

std::vector< String >	m_bm25Corpus

std::unordered_map< String, std::vector< Data > >	m_datasetMap

std::unordered_map< String, std::vector< std::size_t > >	m_datasetIDMap

std::map< String, int >	m_assetCounter

StringArray	m_samplePaths

StringArray	m_sampleFilenames

bool	m_shouldSave = true

SentenceTransformer	m_sentenceTransformer

SharedResourcePointer< UniversalCategorySystem >	m_UCS

CriticalSection	m_cs

bool	m_ttaAvailable {true}

std::vector< std::pair< String, std::vector< float > > >	m_queryEmbeddings

std::unordered_map< String, std::vector< Layer > >	m_layersMap

String	m_claudeApiKeyEncrypted

ListenerList< Listener >	m_listeners

AsyncListenerUpdater< Listener >	m_statusUpdater

AsyncListenerUpdater< Listener >	m_progressUpdater

bool	m_strictMode {false}

Detailed Description

take a user text query and return 4 assets that can be loaded as an ambience preset

Constructor & Destructor Documentation

◆ TextToAssets()

krotos::TextToAssets::TextToAssets ( )

Constructor.

◆ ~TextToAssets()

krotos::TextToAssets::~TextToAssets ( )

Destructor.

Member Function Documentation

◆ addBiasToLayers()

void krotos::TextToAssets::addBiasToLayers	(	const String &	query,
		std::vector< Layer > &	layers )

private

add a non-zero bias to layers that match certain keywords

◆ addListener()

void krotos::TextToAssets::addListener ( Listener * listener )

Registers a listener for callbacks when the TTA status changes.

See also: removeListener

◆ anthropicAPICall()

StringArray krotos::TextToAssets::anthropicAPICall ( String query )

private

send query to an LLM using the anthropic API

Parameters

query the user query

Returns: a StringArray of preset layer names generated by the LLM

◆ appendDataToEmbeddingsFile()

bool krotos::TextToAssets::appendDataToEmbeddingsFile ( )

appends data to the main embeddings file

Returns: true if process was successful

◆ argsort()

std::vector< int > krotos::TextToAssets::argsort ( const std::vector< float > & x ) const

private

returns the indices that would sort the vector.

◆ calculateEmbeddingsForFile()

virtual void krotos::TextToAssets::calculateEmbeddingsForFile ( const File & file )

protectedvirtual

create text embeddings file for the given file

◆ calculateLLMLayerScores()

std::vector< float > krotos::TextToAssets::calculateLLMLayerScores	(	const std::vector< Layer > &	layers,
		const StringArray &	generatedLayerNames )

private

return similarity scores for layers based on the LLM generated layer names

◆ calculateReciprocalRankFusion()

std::vector< float > krotos::TextToAssets::calculateReciprocalRankFusion	(	const std::vector< float > &	scores1,
		const std::vector< float > &	scores2,
		float	k = 2.0f,
		float	alpha = 0.5f ) const

private

calculate the reciprocal rank fusion for scores1 and scores2

◆ capitalize()

String krotos::TextToAssets::capitalize ( const String & text )

private

make first letter of string upper case and the rest lower case

Parameters

text	The text to process

Returns: the processed string

◆ clear()

void krotos::TextToAssets::clear ( )

private

clears internal data

◆ containsKeyword()

bool krotos::TextToAssets::containsKeyword	(	const String &	text,
		const StringArray &	keywords )

private

returns true if the text contains any keyword in keywords.

◆ createFilePath()

virtual File krotos::TextToAssets::createFilePath ( String filename )

virtual

creates the path for the embeddings file

◆ dot()

float krotos::TextToAssets::dot	(	const std::vector< float > &	a,
		const std::vector< float > &	b ) const

private

return dot product between a and b

◆ excludeLayers()

void krotos::TextToAssets::excludeLayers	(	const StringArray &	excludeList,
		std::vector< Layer > &	layers )

private

uses the contents of exclude list to set some preset layers as invalid if certain keywords are detected.

◆ findAsset()

std::tuple< String, float > krotos::TextToAssets::findAsset ( const Layer & layer )

private

find a relevant asset for a given preset layer

Parameters

layer The preset layer

Returns: tuple containing the asset path, the asset score, and the index of the selected asset in the local dataset

◆ findClosestAmbienceCatID()

std::tuple< String, String > krotos::TextToAssets::findClosestAmbienceCatID	(	const String &	query,
		const std::vector< float > &	queryEmbedding )

private

search for closest match to query

Parameters

query	The user query as a string
queryEmbedding	The user query sbert embedding

Returns: tuple of strings representing the closest match and any secondary match

◆ findLayer()

std::tuple< float, int, bool > krotos::TextToAssets::findLayer	(	const std::vector< float > &	queryEmbedding,
		const std::vector< float > &	presetEmbedding,
		const std::vector< Layer > &	layers,
		const std::vector< float > &	layerScoresLLM )

private

find a relevant preset layer

Parameters

queryEmbedding	The user query sbert embedding
presetEmbedding	The cumulative sbert embedding for selected preset layers
layers	The available preset layers
layerScoresLLM	The scores each layer received from matching against the generated LLM layer names

Returns: tuple containing the selected layer score, the index of that layer, and a bool indicating if the search was successful

◆ findPresetFiles()

std::tuple< String, std::array< TTPAsset, 4 > > krotos::TextToAssets::findPresetFiles ( String searchTerm )

find 4 assets suitable for the given query

Parameters

searchTerm The user query

Returns: tuple containing String debug info, and pairs of String asset paths and labels

◆ getInitTTPData()

std::unique_ptr< XmlElement > krotos::TextToAssets::getInitTTPData ( )

static

xml for automated preset generation

◆ getLLMLayers()

StringArray krotos::TextToAssets::getLLMLayers ( std::unique_ptr< InputStream > & stream )

private

returns a StringArray of preset layer names generated by an LLM

◆ getVectorScores()

std::vector< float > krotos::TextToAssets::getVectorScores	(	String	catID,
		const std::vector< float > &	x ) const

private

return vector scores between query embedding and dataset embeddings

◆ isAvailable()

bool krotos::TextToAssets::isAvailable ( )

inline

Returns the status of the file processing.

◆ isModelFileAvailable()

virtual bool krotos::TextToAssets::isModelFileAvailable ( ) const

inlineprotectedvirtual

Returns whether there is a model file available for use, in order to generate embeddings.

◆ isStrictMode()

bool krotos::TextToAssets::isStrictMode ( )

inline

Returns if "Strict Mode" is enabled (default: disabled).

Strict mode disables bias in layer selection, potentially reducing output likelihood. When disabled (default), T2P uses bias towards expected keywords in base content. When enabled, it uses original generation settings without keyword bias.

◆ isUCSValid()

bool krotos::TextToAssets::isUCSValid ( String catID )

check UCS CatID is valid

◆ loadFileEmbeddings()

void krotos::TextToAssets::loadFileEmbeddings ( )

private

will load embeddings data from file to all modules that need it

◆ loadPresetLayers()

void krotos::TextToAssets::loadPresetLayers ( )

private

load preset layers from json stored as binary data

◆ loadQueryEmbeddings()

void krotos::TextToAssets::loadQueryEmbeddings ( )

private

load quer embeddings from json stored as binary data

◆ maskLayersContainingTag()

void krotos::TextToAssets::maskLayersContainingTag	(	String	tag,
		std::vector< Layer > &	layers )

private

set layer.isValid to false for layers containing tag.

◆ matchesCategory()

bool krotos::TextToAssets::matchesCategory	(	const String &	catID,
		StringArray	tags ) const

private

check if UCS catID matches one of our target tags

◆ normalise()

void krotos::TextToAssets::normalise ( std::vector< float > & x ) const

private

L2 normalise the input vector.

◆ notifyAvailabilityStatus()

void krotos::TextToAssets::notifyAvailabilityStatus ( bool isAvailable )

private

Update TTA availability status and notify listeners. TTA should not available whilst file processing is taking place.

Parameters

isAvailable True if TTA is available

◆ notifyProgressUpdate()

void krotos::TextToAssets::notifyProgressUpdate	(	const int	processedFiles,
		const int	totalFiles,
		const String &	estimatedTimeRemaining )

private

Notifies listeners of the progress of the current file processing operation.

Parameters

processedFiles	The number of files that have been processed so far
totalFiles	The total number of files to be processed in the job
estimatedTimeRemaining	The estimated time until file processing completion, in a human readable string format, using RelativeTime::getDescription

◆ parseAnthropicResponse()

StringArray krotos::TextToAssets::parseAnthropicResponse ( std::unique_ptr< InputStream > & stream )

private

parse Anthropic API response and return results in a StringArray

◆ preprocessQuery()

std::tuple< String, StringArray > krotos::TextToAssets::preprocessQuery ( String query ) const

private

return preprocessed query and exclude tags

◆ processFileList()

bool krotos::TextToAssets::processFileList ( Array< File > & filesToProcess )

processes a file list to create text embeddings

Parameters

file	list to process

Returns: true if process was successful

◆ rankify()

std::vector< float > krotos::TextToAssets::rankify ( const std::vector< float > & x ) const

output the rank of each vector element

Parameters

x	The vector of elements to find ranks for

Returns: vector of ranks

◆ readFromFile()

bool krotos::TextToAssets::readFromFile ( File file )

private

load embeddings from file

◆ removeListener()

void krotos::TextToAssets::removeListener ( Listener * listener )

Deregisters a listener.

See also: addListener

◆ run()

void krotos::TextToAssets::run ( )

overrideprivate

run thread

◆ sanitizeString()

String krotos::TextToAssets::sanitizeString ( String text )

private

removes digits and underscores from a string

Parameters

text	The text to process

Returns: the processed string

◆ searchForTagAndProperty()

void krotos::TextToAssets::searchForTagAndProperty	(	ValueTree &	root,
		Identifier	tag,
		Identifier	property,
		String	newSample,
		Identifier	target )

static

this will add a sample that comes from ttp to the corresponding core egine but it also update the xypad labels with the assets proposed,

Parameters

root	the preset root value tree,
tag	the tag uder which the property exists,
the	ptoperty to update
newSample	the sample from ttp
target	the target to update ( core engie or xy pad in this case)

◆ setClaudeApiKey()

void krotos::TextToAssets::setClaudeApiKey ( const String & key )

inline

Set the encrypted API key string for the Claude API.

Parameters

key	The encrypted API key, represented as a 256-character hexadecimal string with the IV prepended as the first 32 characters and the remaining 224 characters as the ciphertext.

◆ setStrictMode()

void krotos::TextToAssets::setStrictMode ( bool isStrictMode )

inline

Sets whether "Strict Mode" is enabled or disabled (default: disabled).

Strict mode disables bias in layer selection, potentially reducing output likelihood. When disabled (default), T2P uses bias towards expected keywords in base content. When enabled, it uses original generation settings without keyword bias.

◆ topk()

std::vector< std::pair< float, int > > krotos::TextToAssets::topk	(	std::vector< std::pair< float, int > >	data,
		int	topk ) const

private

return topk elements

◆ updateCoreEngineParamValue()

void krotos::TextToAssets::updateCoreEngineParamValue	(	ValueTree &	tree,
		const var &	coreEngineID,
		const var &	paramID,
		const String &	value )

static

update a CoreEngine PARAM value

◆ updateTabLabel()

void krotos::TextToAssets::updateTabLabel	(	ValueTree &	root,
		int	index,
		const String &	label )

static

This function searches for a specific TABLABEL by index within the CUSTOMDATA node and updates its text.

Parameters

root	The preset root value tree.
index	The index of the TABLABEL to update.
label	The new label text for the TABLABEL.

Member Data Documentation

◆ m_assetCounter

std::map<String, int> krotos::TextToAssets::m_assetCounter

private

◆ m_bm25

std::unique_ptr<BM25> krotos::TextToAssets::m_bm25

private

◆ m_bm25Corpus

std::vector<String> krotos::TextToAssets::m_bm25Corpus

private

◆ m_claudeApiKeyEncrypted

String krotos::TextToAssets::m_claudeApiKeyEncrypted

private

◆ m_cs

CriticalSection krotos::TextToAssets::m_cs

private

◆ m_datasetIDMap

std::unordered_map<String, std::vector<std::size_t> > krotos::TextToAssets::m_datasetIDMap

private

◆ m_datasetMap

std::unordered_map<String, std::vector<Data> > krotos::TextToAssets::m_datasetMap

private

◆ m_layersMap

std::unordered_map<String, std::vector<Layer> > krotos::TextToAssets::m_layersMap

private

◆ m_listeners

ListenerList<Listener> krotos::TextToAssets::m_listeners

private

◆ m_modelFileAvailable

bool krotos::TextToAssets::m_modelFileAvailable {false}

protected

◆ m_progressUpdater

AsyncListenerUpdater<Listener> krotos::TextToAssets::m_progressUpdater

private

◆ m_queryEmbeddings

std::vector<std::pair<String, std::vector<float> > > krotos::TextToAssets::m_queryEmbeddings

private

◆ m_sampleFilenames

StringArray krotos::TextToAssets::m_sampleFilenames

private

◆ m_samplePaths

StringArray krotos::TextToAssets::m_samplePaths

private

◆ m_sentenceTransformer

SentenceTransformer krotos::TextToAssets::m_sentenceTransformer

private

◆ m_shouldSave

bool krotos::TextToAssets::m_shouldSave = true

private

◆ m_statusUpdater

AsyncListenerUpdater<Listener> krotos::TextToAssets::m_statusUpdater

private

◆ m_strictMode

bool krotos::TextToAssets::m_strictMode {false}

private

◆ m_synchronousUpdates

bool krotos::TextToAssets::m_synchronousUpdates {false}

protected

◆ m_ttaAvailable

bool krotos::TextToAssets::m_ttaAvailable {true}

private

◆ m_UCS

SharedResourcePointer<UniversalCategorySystem> krotos::TextToAssets::m_UCS

private

The documentation for this class was generated from the following files:

krotos_ml/NLP/TextToAssets.h
krotos_ml/NLP/TextToAssets.cpp

Classes

Public Member Functions

Static Public Member Functions

Protected Member Functions

Protected Attributes

Private Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

◆ TextToAssets()

◆ ~TextToAssets()

Member Function Documentation

◆ addBiasToLayers()

◆ addListener()

◆ anthropicAPICall()

◆ appendDataToEmbeddingsFile()

◆ argsort()

◆ calculateEmbeddingsForFile()

◆ calculateLLMLayerScores()

◆ calculateReciprocalRankFusion()

◆ capitalize()

◆ clear()

◆ containsKeyword()

◆ createFilePath()

◆ dot()

◆ excludeLayers()

◆ findAsset()

◆ findClosestAmbienceCatID()

◆ findLayer()

◆ findPresetFiles()

◆ getInitTTPData()

◆ getLLMLayers()

◆ getVectorScores()

◆ isAvailable()

◆ isModelFileAvailable()

◆ isStrictMode()

◆ isUCSValid()

◆ loadFileEmbeddings()

◆ loadPresetLayers()

◆ loadQueryEmbeddings()

◆ maskLayersContainingTag()

◆ matchesCategory()

◆ normalise()

◆ notifyAvailabilityStatus()

◆ notifyProgressUpdate()

◆ parseAnthropicResponse()

◆ preprocessQuery()

◆ processFileList()

◆ rankify()

◆ readFromFile()

◆ removeListener()

◆ run()

◆ sanitizeString()

◆ searchForTagAndProperty()

◆ setClaudeApiKey()

◆ setStrictMode()

◆ topk()

◆ updateCoreEngineParamValue()

◆ updateTabLabel()

Member Data Documentation

◆ m_assetCounter

◆ m_bm25

◆ m_bm25Corpus

◆ m_claudeApiKeyEncrypted

◆ m_cs

◆ m_datasetIDMap

◆ m_datasetMap

◆ m_layersMap

◆ m_listeners

◆ m_modelFileAvailable

◆ m_progressUpdater

◆ m_queryEmbeddings

◆ m_sampleFilenames

◆ m_samplePaths

◆ m_sentenceTransformer

◆ m_shouldSave

◆ m_statusUpdater

◆ m_strictMode

◆ m_synchronousUpdates

◆ m_ttaAvailable