#include <metadata_db.h>

Inheritance diagram for MetaDataDb:

Collaboration diagram for MetaDataDb:

Classes
struct	MDDbIdIterator

struct	MDDbRowIterator

Public Member Functions
void	readPlain (const FileName &inFile, const String &labelsString, const String &separator=" ")

void	addPlain (const FileName &inFile, const String &labelsString, const String &separator=" ")

iterator	begin () override

iterator	end () override

const_iterator	begin () const override

const_iterator	end () const override

id_iterator	id_begin () override

id_iterator	id_end () override

id_const_iterator	id_begin () const override

id_const_iterator	id_end () const override

void	fillExpand (MDLabel label)

void	fillConstant (MDLabel label, const String &value) override

void	fillRandom (MDLabel label, const String &mode, double op1, double op2, double op3=0.) override

void	fillLinear (MDLabel label, double initial, double step) override

void	copyColumn (MDLabel labelDest, MDLabel labelSrc) override

void	copyColumnTo (MetaData &md, MDLabel labelDest, MDLabel labelSrc) override

void	renameColumn (MDLabel oldLabel, MDLabel newLabel) override

void	renameColumn (const std::vector< MDLabel > &oldLabel, const std::vector< MDLabel > &newLabel) override

void	metadataToVec (std::vector< MDRowSql > &vd)

void	vecToMetadata (const std::vector< MDRow > &rowMetadata)

bool	operator== (const MetaDataDb &op) const

Constructors
	MetaDataDb ()

	MetaDataDb (const std::vector< MDLabel > &labelsVector)

	MetaDataDb (const MetaData &md)

	MetaDataDb (const FileName &fileName, const std::vector< MDLabel > &desiredLabels={})

	MetaDataDb (const MetaDataDb &md)

MetaDataDb &	operator= (const MetaDataDb &md)

virtual	~MetaDataDb ()

void	clear () override

Getters and setters
MDSql *	getDatabase ()

void	writeXML (const FileName fn, const FileName blockname, WriteModeMetaData mode) const override

void	writeDB (const FileName fn, const FileName blockname, WriteModeMetaData mode) const

void	writeText (const FileName fn, const std::vector< MDLabel > *desiredLabels) const override

int	getMaxStringLength (const MDLabel thisLabel) const override

MetaData Manipulation
bool	setValueCol (const MDObject &mdValueIn) override

template<class T >
bool	setValueCol (const MDLabel label, const T &valueIn)

bool	setValue (const MDObject &mdValueIn, size_t id) override

template<class T >
bool	setValue (const MDLabel label, const T &valueIn, size_t id)

bool	getValue (MDObject &mdValueOut, size_t id) const override

bool	getRowValues (size_t id, std::vector< MDObject > &values) const override

template<class T >
bool	getValue (const MDLabel label, T &valueOut, size_t id) const

void	getColumnValues (const MDLabel label, std::vector< MDObject > &valuesOut) const override

template<class T >
std::vector< T >	getColumnValues (const MDLabel label) const

template<class T >
void	getColumnValues (const MDLabel label, std::vector< T > &valuesOut) const

template<typename T >
bool	getColumnValuesOpt (const MDLabel label, std::vector< T > &values) const

void	setColumnValues (const std::vector< MDObject > &valuesIn) override

template<class T >
void	setColumnValues (const MDLabel label, const std::vector< T > &valuesIn)

bool	bindValue (size_t id) const

bool	initGetRow (bool addWhereClause) const

bool	execGetRow (MDRow &row) const

void	finalizeGetRow (void) const

std::unique_ptr< MDRow >	getRow (size_t id) override

std::unique_ptr< const MDRow >	getRow (size_t id) const override

MDRowSql	getRowSql (size_t id)

const MDRowSql	getRowSql (size_t id) const

bool	getRow (MDRowSql &row, size_t id) const

bool	getAllRows (std::vector< MDRowSql > &rows) const

bool	getRow2 (MDRow &row, size_t id) const

bool	setRow (const MDRow &row, size_t id)

bool	initAddRow (const MDRow &row)

bool	execAddRow (const MDRow &row)

void	finalizeAddRow (void)

size_t	addRow (const MDRow &row) override

void	addRowOpt (const MDRowSql &row)

void	addRows (const std::vector< MDRowSql > &rows)

void	addMissingLabels (const MDRow &row)

size_t	addRow2 (const MDRow &row)

size_t	size () const override

bool	containsLabel (const MDLabel label) const override

std::vector< MDLabel >	getActiveLabels () const override

bool	addLabel (const MDLabel label, int pos=-1) override

bool	removeLabel (const MDLabel label) override

size_t	addObject () override

void	importObject (const MetaData &md, const size_t id, bool doClear=true) override

void	importObjects (const MetaData &md, const std::vector< size_t > &objectsToAdd, bool doClear=true) override

void	importObjects (const MetaData &md, const MDQuery &query, bool doClear=true) override

bool	removeObject (size_t id) override

void	removeObjects (const std::vector< size_t > &toRemove) override

int	removeObjects (const MDQuery &query) override

int	removeObjects () override

void	addIndex (MDLabel label) const

void	addIndex (const std::vector< MDLabel > &desiredLabels) const

void	removeIndex (MDLabel label)

void	removeIndex (const std::vector< MDLabel > &desiredLabels)

void	addItemId ()

void	removeItemId ()

Iteration functions
size_t	firstRowId () const override

size_t	firstObject (const MDQuery &) const override

size_t	lastRowId () const override

Search operations
void	findObjects (std::vector< size_t > &objectsOut, const MDQuery &query) const override

void	findObjects (std::vector< size_t > &objectsOut, int limit=-1) const override

size_t	countObjects (const MDQuery &query) const override

bool	containsObject (size_t objectId) const override

bool	containsObject (const MDQuery &query) const override

I/O functions
void	_writeRows (std::ostream &os) const override

void	write (const FileName &outFile, WriteModeMetaData mode=MD_OVERWRITE) const override

void	write (std::ostream &os, const String &blockName="", WriteModeMetaData mode=MD_OVERWRITE) const override

bool	existsBlock (const FileName &_inFile)

void	readXML (const FileName &inFile, const std::vector< MDLabel > *desiredLabels=NULL, const String &blockRegExp=DEFAULT_BLOCK_NAME, bool decomposeStack=true)

void	readDB (const FileName &inFile, const std::vector< MDLabel > *desiredLabels=NULL, const String &blockRegExp=DEFAULT_BLOCK_NAME, bool decomposeStack=true)

void	read (const FileName &inFile, const std::vector< MDLabel > *desiredLabels=NULL, bool decomposeStack=true) override

Set Operations
void	aggregate (const MetaDataDb &mdIn, AggregateOperation op, MDLabel aggregateLabel, MDLabel operateLabel, MDLabel resultLabel)

void	aggregate (const MetaDataDb &mdIn, const std::vector< AggregateOperation > &ops, const std::vector< MDLabel > &operateLabels, const std::vector< MDLabel > &resultLabels)

void	aggregateGroupBy (const MetaDataDb &mdIn, AggregateOperation op, const std::vector< MDLabel > &groupByLabels, MDLabel operateLabel, MDLabel resultLabel)

void	aggregateSingle (MDObject &mdValueOut, AggregateOperation op, MDLabel aggregateLabel)

void	aggregateSingleInt (MDObject &mdValueOut, AggregateOperation op, MDLabel aggregateLabel)

void	aggregateSingleSizeT (MDObject &mdValueOut, AggregateOperation op, MDLabel aggregateLabel)

double	getColumnMax (MDLabel column)

double	getColumnMin (MDLabel column)

void	unionDistinct (const MetaDataDb &mdIn, const MDLabel label=MDL_OBJID)

void	unionAll (const MetaDataDb &mdIn)

void	merge (const MetaData &md2)

void	intersection (const MetaDataDb &mdIn, const MDLabel label)

void	subtraction (const MetaDataDb &mdIn, const MDLabel label)

void	distinct (MetaDataDb &MDin, MDLabel label)

void	join1 (const MetaDataDb &mdInLeft, const MetaDataDb &mdInRight, const MDLabel label, JoinType type=LEFT)

void	join2 (const MetaDataDb &mdInLeft, const MetaDataDb &mdInRight, const MDLabel labelLeft, const MDLabel labelRight, JoinType type=LEFT)

void	join1 (const MetaDataDb &mdInLeft, const MetaDataDb &mdInRight, const std::vector< MDLabel > &labels, JoinType type=LEFT)

void	join2 (const MetaDataDb &mdInLeft, const MetaDataDb &mdInRight, const std::vector< MDLabel > &labelsLeft, const std::vector< MDLabel > &labelsRight, JoinType type=LEFT)

void	joinNatural (const MetaDataDb &mdInLeft, const MetaDataDb &mdInRight)

void	operate (const String &expression)

void	replace (const MDLabel label, const String &oldStr, const String &newStr)

void	randomize (const MetaDataDb &MDin)

void	removeDuplicates (MetaDataDb &MDin, MDLabel label=MDL_UNDEFINED)

void	sort (MetaDataDb &MDin, const MDLabel sortLabel, bool asc=true, int limit=-1, int offset=0)

void	sort (MetaDataDb &MDin, const String &sortLabel, bool asc=true, int limit=-1, int offset=0)

void	split (size_t n, std::vector< MetaDataDb > &results, const MDLabel sortLabel=MDL_OBJID)

void	selectSplitPart (const MetaData &mdIn, size_t n, size_t part, const MDLabel sortLabel=MDL_OBJID)

void	selectRandomSubset (const MetaData &mdIn, size_t numberOfObjects, const MDLabel sortLabel=MDL_OBJID) override

void	selectPart (const MetaData &mdIn, size_t startPosition, size_t numberOfObjects, const MDLabel sortLabel=MDL_OBJID) override

void	makeAbsPath (const MDLabel label=MDL_IMAGE)

Public Member Functions inherited from MetaData
virtual bool	nextBlock (mdBuffer &buffer, mdBlock &block)

	MetaData ()=default

virtual	~MetaData ()

virtual bool	isColumnFormat () const

virtual void	setMaxRows (size_t maxRows=0)

virtual size_t	getParsedLines ()

virtual void	setPrecission (int _precision)

virtual void	setColumnFormat (bool column)

virtual void	writeStar (const FileName &outFile, const String &blockName, WriteModeMetaData mode) const

virtual String	getComment () const

virtual void	setComment (const String &newComment="No comment")

virtual FileName	getFilename () const

virtual void	setFilename (const FileName &_filename)

template<class T >
bool	setValueCol (const MDLabel label, const T &valueIn)

template<class T >
bool	setValue (const MDLabel label, const T &valueIn, size_t id)

MetaData &	operator= (const MetaData &md)

template<class T >
T	getValue (const MDLabel label, size_t id) const

template<class T >
bool	getValue (const MDLabel label, T &valueOut, size_t id) const

template<class T >
const T &	getValueOrAbort (const MDLabel label, size_t id) const

template<class T >
T &	getValueOrAbort (const MDLabel label, size_t id)

template<class T >
void	getValueOrAbort (const MDLabel label, T &valueOut, size_t id) const

template<typename T >
const T	getValueOrDefault (const MDLabel label, size_t id, const T &_default) const

template<typename T >
T	getValueOrDefault (const MDLabel label, size_t id, T &_default)

template<typename T , typename T1 >
void	getValueOrDefault (const MDLabel label, T &valueOut, size_t id, const T1 &_default) const

template<class T >
std::vector< T >	getColumnValues (const MDLabel label) const

template<class T >
void	getColumnValues (const MDLabel label, std::vector< T > &out) const

template<class T >
void	setColumnValues (const MDLabel label, const std::vector< T > &valuesIn)

virtual bool	setValueFromStr (const MDLabel label, const String &value, size_t id)

virtual bool	getStrFromValue (const MDLabel label, String &strOut, size_t id) const

virtual bool	isEmpty () const

void	keepLabels (const std::vector< MDLabel > &labels)

virtual void	print () const

void	append (const FileName &outFile) const

virtual void	removeDisabled ()

virtual IdIteratorProxy< false >	ids ()

virtual IdIteratorProxy< true >	ids () const

Protected Member Functions
void	init (const std::vector< MDLabel > &labelsVector)

void	copyMetadata (const MetaDataDb &md, bool copyObjects=true)

void	_selectSplitPart (const MetaDataDb &mdIn, int n, int part, size_t mdSize, const MDLabel sortLabel)

void	_selectSplitPart (const MetaDataDb &mdIn, size_t n, size_t part, const MDLabel sortLabel=MDL_OBJID)

void	_selectRandomSubset (const MetaDataDb &mdIn, size_t numberOfObjects, const MDLabel sortLabel=MDL_OBJID)

void	_selectPart (const MetaDataDb &mdIn, size_t startPosition, size_t numberOfObjects, const MDLabel sortLabel=MDL_OBJID)

void	_setOperates (const MetaDataDb &mdIn, const MDLabel label, SetOperation operation)

void	_setOperates (const MetaDataDb &mdIn, const std::vector< MDLabel > &labels, SetOperation operation)

void	_setOperates (const MetaDataDb &mdInLeft, const MetaDataDb &mdInRight, const std::vector< MDLabel > &labelsLeft, const std::vector< MDLabel > &labelsRight, SetOperation operation)

void	_setOperatesLabel (const MetaDataDb &mdIn, const MDLabel label, SetOperation operation)

void	_clear (bool onlyData=false)

void	_readRowsStar (mdBlock &block, std::vector< MDObject > &columnValues, const std::vector< MDLabel > desiredLabels) override

void	_readRowFormat (std::istream &is)

void	_parseObjects (std::istream &is, std::vector< MDObject > &columnValues, const std::vector< MDLabel > desiredLabels, bool firstTime) override

std::vector< MDObject >	getObjectsForActiveLabels () const

void	_importObjectsDb (const MetaDataDb &md, const MDQuery &query, bool doClear=true)

void	_importObjectsGeneral (const MetaData &md, const MDQuery &query, bool doClear=true)

Protected Member Functions inherited from MetaData
void	copyInfo (const MetaData &md)

double	precision () const

virtual void	readStar (const FileName &filename, const std::vector< MDLabel > *desiredLabels, const String &blockRegExp, bool decomposeStack)

virtual void	_readColumns (std::istream &is, std::vector< MDObject > &columnValues, const std::vector< MDLabel > desiredLabels=nullptr)

virtual void	_readColumnsStar (mdBlock &block, std::vector< MDObject > &columnValues, const std::vector< MDLabel > desiredLabels, bool addColumns=true, size_t id=BAD_OBJID)

virtual void	_parseObject (std::istream &is, MDObject &object, size_t id=BAD_OBJID)

virtual void	_readRows (std::istream &is, std::vector< MDObject *> &columnValues, bool useCommentAsImage)

Protected Attributes
MDSql *	myMDSql

std::vector< MDLabel >	_activeLabels

Protected Attributes inherited from MetaData
std::map< String, size_t >	_fastStringSearch

MDLabel	_fastStringSearchLabel

String	_comment

int	_precision = 1000

FileName	_inFile

size_t	_maxRows = 0

size_t	_parsedLines = 0

Friends
class	MDSql

Additional Inherited Members
Public Types inherited from MetaData
using	iterator = rowIterator< false >

using	const_iterator = rowIterator< true >

using	id_iterator = idIterator< false >

using	id_const_iterator = idIterator< true >

Public Attributes inherited from MetaData
bool	isMetadataFile

Detailed Description

Original database implementation of MetaData. MetaData are stored in SQL database. Some database-specific commands are available in this implementation only.

Notes

It's quite fast to iterate over ids.
It's slow to iterate over rows.
Best practices: https://github.com/I2PC/xmipp-portal/wiki/MetaData—SQL-best-practices

Definition at line 55 of file metadata_db.h.

Constructor & Destructor Documentation

◆ MetaDataDb() [1/5]

MetaDataDb::MetaDataDb ( )

Empty Constructor.

The MetaDataDb is created with no data stored on it. You can fill in it programmatically or by a later reading from a MetaDataDb file or old Xmipp formatted type. if labels vectors is passed this labels are created on metadata

Definition at line 496 of file metadata_db.cpp.

 {
     myMDSql = new MDSql(this);
     init({});
 }//close MetaData default Constructor

◆ MetaDataDb() [2/5]

MetaDataDb::MetaDataDb ( const std::vector< MDLabel > & labelsVector )

Definition at line 508 of file metadata_db.cpp.

 {
     myMDSql = new MDSql(this);
     init(labelsVector);
 }//close MetaData default Constructor

◆ MetaDataDb() [3/5]

MetaDataDb::MetaDataDb ( const MetaData & md )

Definition at line 502 of file metadata_db.cpp.

                                          {
     myMDSql = new MDSql(this);
     init({});
     MetaData::operator=(md);
 }

◆ MetaDataDb() [4/5]

MetaDataDb::MetaDataDb	(	const FileName &	fileName,
		const std::vector< MDLabel > &	desiredLabels = `{}`
	)

From File Constructor.

The MetaData is created and data is read from provided FileName. Optionally, a vector of labels can be provided to read just those required labels

Definition at line 514 of file metadata_db.cpp.

 {
     myMDSql = new MDSql(this);
     init(desiredLabels);
     read(fileName, desiredLabels.empty() ? nullptr : &desiredLabels);
 }//close MetaData from file Constructor

◆ MetaDataDb() [5/5]

MetaDataDb::MetaDataDb ( const MetaDataDb & md )

Copy constructor

Created a new metadata by copying all data from an existing MetaData object.

Definition at line 521 of file metadata_db.cpp.

 {
     myMDSql = new MDSql(this);
     copyMetadata(md);
 }//close MetaData copy Constructor

◆ ~MetaDataDb()

MetaDataDb::~MetaDataDb ( )

virtual

Destructor

Frees all used memory and destroys object.

Definition at line 533 of file metadata_db.cpp.

 {
     _clear();
     delete myMDSql;
 }//close MetaData Destructor

Member Function Documentation

◆ _clear()

void MetaDataDb::_clear ( bool onlyData = false )

protected

clear data and table structure

Definition at line 40 of file metadata_db.cpp.

 {
     if (onlyData)
     {
         myMDSql->deleteObjects();
     }
     else
     {
         MetaData::clear();
         _activeLabels.clear();
         myMDSql->clearMd();
     }
 }//close clear

◆ _importObjectsDb()

void MetaDataDb::_importObjectsDb	(	const MetaDataDb &	md,
		const MDQuery &	query,
		bool	doClear = `true`
	)

protected

Definition at line 612 of file metadata_db.cpp.

 {
     if (doClear)
     {
         //Copy all structure and info from the other metadata
         init(md._activeLabels);
         copyInfo(md);
     }
     else
     {
         //If not clear, ensure that the have the same labels
         for (size_t i = 0; i < md._activeLabels.size(); i++)
             addLabel(md._activeLabels[i]);
     }
     md.myMDSql->copyObjects(this, &query);
 }

◆ _importObjectsGeneral()

void MetaDataDb::_importObjectsGeneral	(	const MetaData &	md,
		const MDQuery &	query,
		bool	doClear = `true`
	)

protected

◆ _parseObjects()

void MetaDataDb::_parseObjects	(	std::istream &	is,
		std::vector< MDObject *> &	columnValues,
		const std::vector< MDLabel > *	desiredLabels,
		bool	firstTime
	)

overrideprotectedvirtual

Implements MetaData.

Definition at line 853 of file metadata_db.cpp.

 {
     size_t i=0;             // Loop counter.
     size_t size=0;          // Column values vector size.
 
     // Columns loop.
     size = columnValues.size();
     for (i=0; i<size ;i++)
     {
         columnValues[i]->fromStream(is);
         if (is.fail())
         {
            String errorMsg = formatString("MetaData: Error parsing column '%s' value.", MDL::label2Str(columnValues[i]->label).c_str());
            columnValues[i]->failed = true;
            std::cerr << "WARNING: " << errorMsg << std::endl;
            //REPORT_ERROR(ERR_MD_BADLABEL, (String)"read: Error parsing data column, expecting " + MDL::label2Str(object.label));
         }
         else
         {
             if (firstTime)
             {
                 // Check if current column label exists.
                 if (columnValues[i]->label != MDL_UNDEFINED)
                 {
                     // If there are no desired labels then add all.
                     bool reallyAdd=false;
                     if (desiredLabels==NULL)
                     {
                         reallyAdd=true;
                     }
                     else
                     {
                         // Check if current column belongs to desired labels.
                         for (size_t j=0; j<desiredLabels->size(); ++j)
                         {
                             if ((*desiredLabels)[j]==columnValues[i]->label)
                             {
                                 reallyAdd=true;
                                 break;
                             }
                         }
                     }
 
                     // Add label if not exists.
                     if (reallyAdd)
                     {
                         addLabel(columnValues[i]->label);
                     }
                 }
             }
         }
     }
 
     // Insert elements in DB.
     myMDSql->setObjectValues( -1, columnValues, desiredLabels);
 }

◆ _readRowFormat()

void MetaDataDb::_readRowFormat ( std::istream & is )

protected

Some private reading functions

Definition at line 1096 of file metadata_db.cpp.

                                               {
     String line, token;
     MDLabel label;
 
     size_t objectID = addObject();
 
     // Read data and fill structures accordingly
     while (getline(is, line, '\n'))
     {
         if (line[0] == '#' || line[0] == '\0' || line[0] == ';')
             continue;
 
         // Parse labels
         std::stringstream os(line);
 
         os >> token;
         label = MDL::str2Label(token);
         MDObject value(label);
         os >> value;
         if (label != MDL_UNDEFINED)
             setValue(value, objectID);
     }
 }

◆ _readRowsStar()

void MetaDataDb::_readRowsStar	(	mdBlock &	block,
		std::vector< MDObject *> &	columnValues,
		const std::vector< MDLabel > *	desiredLabels
	)

overrideprotectedvirtual

This function will be used to parse the rows data in START format

Parameters

[out]	columnValues	MDRow with values to fill in
	pchStart	pointer to the position of '_loop' in memory
	pEnd	pointer to the position of the next '_data' in memory
	maxRows	if this number if greater than 0, only this number of rows will be parsed.

Reimplemented from MetaData.

Definition at line 1051 of file metadata_db.cpp.

                                                                         {
     String line;
     std::stringstream ss;
     size_t n = block.end - block.loop;
     bool firstTime = true;
 
     if (n == 0)
         return;
 
     char * buffer = new char[n];
     memcpy(buffer, block.loop, n);
     char *iter = buffer, *end = iter + n, * newline = NULL;
     _parsedLines = 0; //Check how many lines the md have
 
     if (myMDSql->initializeInsert( desiredLabels, columnValues))
     {
         while (iter < end) { //while there are data lines
             //Adding \n position and check if NULL at the same time
             if (!(newline = END_OF_LINE()))
                 newline = end;
             line.assign(iter, newline - iter);
             trim(line);
 
             if (!line.empty() && line[0] != '#') {
                 //_maxRows would be > 0 if we only want to read some
                 // rows from the md for performance reasons...
                 // anyway the number of lines will be counted in _parsedLines
                 if (_maxRows == 0 || _parsedLines < _maxRows) {
                     std::stringstream ss(line);
                     this->_parseObjects(ss, columnValues, desiredLabels, firstTime);
                     firstTime = false;
                 }
                 _parsedLines++;
             }
             iter = newline + 1; //go to next line
         }
 
         myMDSql->finalizePreparedStmt();
     }
 
     delete[] buffer;
 }

◆ _selectPart()

void MetaDataDb::_selectPart	(	const MetaDataDb &	mdIn,
		size_t	startPosition,
		size_t	numberOfObjects,
		const MDLabel	sortLabel = `MDL_OBJID`
	)

protected

Definition at line 1609 of file metadata_db.cpp.

 {
     size_t mdSize = mdIn.size();
     if (startPosition < 0 || startPosition >= mdSize)
         REPORT_ERROR(ERR_MD, "selectPart: 'startPosition' should be between 0 and size()-1");
     init(mdIn._activeLabels);
     copyInfo(mdIn);
     mdIn.myMDSql->copyObjects(this, new MDQuery(numberOfObjects, startPosition, sortLabel));
 }

◆ _selectRandomSubset()

void MetaDataDb::_selectRandomSubset	(	const MetaDataDb &	mdIn,
		size_t	numberOfObjects,
		const MDLabel	sortLabel = `MDL_OBJID`
	)

protected

Definition at line 1591 of file metadata_db.cpp.

 {
     clear();
 
     MetaDataDb mdAux, mdAux2;
     mdAux.randomize(mdIn);
     mdAux2.selectPart(mdAux, 0, numberOfObjects);
     sort(mdAux2,sortLabel);
 }

◆ _selectSplitPart() [1/2]

void MetaDataDb::_selectSplitPart	(	const MetaDataDb &	mdIn,
		int	n,
		int	part,
		size_t	mdSize,
		const MDLabel	sortLabel
	)

protected

This have the same logic of the public one, but doesn't perform any range(which implies do a size()) checks.

Definition at line 1555 of file metadata_db.cpp.

 {
     size_t first, last, n_images;
     n_images = divide_equally(mdSize, n, part, first, last);
     init(mdIn._activeLabels);
     copyInfo(mdIn);
     mdIn.myMDSql->copyObjects(this, new MDQuery(n_images, first, sortLabel));
 }

◆ _selectSplitPart() [2/2]

void MetaDataDb::_selectSplitPart	(	const MetaDataDb &	mdIn,
		size_t	n,
		size_t	part,
		const MDLabel	sortLabel = `MDL_OBJID`
	)

protected

Definition at line 1573 of file metadata_db.cpp.

 {
     size_t mdSize = mdIn.size();
     if (n > mdSize)
         REPORT_ERROR(ERR_MD, "selectSplitPart: Couldn't split a metadata in more parts than its size");
     if (part < 0 || part >= n)
         REPORT_ERROR(ERR_MD, "selectSplitPart: 'part' should be between 0 and n-1");
     _selectSplitPart(mdIn, n, part, mdSize, sortLabel);
 
 }

◆ _setOperates() [1/3]

void MetaDataDb::_setOperates	(	const MetaDataDb &	mdIn,
		const MDLabel	label,
		SetOperation	operation
	)

protected

This function is for generalize the sets operations of unionDistinct, intersection, subtraction which can be expressed in terms of ADD, SUBSTRACT of intersection part

Definition at line 1293 of file metadata_db.cpp.

 {
     std::vector<MDLabel> labels;
     labels.emplace_back(label);
     _setOperates(mdIn,labels,operation);
 }

◆ _setOperates() [2/3]

void MetaDataDb::_setOperates	(	const MetaDataDb &	mdIn,
		const std::vector< MDLabel > &	labels,
		SetOperation	operation
	)

protected

Definition at line 1302 of file metadata_db.cpp.

 {
     if (this == &mdIn) //not sense to operate on same metadata
         REPORT_ERROR(ERR_MD, "Couldn't perform this operation on input metadata");
     if (size() == 0 && mdIn.size() == 0)
         REPORT_ERROR(ERR_MD, "Couldn't perform this operation if both metadata are empty");
     //Add labels to be sure are present
     for (size_t i = 0; i < mdIn._activeLabels.size(); i++)
         addLabel(mdIn._activeLabels[i]);
 
     mdIn.myMDSql->setOperate(this, labels, operation);
 }

◆ _setOperates() [3/3]

void MetaDataDb::_setOperates	(	const MetaDataDb &	mdInLeft,
		const MetaDataDb &	mdInRight,
		const std::vector< MDLabel > &	labelsLeft,
		const std::vector< MDLabel > &	labelsRight,
		SetOperation	operation
	)

protected

Definition at line 1332 of file metadata_db.cpp.

 {
     if (this == &mdInLeft || this == &mdInRight) //not sense to operate on same metadata
         REPORT_ERROR(ERR_MD, "Couldn't perform this operation on input metadata");
     //Add labels to be sure are present
     for (size_t i = 0; i < mdInLeft._activeLabels.size(); i++)
         addLabel(mdInLeft._activeLabels[i]);
     for (size_t i = 0; i < mdInRight._activeLabels.size(); i++)
     {
         bool found=false;
         for (size_t j=0; j<labelsRight.size(); ++j)
             if (mdInRight._activeLabels[i]==labelsRight[j])
             {
                 found=true;
                 break;
             }
         if (!found)
             addLabel(mdInRight._activeLabels[i]);
     }
 
     myMDSql->setOperate(&mdInLeft, &mdInRight, labelsLeft,labelsRight, operation);
 }

◆ _setOperatesLabel()

void MetaDataDb::_setOperatesLabel	(	const MetaDataDb &	mdIn,
		const MDLabel	label,
		SetOperation	operation
	)

protected

This function is for generalize the sets operations in which the output has a single label a vector of labels instead of a single label may be implemented in the future

Definition at line 1317 of file metadata_db.cpp.

 {
     if (this == &mdIn) //not sense to operate on same metadata
         REPORT_ERROR(ERR_MD, "Couldn't perform this operation on input metadata");
     if (mdIn.size() == 0)
         REPORT_ERROR(ERR_MD, "Couldn't perform this operation if both metadata are empty");
     //Add label to be sure is present in output
     addLabel(label);
     std::vector<MDLabel> labels;
     labels.emplace_back(label);
     mdIn.myMDSql->setOperate(this, labels, operation);
 }

◆ _writeRows()

void MetaDataDb::_writeRows ( std::ostream & os ) const

overridevirtual

Write rows data to disk.

Implements MetaData.

Definition at line 774 of file metadata_db.cpp.

 {
 
     auto sortedLabels = this->_activeLabels;
     std::sort(sortedLabels.begin(), sortedLabels.end());    
     for (const auto& row : *this)
     {
         for (size_t i = 0; i < sortedLabels.size(); i++)
         {
             if (sortedLabels[i] != MDL_STAR_COMMENT)
             {
                 os.width(1);
                 row.getObject(sortedLabels[i])->toStream(os, true);
                 os << " ";
             }
         }
 
         os << '\n';
     }
 }

◆ addIndex() [1/2]

void MetaDataDb::addIndex ( MDLabel label ) const

Add and remove indexes for fast search in other labels, but insert are more expensive

Definition at line 653 of file metadata_db.cpp.

 {
     std::vector<MDLabel> labels(1);
     labels[0]=label;
     addIndex(labels);
 }

◆ addIndex() [2/2]

void MetaDataDb::addIndex ( const std::vector< MDLabel > & desiredLabels ) const

Definition at line 659 of file metadata_db.cpp.

 {
 
     myMDSql->indexModify(desiredLabels, true);
 }

◆ addItemId()

void MetaDataDb::addItemId ( )

Add item id. From 1 to last.

Definition at line 677 of file metadata_db.cpp.

 {
     addLabel(MDL_ITEM_ID);
     fillLinear(MDL_ITEM_ID,1,1);
 }

◆ addLabel()

bool MetaDataDb::addLabel	(	const MDLabel	label,
		int	pos = `-1`
	)

overridevirtual

Add a new label to the metadata. By default the label is added at the end, if the position is specified and is between 0 and n-1 the new label is inserted at that position.

Implements MetaData.

Definition at line 554 of file metadata_db.cpp.

 {
     if (containsLabel(label))
         return false;
     if (pos < 0 || pos >= (int)this->_activeLabels.size())
         this->_activeLabels.emplace_back(label);
     else
         this->_activeLabels.insert(this->_activeLabels.begin() + pos, label);
     myMDSql->addColumn(label);
     return true;
 }

◆ addMissingLabels()

void MetaDataDb::addMissingLabels ( const MDRow & row )

Definition at line 422 of file metadata_db.cpp.

                                                   {
     // find missing labels
     std::vector<MDLabel> missingLabels;
     auto definedLabels = row.labels();
     for (const auto &l : definedLabels){
         if ( ! containsLabel(l)) {
             missingLabels.emplace_back(l);
         }
     }
     // add missing labels
     if ( ! missingLabels.empty()) {
         sqlUtils::addColumns(missingLabels,
                     myMDSql->db,
                     myMDSql->tableName(myMDSql->tableId));
         this->_activeLabels.insert(this->_activeLabels.end(), missingLabels.begin(), missingLabels.end());
     }
 }

◆ addObject()

size_t MetaDataDb::addObject ( )

overridevirtual

Adds a new, empty object to the objects map. If objectId == -1 the new ID will be that for the last object inserted + 1, else the given objectId is used. If there is already an object whose objectId == input objectId, just removes it and creates an empty one

Implements MetaData.

Definition at line 578 of file metadata_db.cpp.

 {
     return (size_t)myMDSql->addRow();
 }

◆ addPlain()

void MetaDataDb::addPlain	(	const FileName &	inFile,
		const String &	labelsString,
		const String &	separator = `" "`
	)

Same as readPlain, but instead of cleanning data, the readed values will be added. If there are common columns in metadata and the plain text, the lattest will be setted

Definition at line 975 of file metadata_db.cpp.

 {
     MetaDataDb md2;
     md2.readPlain(inFile, labelsString);
     merge(md2);
 }

◆ addRow()

size_t MetaDataDb::addRow ( const MDRow & row )

overridevirtual

Implements MetaData.

Definition at line 394 of file metadata_db.cpp.

 {
     size_t id = addObject();
     for (auto obj : row)
         if (obj->label != MDL_FIRST_LABEL)
             setValue(*obj, id);
 
     return id;
 }

◆ addRow2()

size_t MetaDataDb::addRow2 ( const MDRow & row )

Definition at line 475 of file metadata_db.cpp.

 {
     size_t id = BAD_OBJID;
 
     // Initialize INSERT.
     if (initAddRow( row))
     {
         // Execute INSERT.
         if (execAddRow( row))
         {
             // Get last inserted row id.
             id = myMDSql->getObjId();
         }
 
         // Finalize INSERT.
         finalizeAddRow();
     }
 
     return(id);
 }

◆ addRowOpt()

void MetaDataDb::addRowOpt ( const MDRowSql & row )

Definition at line 417 of file metadata_db.cpp.

 {
     addRows({row});
 }

◆ addRows()

void MetaDataDb::addRows ( const std::vector< MDRowSql > & rows )

Definition at line 440 of file metadata_db.cpp.

 {
     const auto noOfRows = rows.size();
     if (0 == noOfRows) {
         return;
     }
     const auto &firstRow = rows.at(0);
 
     // assuming all rows are using the same labels
     addMissingLabels(firstRow);
 
     // create mask of valid labels
     std::vector<MDLabel> labels;
     labels.reserve(firstRow.size());
     for (const MDObject* obj : firstRow)
         labels.emplace_back(obj->label);
     const auto noOfLabels = labels.size();
 
     // extract values to be added
     std::vector<std::vector<const MDObject*>> records;
     records.reserve(noOfRows);
     for (const auto &r : rows) {
         records.emplace_back(std::vector<const MDObject*>());
         auto &vals = records.back();
         vals.reserve(noOfLabels);
         for (const auto &l : labels) {
             vals.emplace_back(r.getObject(l));
         }
     }
     // insert values to db
     sqlUtils::insert(records, myMDSql->db,
                     myMDSql->tableName(myMDSql->tableId));
 }

◆ aggregate() [1/2]

void MetaDataDb::aggregate	(	const MetaDataDb &	mdIn,
		AggregateOperation	op,
		MDLabel	aggregateLabel,
		MDLabel	operateLabel,
		MDLabel	resultLabel
	)

Aggregate metadata objects, result in calling metadata object (except for aggregateSingle) thisLabel label is used for aggregation, second. Valid operations are:

MDL_AVG: The avg function returns the average value of all operationLabel within a group. The result of avg is always a floating point value as long as at there is at least one non-NULL input even if all inputs are integers. The result of avg is NULL if and only if there are no non-NULL inputs.

AGGR_COUNT: The count function returns a count of the number of times that operationLabel is in a group.

AGGR_MAX The max aggregate function returns the maximum value of all values in the group.

AGGR_MIN The min aggregate function returns the minimum value of all values in the group.

AGGRL_SUM The total aggregate functions return sum of all values in the group. If there are no non-NULL input rows then returns 0.0.

Definition at line 1255 of file metadata_db.cpp.

 {
     std::vector<MDLabel> labels(2);
     std::vector<MDLabel> operateLabels(1);
     labels[0] = aggregateLabel;
     labels[1] = resultLabel;
     operateLabels[0]=operateLabel;
     init(labels);
     std::vector<AggregateOperation> ops(1);
     ops[0] = op;
     mdIn.myMDSql->aggregateMd(this, ops, operateLabels);
 }

◆ aggregate() [2/2]

void MetaDataDb::aggregate	(	const MetaDataDb &	mdIn,
		const std::vector< AggregateOperation > &	ops,
		const std::vector< MDLabel > &	operateLabels,
		const std::vector< MDLabel > &	resultLabels
	)

Definition at line 1269 of file metadata_db.cpp.

 {
     if (resultLabels.size() - ops.size() != 1)
         REPORT_ERROR(ERR_MD, "Labels vectors should contain one element more than operations");
     init(resultLabels);
     mdIn.myMDSql->aggregateMd(this, ops, operateLabels);
 }

◆ aggregateGroupBy()

void MetaDataDb::aggregateGroupBy	(	const MetaDataDb &	mdIn,
		AggregateOperation	op,
		const std::vector< MDLabel > &	groupByLabels,
		MDLabel	operateLabel,
		MDLabel	resultLabel
	)

Definition at line 1279 of file metadata_db.cpp.

 {
     std::vector<MDLabel> labels;
     labels = groupByLabels;
     labels.emplace_back(resultLabel);
     init(labels);
     mdIn.myMDSql->aggregateMdGroupBy(this, op, groupByLabels, operateLabel, resultLabel);
 }

◆ aggregateSingle()

void MetaDataDb::aggregateSingle	(	MDObject &	mdValueOut,
		AggregateOperation	op,
		MDLabel	aggregateLabel
	)

This function performs aggregation operations. without grouping. (i.e. absolute maximum of a metadata column) for double

Definition at line 1216 of file metadata_db.cpp.

 {
     mdValueOut.setValue(myMDSql->aggregateSingleDouble(op,aggregateLabel));
 }

◆ aggregateSingleInt()

void MetaDataDb::aggregateSingleInt	(	MDObject &	mdValueOut,
		AggregateOperation	op,
		MDLabel	aggregateLabel
	)

This function performs aggregation operations. without grouping. (i.e. absolute maximum of a metadata column) for int

Definition at line 1246 of file metadata_db.cpp.

 {
     size_t aux = myMDSql->aggregateSingleSizeT(op,aggregateLabel);
     int aux2 = (int) aux;
     mdValueOut.setValue(aux2);
 }

◆ aggregateSingleSizeT()

void MetaDataDb::aggregateSingleSizeT	(	MDObject &	mdValueOut,
		AggregateOperation	op,
		MDLabel	aggregateLabel
	)

This function performs aggregation operations. without grouping. (i.e. absolute maximum of a metadata column) for size_t

Definition at line 1223 of file metadata_db.cpp.

 {
     mdValueOut.setValue(myMDSql->aggregateSingleSizeT(op,aggregateLabel));
 }

◆ begin() [1/2]

iterator MetaDataDb::begin ( )

inlineoverridevirtual

Implements MetaData.

Definition at line 746 of file metadata_db.h.

                               {
         return {memoryUtils::make_unique<MDDbRowIterator<false>>(*this, 0)};
     }

◆ begin() [2/2]

const_iterator MetaDataDb::begin ( ) const

inlineoverridevirtual

Implements MetaData.

Definition at line 753 of file metadata_db.h.

                                           {
         return {memoryUtils::make_unique<MDDbRowIterator<true>>(*this, 0)};
     }

◆ bindValue()

bool MetaDataDb::bindValue ( size_t id ) const

Get all values of an MetaData row of an specified objId

Definition at line 155 of file metadata_db.cpp.

 {
     bool success=true;
 
     // Prepare statement.
     if (!myMDSql->bindStatement( id))
     {
         success = false;
     }
 
     return success;
 }

◆ clear()

void MetaDataDb::clear ( )

overridevirtual

Clear all data

Reimplemented from MetaData.

Definition at line 54 of file metadata_db.cpp.

 {
     init({});
 }

◆ containsLabel()

bool MetaDataDb::containsLabel ( const MDLabel label ) const

inlineoverridevirtual

Check whether a label is contained in metadata.

Implements MetaData.

Definition at line 305 of file metadata_db.h.

                                                            {
         return vectorContainsLabel(this->_activeLabels, label);
     }

◆ containsObject() [1/2]

bool MetaDataDb::containsObject ( size_t objectId ) const

overridevirtual

Implements MetaData.

Definition at line 729 of file metadata_db.cpp.

 {
     return containsObject(MDValueEQ(MDL_OBJID, objectId));
 }

◆ containsObject() [2/2]

bool MetaDataDb::containsObject ( const MDQuery & query ) const

overridevirtual

Implements MetaData.

Definition at line 734 of file metadata_db.cpp.

 {
     std::vector<size_t> objects;
     findObjects(objects, query);
     return objects.size() > 0;
 }

◆ copyColumn()

void MetaDataDb::copyColumn	(	MDLabel	labelDest,
		MDLabel	labelSrc
	)

overridevirtual

Copy all values from one column to another. Source column should exist

Implements MetaData.

Definition at line 1175 of file metadata_db.cpp.

 {
     String srcName = MDL::label2Str(labelSrc);
     if (!containsLabel(labelSrc))
         REPORT_ERROR(ERR_ARG_MISSING, formatString("Source label: '%s' doesn't exist on metadata", srcName.c_str()));
     addLabel(labelDest);
 
     String destName = MDL::label2Str(labelDest);
     String cmd = formatString("%s=%s", destName.c_str(), srcName.c_str());
     operate(cmd);
 }

◆ copyColumnTo()

void MetaDataDb::copyColumnTo	(	MetaData &	md,
		MDLabel	labelDest,
		MDLabel	labelSrc
	)

overridevirtual

Same as previous, but copy to another metadata

Implements MetaData.

Definition at line 1187 of file metadata_db.cpp.

 {
     if (!containsLabel(labelSrc))
         REPORT_ERROR(ERR_ARG_MISSING, formatString("Source label: '%s' doesn't exist on metadata",
                      (MDL::label2Str(labelSrc)).c_str()));
     md.addLabel(labelDest);
     std::vector<MDObject> values;
     getColumnValues(labelSrc, values);
     md.setColumnValues(values);
 }

◆ copyMetadata()

void MetaDataDb::copyMetadata	(	const MetaDataDb &	md,
		bool	copyObjects = `true`
	)

protected

Copy all data from another metadata

Definition at line 71 of file metadata_db.cpp.

 {
     if (this == &md) //not sense to copy same metadata
         return;
     init(md._activeLabels);
     copyInfo(md);
     if (!md._activeLabels.empty())
     {
         if (copyObjects)
             md.myMDSql->copyObjects(this);
     }
     else
     {
         int n = md.size();
         for (int i = 0; i < n; i++)
             addObject();
     }
 }

◆ countObjects()

size_t MetaDataDb::countObjects ( const MDQuery & query ) const

overridevirtual

Implements MetaData.

Definition at line 722 of file metadata_db.cpp.

 {
     std::vector<size_t> objects;
     findObjects(objects, query);
     return objects.size();
 }

◆ distinct()

void MetaDataDb::distinct	(	MetaDataDb &	MDin,
		MDLabel	label
	)

Return only distinct (different) values of column label. Result in "calling" metadata with a single column

Definition at line 1389 of file metadata_db.cpp.

 {
     if(MDin.isEmpty())
         return;
     _setOperatesLabel(MDin, label, DISTINCT);
 }

◆ end() [1/2]

iterator MetaDataDb::end ( )

inlineoverridevirtual

Implements MetaData.

Definition at line 749 of file metadata_db.h.

                             {
         return {memoryUtils::make_unique<MDDbRowIterator<false>>(*this, this->size())};
     }

◆ end() [2/2]

const_iterator MetaDataDb::end ( ) const

inlineoverridevirtual

Implements MetaData.

Definition at line 756 of file metadata_db.h.

                                         {
         return {memoryUtils::make_unique<MDDbRowIterator<true>>(*this, this->size())};
     }

◆ execAddRow()

bool MetaDataDb::execAddRow ( const MDRow & row )

Definition at line 361 of file metadata_db.cpp.

 {
     int j = 0;
     bool success = true;
     std::vector<const MDObject*> mdValues;
 
     // Set values vector size.
     mdValues.resize(row.size());
 
     // Get values to insert.
     j = 0;
     for (const MDObject* obj : row) {
         addLabel(obj->label);
         mdValues[j] = row.getObject(obj->label);
         j++;
     }
     mdValues.resize(j);
 
     // Execute statement.
     if (!myMDSql->setObjectValues( -1, mdValues))
     {
         std::cerr << "execAddRow: error executing myMDSql->setObjectValues" << std::endl;
         success = false;
     }
 
     return(success);
 }

◆ execGetRow()

bool MetaDataDb::execGetRow ( MDRow & row ) const

Definition at line 181 of file metadata_db.cpp.

 {
     std::vector<MDObject> mdValues;
     mdValues.reserve(this->_activeLabels.size());
 
     row.clear();
 
     bool success = myMDSql->getObjectsValues(this->_activeLabels, mdValues);
     if (success)
         for (const auto &obj : mdValues)
             row.setValue(obj);
 
     return success;
 }

◆ existsBlock()

bool MetaDataDb::existsBlock ( const FileName & _inFile )

Check if block exists in metadata file input full parh block return false if metadata block does not exits

Definition at line 982 of file metadata_db.cpp.

 {
 #ifdef XMIPP_MMAP
     String blockName;
     FileName outFile;
 
     blockName=_inFile.getBlockName();
     outFile = _inFile.removeBlockName();
 
     struct stat file_status;
     int fd;
     char *map;
 
     //check if file exists or not block name has been given
     //in our format no two identical data_xxx strings may exists
 
     if (blockName.empty() || !outFile.exists())
         return false;
     else
     {
         //does blockname exists?
         //remove it from file in this case
         // get length of file:
         if(stat(outFile.c_str(), &file_status) != 0)
             REPORT_ERROR(ERR_IO_NOPATH,"Metadata:existsBlock can not get filesize for file "+outFile);
         size_t size = file_status.st_size;
         if(size!=0)//size=0 for /dev/stderr
         {
             fd = open(outFile.c_str(),  O_RDWR, S_IREAD | S_IWRITE);
             if (fd == -1)
                 REPORT_ERROR(ERR_IO_NOPATH,"Metadata:existsBlock can not read file named "+outFile);
 
             map = (char *) mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
             if (map == MAP_FAILED)
                 REPORT_ERROR(ERR_MEM_BADREQUEST,"Metadata:existsBlock can not map memory ");
 
             // Is this a START formatted FILE
             String _szBlockName = (String)("\ndata_") + blockName;
             size_t blockNameSize = _szBlockName.size();
             close(fd);
             bool found=_memmem(map, size, _szBlockName.data(), blockNameSize) != NULL;
             if (munmap(map, size) == -1)
                 REPORT_ERROR(ERR_MEM_NOTDEALLOC,"metadata:write, Can not unmap memory");
             return found;
         }
         return false;
     }
 #else
     REPORT_ERROR(ERR_MMAP,"Mapping not supported in Windows");
 #endif
 }

◆ fillConstant()

void MetaDataDb::fillConstant	(	MDLabel	label,
		const String &	value
	)

overridevirtual

Fill column with constant value

Implements MetaData.

Definition at line 1157 of file metadata_db.cpp.

 {
     MDConstGenerator generator(value);
     SET_AND_FILL();
 }

◆ fillExpand()

void MetaDataDb::fillExpand ( MDLabel label )

Expand Metadata with metadata pointed by label Given a metadata md1, with a column containing the name of another column metdata file mdxx add the columns in mdxx to md1

Definition at line 1131 of file metadata_db.cpp.

 {
     //aggregate metadata by label (that is, avoid repetitions
     MetaDataDb mdCTFs;
     mdCTFs.distinct(*this,label);
     //read file-metadatas in new metadata
     MetaDataDb ctfModel;
     FileName fn;
     MDRowSql row;
 
     for (size_t id : mdCTFs.ids())
     {
         if (mdCTFs.getValue(label, fn, id))
         {
             ctfModel.read(fn);
             if (ctfModel.isEmpty())
                 REPORT_ERROR(ERR_VALUE_INCORRECT, "Only can expand non empty metadatas");
             ctfModel.getRow(row, ctfModel.firstRowId());
             mdCTFs.setRow(row, id);
         }
     }
     //join
     MetaDataDb md(*this);
     join1(md, mdCTFs, label);
 }

◆ fillLinear()

void MetaDataDb::fillLinear	(	MDLabel	label,
		double	initial,
		double	step
	)

overridevirtual

Fill lineal, starting at some value and with some step

Implements MetaData.

Definition at line 1169 of file metadata_db.cpp.

 {
     MDLinealGenerator generator(initial, step);
     SET_AND_FILL();
 }

◆ fillRandom()

void MetaDataDb::fillRandom	(	MDLabel	label,
		const String &	mode,
		double	op1,
		double	op2,
		double	op3 = `0.`
	)

overridevirtual

Fill column with random value mode should be: uniform, gaussian or student op1, op2 and op2 are interpreted for each mode: uniform: op1 and op2 are the limits of the interval gaussian: op1 and op2 are mean and std student: same as gaussian and use op3

Implements MetaData.

Definition at line 1163 of file metadata_db.cpp.

 {
     MDRandGenerator generator(op1, op2, mode, op3);
     SET_AND_FILL();
 }

◆ finalizeAddRow()

void MetaDataDb::finalizeAddRow ( void )

Definition at line 389 of file metadata_db.cpp.

 {
     myMDSql->finalizePreparedStmt();
 }

◆ finalizeGetRow()

void MetaDataDb::finalizeGetRow ( void ) const

Definition at line 196 of file metadata_db.cpp.

 {
     myMDSql->finalizePreparedStmt();
 }

◆ findObjects() [1/2]

void MetaDataDb::findObjects	(	std::vector< size_t > &	objectsOut,
		const MDQuery &	query
	)		const

overridevirtual

Find all objects that match a query. if called without query, all objects are returned if limit is provided only return a maximun of 'limit'

Implements MetaData.

Definition at line 709 of file metadata_db.cpp.

 {
     objectsOut.clear();
     myMDSql->selectObjects(objectsOut, &query);
 }

◆ findObjects() [2/2]

void MetaDataDb::findObjects	(	std::vector< size_t > &	objectsOut,
		int	limit = `-1`
	)		const

overridevirtual

Implements MetaData.

Definition at line 715 of file metadata_db.cpp.

 {
     objectsOut.clear();
     MDQuery query(limit);
     myMDSql->selectObjects(objectsOut, &query);
 }

◆ firstObject()

size_t MetaDataDb::firstObject ( const MDQuery & query ) const

overridevirtual

Implements MetaData.

Definition at line 695 of file metadata_db.cpp.

 {
     std::vector<size_t> ids;
     findObjects(ids, query);
     size_t id = ids.size() == 1 ? ids[0] : BAD_OBJID;
     return id;
 }

◆ firstRowId()

size_t MetaDataDb::firstRowId ( ) const

overridevirtual

Return the object id of the first element in metadata.

Implements MetaData.

Definition at line 690 of file metadata_db.cpp.

 {
     return myMDSql->firstRow();
 }

◆ getActiveLabels()

std::vector<MDLabel> MetaDataDb::getActiveLabels ( ) const

inlineoverridevirtual

Get safe access to active labels.

Implements MetaData.

Definition at line 309 of file metadata_db.h.

                                                         {
         return this->_activeLabels;
     }

◆ getAllRows()

bool MetaDataDb::getAllRows ( std::vector< MDRowSql > & rows ) const

Definition at line 212 of file metadata_db.cpp.

 {
     std::vector<std::vector<MDObject>> rawRows;
     rawRows.reserve(this->size());
     auto columns = getObjectsForActiveLabels();
     if ( !  sqlUtils::select(myMDSql->db,
             myMDSql->tableName(myMDSql->tableId),
             columns,
             rawRows)) return false;
 
     rows.clear();
     const auto noOfRows = rawRows.size();
     rows.resize(noOfRows);
     for (size_t i = 0; i < noOfRows; ++i) {
         auto &row = rows.at(i);
         const auto &vals = rawRows.at(i);
         // fill the row
         for (auto &v : vals) {
             row.setValue(v);
         }
     }
     return true;
 }

◆ getColumnMax()

double MetaDataDb::getColumnMax ( MDLabel column )

Returns Max and Min values from a column in metadata These functions can only be used for labels of type double

Definition at line 1231 of file metadata_db.cpp.

 {
     MDObject result(column);
     aggregateSingle(result, AGGR_MAX, column);
     return result.getValue2(double());
 }

◆ getColumnMin()

double MetaDataDb::getColumnMin ( MDLabel column )

Definition at line 1238 of file metadata_db.cpp.

 {
     MDObject result(column);
     aggregateSingle(result, AGGR_MIN, column);
     return result.getValue2(double());
 }

◆ getColumnValues() [1/3]

void MetaDataDb::getColumnValues	(	const MDLabel	label,
		std::vector< MDObject > &	valuesOut
	)		const

overridevirtual

Get all values of a column as a vector.

Implements MetaData.

Definition at line 120 of file metadata_db.cpp.

 {
     MDObject mdValueOut(label);
     std::vector<size_t> objectsId;
     findObjects(objectsId);
     size_t n = objectsId.size();
     valuesOut.resize(n,mdValueOut);
     for (size_t i = 0; i < n; ++i)
     {
         getValue(mdValueOut, objectsId[i]);
         valuesOut[i] = mdValueOut;
     }
 }

◆ getColumnValues() [2/3]

template<class T >

std::vector<T> MetaDataDb::getColumnValues ( const MDLabel label ) const

inline

Definition at line 247 of file metadata_db.h.

                                                             {
         return MetaData::getColumnValues<T>(label);
     }

◆ getColumnValues() [3/3]

template<class T >

void MetaDataDb::getColumnValues	(	const MDLabel	label,
		std::vector< T > &	valuesOut
	)		const

inline

Definition at line 252 of file metadata_db.h.

                                                                              {
         return MetaData::getColumnValues(label, valuesOut);
     }

◆ getColumnValuesOpt()

template<typename T >

bool MetaDataDb::getColumnValuesOpt	(	const MDLabel	label,
		std::vector< T > &	values
	)		const

Get all values of a column as a vector.

Definition at line 845 of file metadata_db.h.

                                                                                    {
     if (!containsLabel(label))
             return false;
     return sqlUtils::select(label,
             myMDSql->db,
             myMDSql->tableName(myMDSql->tableId),
             values);
 }

◆ getDatabase()

MDSql* MetaDataDb::getDatabase ( )

inline

Definition at line 182 of file metadata_db.h.

182 { return myMDSql; }

MetaDataDb::myMDSql

MDSql * myMDSql

Definition: metadata_db.h:62

◆ getMaxStringLength()

int MetaDataDb::getMaxStringLength ( const MDLabel thisLabel ) const

overridevirtual

Get maximum string length of column values.

Implements MetaData.

Definition at line 541 of file metadata_db.cpp.

 {
     if (!containsLabel(thisLabel))
         return -1;
 
     return myMDSql->columnMaxLength(thisLabel);
 }

◆ getObjectsForActiveLabels()

std::vector< MDObject > MetaDataDb::getObjectsForActiveLabels ( ) const

protected

Get a vector of (empty) objects for each active label

Definition at line 201 of file metadata_db.cpp.

                                                                 {
     // get active labels
     std::vector<MDObject> values;
     const auto &labels = this->_activeLabels;
     values.reserve(labels.size());
     for (auto &l : labels) {
         values.emplace_back(l);
     }
     return values;
 }

◆ getRow() [1/3]

std::unique_ptr< MDRow > MetaDataDb::getRow ( size_t id )

overridevirtual

Implements MetaData.

Definition at line 236 of file metadata_db.cpp.

                                                  {
     std::unique_ptr<MDRowSql> row(new MDRowSql());
     if (!getRow(*row, id))
         return nullptr;
     return std::move(row);
 }

◆ getRow() [2/3]

std::unique_ptr< const MDRow > MetaDataDb::getRow ( size_t id ) const

overridevirtual

Implements MetaData.

Definition at line 243 of file metadata_db.cpp.

                                                              {
     std::unique_ptr<MDRowSql> row(new MDRowSql());
     if (!getRow(*row, id))
         return nullptr;
     return std::move(row);
 }

◆ getRow() [3/3]

bool MetaDataDb::getRow	(	MDRowSql &	row,
		size_t	id
	)		const

Definition at line 264 of file metadata_db.cpp.

 {
     if (id == BAD_OBJID)
         REPORT_ERROR(ERR_MD_NOACTIVE, "getValue: please provide objId other than -1");
     // clear whatever is there now
     row.clear();
     // get active labels
     auto values = getObjectsForActiveLabels();
     // get values from the row
     if ( ! sqlUtils::select(id,
             myMDSql->db,
             myMDSql->tableName(myMDSql->tableId),
             values)) return false;
     // fill them
     for (auto &v : values)
         row.setValue(v);
     return true;
 }

◆ getRow2()

bool MetaDataDb::getRow2	(	MDRow &	row,
		size_t	id
	)		const

Definition at line 283 of file metadata_db.cpp.

 {
     bool success=true;
 
     // Clear row.
     row.clear();
 
     // Initialize SELECT.
     success = this->initGetRow( true);
     if (success)
     {
         bindValue( id);
 
         // Execute SELECT.
         success = execGetRow( row);
 
         // Finalize SELECT.
         finalizeGetRow();
     }
 
     return(success);
 }

◆ getRowSql() [1/2]

MDRowSql MetaDataDb::getRowSql ( size_t id )

Definition at line 250 of file metadata_db.cpp.

                                         {
     MDRowSql row;
     if (!getRow(row, id))
         throw ObjectDoesNotExist(id, getFilename());
     return row;
 }

◆ getRowSql() [2/2]

const MDRowSql MetaDataDb::getRowSql ( size_t id ) const

Definition at line 257 of file metadata_db.cpp.

                                                     {
     MDRowSql row;
     if (!getRow(row, id))
         throw ObjectDoesNotExist(id, getFilename());
     return row;
 }

◆ getRowValues()

bool MetaDataDb::getRowValues	(	size_t	id,
		std::vector< MDObject > &	values
	)		const

overridevirtual

Implements MetaData.

Definition at line 404 of file metadata_db.cpp.

                                                                           {
     for (auto &v : values) {
         if (!containsLabel(v.label))
                 return false;
     }
     if (id == BAD_OBJID)
         REPORT_ERROR(ERR_MD_NOACTIVE, "getValue: please provide objId other than -1");
     return sqlUtils::select(id,
             myMDSql->db,
             myMDSql->tableName(myMDSql->tableId),
             values);
 }

◆ getValue() [1/2]

bool MetaDataDb::getValue	(	MDObject &	mdValueOut,
		size_t	id
	)		const

overridevirtual

Implements MetaData.

Definition at line 109 of file metadata_db.cpp.

 {
     if (!containsLabel(mdValueOut.label))
         return false;
 
     if (id == BAD_OBJID)
         REPORT_ERROR(ERR_MD_NOACTIVE, "getValue: please provide objId other than -1");
 
     return myMDSql->getObjectValue(id, mdValueOut);
 }

◆ getValue() [2/2]

template<class T >

bool MetaDataDb::getValue	(	const MDLabel	label,
		T &	valueOut,
		size_t	id
	)		const

inline

Definition at line 238 of file metadata_db.h.

                                                                      {
         return MetaData::getValue(label, valueOut, id);
     }

◆ id_begin() [1/2]

id_iterator MetaDataDb::id_begin ( )

inlineoverridevirtual

Implements MetaData.

Definition at line 793 of file metadata_db.h.

                                     {
         return {memoryUtils::make_unique<MDDbIdIterator<false>>(*this)};
     }

◆ id_begin() [2/2]

id_const_iterator MetaDataDb::id_begin ( ) const

inlineoverridevirtual

Implements MetaData.

Definition at line 801 of file metadata_db.h.

                                                 {
         return {memoryUtils::make_unique<MDDbIdIterator<true>>(*this)};
     }

◆ id_end() [1/2]

id_iterator MetaDataDb::id_end ( )

inlineoverridevirtual

Implements MetaData.

Definition at line 797 of file metadata_db.h.

                                   {
         return {memoryUtils::make_unique<MDDbIdIterator<false>>(*this, true)};
     }

◆ id_end() [2/2]

id_const_iterator MetaDataDb::id_end ( ) const

inlineoverridevirtual

Implements MetaData.

Definition at line 805 of file metadata_db.h.

                                               {
         return {memoryUtils::make_unique<MDDbIdIterator<true>>(*this, true)};
     }

◆ importObject()

void MetaDataDb::importObject	(	const MetaData &	md,
		const size_t	id,
		bool	doClear = `true`
	)

overridevirtual

Import objects from another metadata.

//Import object 1000 from metadata B into metadata A
A.importObject(B, 1000);
//Import all objects with rotational angle greater that 60
A.importObjects(B, MDValuesGT(MDL_ANGLE_ROT, 60));
//Import all objects
A.importObjects(B);     *

Implements MetaData.

Definition at line 583 of file metadata_db.cpp.

 {
     // Currently supports importing only from MetaDataDb
     assert(dynamic_cast<const MetaDataDb*>(&md) != nullptr);
 
     const MetaDataDb& mdd = dynamic_cast<const MetaDataDb&>(md);
     MDValueEQ query(MDL_OBJID, id);
     mdd.myMDSql->copyObjects(this, &query);
 }

◆ importObjects() [1/2]

void MetaDataDb::importObjects	(	const MetaData &	md,
		const std::vector< size_t > &	objectsToAdd,
		bool	doClear = `true`
	)

overridevirtual

Implements MetaData.

Definition at line 593 of file metadata_db.cpp.

 {
     const std::vector<MDLabel>& labels = md.getActiveLabels();
     init(labels);
     copyInfo(md);
     int size = objectsToAdd.size();
     for (int i = 0; i < size; i++)
         importObject(md, objectsToAdd[i]);
 }

◆ importObjects() [2/2]

void MetaDataDb::importObjects	(	const MetaData &	md,
		const MDQuery &	query,
		bool	doClear = `true`
	)

overridevirtual

Implements MetaData.

Definition at line 603 of file metadata_db.cpp.

 {
     // Currently supports importing only from MetaDataDb
     assert(dynamic_cast<const MetaDataDb*>(&md) != nullptr);
 
     const MetaDataDb& mdd = dynamic_cast<const MetaDataDb&>(md);
     this->_importObjectsDb(mdd, query, doClear);
 }

◆ init()

void MetaDataDb::init ( const std::vector< MDLabel > & labelsVector )

protected

Init, do some initializations tasks, used in constructors

Definition at line 59 of file metadata_db.cpp.

 {
     _clear();
     _maxRows = 0; //by default read all rows
     _parsedLines = 0; //no parsed line;
     _activeLabels = labelsVector;
     //Create table in database
     myMDSql->createMd();
     _precision = 100;
     isMetadataFile = false;
 }//close init

◆ initAddRow()

bool MetaDataDb::initAddRow ( const MDRow & row )

Add a new Row and set values, return the objId of newly added object

Definition at line 331 of file metadata_db.cpp.

 {
     int     j=0;                    // Loop counter.
     bool    success=true;               // Return value.
     std::vector<MDLabel>    labels;     // Columns labels.
     std::vector<MDObject*>  mdValues;   // Vector to store values.
 
     // Set vector size.
     labels.resize(row.size());
 
     // Get labels.
     j=0;
     for (const MDObject* obj : row) {
         addLabel(obj->label);
         labels[j] = obj->label;
         j++;
     }
     labels.resize(j);
 
     // Prepare statement (mdValues is not used).
     if (!myMDSql->initializeInsert( &labels, mdValues))
     {
         std::cerr << "initAddRow: error executing myMDSql->initializeInsert" << std::endl;
         success = false;
     }
 
     return success;
 }

◆ initGetRow()

bool MetaDataDb::initGetRow ( bool addWhereClause ) const

Definition at line 168 of file metadata_db.cpp.

 {
     bool success=true;
 
     // Prepare statement.
     if (!myMDSql->initializeSelect( addWhereClause, this->_activeLabels))
     {
         success = false;
     }
 
     return success;
 }

◆ intersection()

void MetaDataDb::intersection	(	const MetaDataDb &	mdIn,
		const MDLabel	label
	)

Intersects two Metadatas. Result in "calling" Metadata

Definition at line 1374 of file metadata_db.cpp.

 {
     if(mdIn.isEmpty())
         clear();
     else
         _setOperates(mdIn, label, INTERSECTION);
 }

◆ join1() [1/2]

void MetaDataDb::join1	(	const MetaDataDb &	mdInLeft,
		const MetaDataDb &	mdInRight,
		const MDLabel	label,
		JoinType	type = `LEFT`
	)

Join two Metadatas Result in "calling" metadata

Definition at line 1403 of file metadata_db.cpp.

 {
     join2(mdInLeft, mdInRight, label, label, type);
 }

◆ join1() [2/2]

void MetaDataDb::join1	(	const MetaDataDb &	mdInLeft,
		const MetaDataDb &	mdInRight,
		const std::vector< MDLabel > &	labels,
		JoinType	type = `LEFT`
	)

Join two Metadatas Result in "calling" metadata

Definition at line 1418 of file metadata_db.cpp.

 {
     join2(mdInLeft, mdInRight, labels, labels, type);
 }

◆ join2() [1/2]

void MetaDataDb::join2	(	const MetaDataDb &	mdInLeft,
		const MetaDataDb &	mdInRight,
		const MDLabel	labelLeft,
		const MDLabel	labelRight,
		JoinType	type = `LEFT`
	)

Join two Metadatas Result in "calling" metadata. join may be done using different labels in each metadata

Definition at line 1408 of file metadata_db.cpp.

 {
     clear();
     std::vector<MDLabel> labelsLeft, labelsRight;
     labelsLeft.emplace_back(labelLeft);
     labelsRight.emplace_back(labelRight);
     _setOperates(mdInLeft, mdInRight, labelsLeft,labelsRight, (SetOperation)type);
 }

◆ join2() [2/2]

void MetaDataDb::join2	(	const MetaDataDb &	mdInLeft,
		const MetaDataDb &	mdInRight,
		const std::vector< MDLabel > &	labelsLeft,
		const std::vector< MDLabel > &	labelsRight,
		JoinType	type = `LEFT`
	)

Join two Metadatas Result in "calling" metadata. join may be done using different labels in each metadata

Definition at line 1423 of file metadata_db.cpp.

 {
     clear();
     _setOperates(mdInLeft, mdInRight, labelsLeft,labelsRight, (SetOperation)type);
 }

◆ joinNatural()

void MetaDataDb::joinNatural	(	const MetaDataDb &	mdInLeft,
		const MetaDataDb &	mdInRight
	)

Join two Metadatas using all common labels (NATURAL_JOIN)

Definition at line 1430 of file metadata_db.cpp.

 {
     join2(mdInLeft, mdInRight, MDL_UNDEFINED, MDL_UNDEFINED, NATURAL);
 }

◆ lastRowId()

size_t MetaDataDb::lastRowId ( ) const

overridevirtual

Goto last metadata object.

Implements MetaData.

Definition at line 703 of file metadata_db.cpp.

 {
     return myMDSql->lastRow();
 }

◆ makeAbsPath()

void MetaDataDb::makeAbsPath ( const MDLabel label = MDL_IMAGE )

Makes filenames with absolute paths

Definition at line 1620 of file metadata_db.cpp.

 {
 
     String aux_string;
     String aux_string_path;
     char buffer[1024];
 
     if (!getcwd(buffer, 1023))
         REPORT_ERROR(ERR_UNCLASSIFIED,"Cannot get the current directory");
     String path_str(buffer);
     path_str += "/";
     getValue(label, aux_string, firstRowId());
 
     if (aux_string[0] == '/')
         return;
 
     FileName auxFile;
     for (size_t id : this->ids())
     {
         aux_string_path = path_str;
         getValue(label, auxFile, id);
 
         if (auxFile.isInStack())
         {
             size_t id = auxFile.find('@',0);
             auxFile.insert(id+1,aux_string_path);
             setValue(label, auxFile, id);
         }
         else
         {
             auxFile.addPrefix(aux_string_path);
             setValue(label, auxFile, id);
         }
     }
 }

◆ merge()

void MetaDataDb::merge ( const MetaData & md2 )

Merge of two Metadata. This function reads another metadata and add all columns values. The size of the two Metadatas should be the same. If there are common columns, the values in md2 will be setted.

Definition at line 1120 of file metadata_db.cpp.

 {
     if (size() != md2.size())
         REPORT_ERROR(ERR_MD, "Size of two metadatas should coincide for merging.");
 
     for (const auto& row : md2)
         this->setRow(row, row.id());
 }

◆ metadataToVec()

void MetaDataDb::metadataToVec ( std::vector< MDRowSql > & vd )

Definition at line 1707 of file metadata_db.cpp.

 {
     for (const auto& row : *this)
         vd.emplace_back(dynamic_cast<const MDRowSql&>(row));
 }

◆ operate()

void MetaDataDb::operate ( const String & expression )

Basic operations on columns data. Mainly perform replacements on string values and basic algebraic operations on numerical ones.

Definition at line 1435 of file metadata_db.cpp.

 {
     if (!myMDSql->operate(expression))
         REPORT_ERROR(ERR_MD, "MetaDataDb::operate: error doing operation");
 }

◆ operator=()

MetaDataDb & MetaDataDb::operator= ( const MetaDataDb & md )

Assignment operator

Copies MetaDataDb from an existing MetaData object.

Definition at line 527 of file metadata_db.cpp.

 {
     copyMetadata(md);
     return *this;
 }

◆ operator==()

bool MetaDataDb::operator== ( const MetaDataDb & op ) const

'is equal to' (equality).

Definition at line 1721 of file metadata_db.cpp.

 {
     return myMDSql->equals(*(op.myMDSql));
 }

◆ randomize()

void MetaDataDb::randomize ( const MetaDataDb & MDin )

Randomize a metadata. MDin is input and the "randomized" result will be in the "calling" Metadata.

Definition at line 1450 of file metadata_db.cpp.

 {
     std::random_device rd;
     auto g = std::mt19937(rd());
     std::vector<size_t> objects;
     MDin.myMDSql->selectObjects(objects);
     std::shuffle(objects.begin(), objects.end(), g);
     importObjects(MDin, objects);
 }

◆ read()

void MetaDataDb::read	(	const FileName &	inFile,
		const std::vector< MDLabel > *	desiredLabels = `NULL`,
		bool	decomposeStack = `true`
	)

overridevirtual

Read data from file. Guess the blockname from the filename

inFilename="first@md1.doc" -> filename = md1.doc, blockname = first

Implements MetaData.

Definition at line 912 of file metadata_db.cpp.

 {
     String blockName;
     FileName inFile;
 
     blockName=_filename.getBlockName();
     //    if (blockName.empty())
     //        blockName = DEFAULT_BLOCK_NAME;
     inFile = _filename.removeBlockName();
     String extFile = _filename.getExtension();
     blockName=escapeForRegularExpressions(blockName);
 
     _clear();
     myMDSql->createMd();
     this->setColumnFormat(true);
 
     if (extFile=="xml")
         readXML(inFile, desiredLabels, blockName, decomposeStack);
     else if(extFile=="sqlite")
         readDB(inFile, desiredLabels, blockName, decomposeStack);
     else
         readStar(_filename, desiredLabels, blockName, decomposeStack);
 }

◆ readDB()

void MetaDataDb::readDB	(	const FileName &	inFile,
		const std::vector< MDLabel > *	desiredLabels = `NULL`,
		const String &	blockRegExp = `DEFAULT_BLOCK_NAME`,
		bool	decomposeStack = `true`
	)

Read metadata from sqlite file

Definition at line 1041 of file metadata_db.cpp.

 {
     myMDSql->copyTableFromFileDB(blockRegExp, filename, desiredLabels, _maxRows);
 }

◆ readPlain()

void MetaDataDb::readPlain	(	const FileName &	inFile,
		const String &	labelsString,
		const String &	separator = `" "`
	)

Try to read a metadata from plain text with some columns. Labels for each columns should be provided in an string separated by spaces. Return false if couldn't read

Definition at line 939 of file metadata_db.cpp.

 {
     constexpr size_t LINE_LENGTH = 1024;
     clear();
     std::vector<MDLabel> labels;
     MDL::str2LabelVector(labelsString, labels);
 
     char lineBuffer[LINE_LENGTH];
     String line;
     std::ifstream is(inFile.c_str(), std::ios_base::in);
     size_t lineCounter = 0;
     size_t columnsNumber = labels.size();
     size_t objId;
     StringVector parts;
 
     while (is.getline(lineBuffer, LINE_LENGTH))
     {
         ++lineCounter;
         line.assign(lineBuffer);
         trim(line);
         if (line[0]=='#') // This is an old Xmipp comment
             continue;
         if (!line.empty())
         {
             std::stringstream ss(line);
             objId = addObject();
             for (size_t i = 0; i < columnsNumber; ++i)
             {
                 MDObject obj(labels[i]);
                 _parseObject(ss, obj, objId);
                 setValue(obj, objId);
             }
         }
     }
 }

◆ readXML()

void MetaDataDb::readXML	(	const FileName &	inFile,
		const std::vector< MDLabel > *	desiredLabels = `NULL`,
		const String &	blockRegExp = `DEFAULT_BLOCK_NAME`,
		bool	decomposeStack = `true`
	)

Read metadata from xml file

Definition at line 1033 of file metadata_db.cpp.

 {
     REPORT_ERROR(ERR_NOT_IMPLEMENTED,"readXML not implemented yet");
 }

◆ removeDuplicates()

void MetaDataDb::removeDuplicates	(	MetaDataDb &	MDin,
		MDLabel	label = `MDL_UNDEFINED`
	)

Remove duplicate entries for attribute in label

Definition at line 1382 of file metadata_db.cpp.

 {
     if(MDin.isEmpty())
         return;
     _setOperates(MDin, label, REMOVE_DUPLICATE);
 }

◆ removeIndex() [1/2]

void MetaDataDb::removeIndex ( MDLabel label )

Definition at line 665 of file metadata_db.cpp.

 {
     std::vector<MDLabel> labels(1);
     labels[0]=label;
     removeIndex(labels);
 }

◆ removeIndex() [2/2]

void MetaDataDb::removeIndex ( const std::vector< MDLabel > & desiredLabels )

Definition at line 672 of file metadata_db.cpp.

 {
     myMDSql->indexModify(desiredLabels, false);
 }

◆ removeItemId()

void MetaDataDb::removeItemId ( )

Remove item id.

Definition at line 683 of file metadata_db.cpp.

 {
     removeLabel(MDL_ITEM_ID);
 }

◆ removeLabel()

bool MetaDataDb::removeLabel ( const MDLabel label )

overridevirtual

Remove a label from the metadata. The data is still in the table. If you want to remove the data, make a copy of the MetaData.

Implements MetaData.

Definition at line 566 of file metadata_db.cpp.

 {
     std::vector<MDLabel>::iterator location;
     location = std::find(this->_activeLabels.begin(), this->_activeLabels.end(), label);
 
     if (location == this->_activeLabels.end())
         return false;
 
     this->_activeLabels.erase(location);
     return true;
 }

◆ removeObject()

bool MetaDataDb::removeObject ( size_t id )

overridevirtual

Remove the object with this id. Returns true if the object was removed or false if the object did not exist

Implements MetaData.

Definition at line 629 of file metadata_db.cpp.

                                        {
     int removed = removeObjects(MDValueEQ(MDL_OBJID, id));
     return (removed > 0);
 }

◆ removeObjects() [1/3]

void MetaDataDb::removeObjects ( const std::vector< size_t > & toRemove )

overridevirtual

Removes the collection of objects of given vector id's NOTE: The iterator will point to the first object after any of these operations

Implements MetaData.

Definition at line 634 of file metadata_db.cpp.

 {
     int size = toRemove.size();
     for (int i = 0; i < size; i++)
         removeObject(toRemove[i]);
 }

◆ removeObjects() [2/3]

int MetaDataDb::removeObjects ( const MDQuery & query )

overridevirtual

Removes objects from metadata. return the number of deleted rows if not query, all objectes are removed Queries can be used in the same way as in the importObjects function

Implements MetaData.

Definition at line 641 of file metadata_db.cpp.

 {
     int removed = myMDSql->deleteObjects(&query);
     return removed;
 }

◆ removeObjects() [3/3]

int MetaDataDb::removeObjects ( )

overridevirtual

Implements MetaData.

Definition at line 647 of file metadata_db.cpp.

 {
     int removed = myMDSql->deleteObjects();
     return removed;
 }

◆ renameColumn() [1/2]

void MetaDataDb::renameColumn	(	MDLabel	oldLabel,
		MDLabel	newLabel
	)

overridevirtual

Rename column.

Implements MetaData.

Definition at line 1198 of file metadata_db.cpp.

 {
     if (!containsLabel(oldLabel))
         REPORT_ERROR(ERR_ARG_MISSING, formatString("Source label: '%s' doesn't exist on metadata",
                      (MDL::label2Str(oldLabel)).c_str()));
     std::vector<MDLabel> vOldLabel(1);
     vOldLabel[0]=oldLabel;
     std::vector<MDLabel> vNewLabel(1);
     vNewLabel[0]=newLabel;
     renameColumn(vOldLabel,vNewLabel);
 }

◆ renameColumn() [2/2]

void MetaDataDb::renameColumn	(	const std::vector< MDLabel > &	oldLabel,
		const std::vector< MDLabel > &	newLabel
	)

overridevirtual

Rename several columns. This is an expensive operations so if several columns need to be changed do it using this function instead one by one

Implements MetaData.

Definition at line 1210 of file metadata_db.cpp.

 {
     myMDSql->renameColumn(vOldLabel,vNewLabel);
 }

◆ replace()

void MetaDataDb::replace	(	const MDLabel	label,
		const String &	oldStr,
		const String &	newStr
	)

Replace an string in some column(label). The type of the column should be string. This function is a shortcut of the more genereal function operate

Definition at line 1441 of file metadata_db.cpp.

 {
     String labelStr = MDL::label2Str(label);
     String expression = formatString("%s=replace(%s,'%s', '%s')",
                                      labelStr.c_str(), labelStr.c_str(), oldStr.c_str(), newStr.c_str());
     if (!myMDSql->operate(expression))
         REPORT_ERROR(ERR_MD, "MetaDataDb::replace: error doing operation");
 }

◆ selectPart()

void MetaDataDb::selectPart	(	const MetaData &	mdIn,
		size_t	startPosition,
		size_t	numberOfObjects,
		const MDLabel	sortLabel = `MDL_OBJID`
	)

overridevirtual

Select some part from Metadata. Select elements from input Metadata at some starting position if the numberOfObjects is -1, all objects will be returned from startPosition to the end.

Implements MetaData.

Definition at line 1601 of file metadata_db.cpp.

 {
     if (dynamic_cast<const MetaDataDb*>(&mdIn) != nullptr)
         return _selectPart(dynamic_cast<const MetaDataDb&>(mdIn), startPosition, numberOfObjects, sortLabel);
     throw std::logic_error("Not yet implemented"); // TODO: use universal functions just on MetaData
 }

◆ selectRandomSubset()

void MetaDataDb::selectRandomSubset	(	const MetaData &	mdIn,
		size_t	numberOfObjects,
		const MDLabel	sortLabel = `MDL_OBJID`
	)

overridevirtual

Select random subset

Implements MetaData.

Definition at line 1584 of file metadata_db.cpp.

 {
     if (dynamic_cast<const MetaDataDb*>(&mdIn) != nullptr)
         return _selectRandomSubset(dynamic_cast<const MetaDataDb&>(mdIn), numberOfObjects, sortLabel);
     throw std::logic_error("Not yet implemented"); // TODO: use universal functions just on MetaData
 }

◆ selectSplitPart()

void MetaDataDb::selectSplitPart	(	const MetaData &	mdIn,
		size_t	n,
		size_t	part,
		const MDLabel	sortLabel = `MDL_OBJID`
	)

Take a part from MetaData. This function is equivallent to divide the input MetaData in n parts and take one. The result will be in "calling" MetaData.

Definition at line 1566 of file metadata_db.cpp.

 {
     if (dynamic_cast<const MetaDataDb*>(&mdIn) != nullptr)
         return _selectSplitPart(dynamic_cast<const MetaDataDb&>(mdIn), n, part, sortLabel);
     throw std::logic_error("Not yet implemented"); // TODO: use universal functions just on MetaData
 }

◆ setColumnValues() [1/2]

void MetaDataDb::setColumnValues ( const std::vector< MDObject > & valuesIn )

overridevirtual

Get all values of a column as a vector.

Implements MetaData.

Definition at line 134 of file metadata_db.cpp.

 {
     bool addObjects=false;
     if (size()==0)
         addObjects=true;
     if (valuesIn.size()!=size() && !addObjects)
         REPORT_ERROR(ERR_MD_OBJECTNUMBER,"Input vector must be of the same size as the metadata");
     if (!addObjects)
     {
         size_t n = 0;
         for (size_t objId : this->ids())
             setValue(valuesIn[n++], objId);
     }
     else
     {
         size_t nmax=valuesIn.size();
         for (size_t n=0; n<nmax; ++n)
             setValue(valuesIn[n],addObject());
     }
 }

◆ setColumnValues() [2/2]

template<class T >

void MetaDataDb::setColumnValues	(	const MDLabel	label,
		const std::vector< T > &	valuesIn
	)

inline

Definition at line 266 of file metadata_db.h.

                                                                             {
         return MetaData::setColumnValues(label, valuesIn);
     }

◆ setRow()

bool MetaDataDb::setRow	(	const MDRow &	row,
		size_t	id
	)

Copy all the values in the input row in the current metadata

Definition at line 306 of file metadata_db.cpp.

 {
     if (row.empty()) {
         return true;
     }
     addMissingLabels(row);
 
     // create mask of valid labels
     std::vector<MDLabel> labels;
     labels.reserve(row.size());
     for (const MDObject* obj : row) {
         labels.emplace_back(obj->label);
     }
     // extract values to be added
     std::vector<const MDObject*> vals;
     vals.reserve(row.size());
     for (const auto &l : labels) {
         vals.emplace_back(row.getObject(l));
     }
     // update values to db
     return sqlUtils::update(vals, MDSql::db,
                     myMDSql->tableName(myMDSql->tableId), id);
 }

◆ setValue() [1/2]

bool MetaDataDb::setValue	(	const MDObject &	mdValueIn,
		size_t	id
	)

overridevirtual

This functions are using MDObject for set real values there is an explicit function signature foreach type supported in Metadata. This is done for some type checking of Metadata labels and values

Implements MetaData.

Definition at line 90 of file metadata_db.cpp.

 {
     if (id == BAD_OBJID)
     {
         REPORT_ERROR(ERR_MD_NOACTIVE, "setValue: please provide objId other than -1");
         exit(1);
     }
     //add label if not exists, this is checked in addlabel
     addLabel(mdValueIn.label);
     return myMDSql->setObjectValue(id, mdValueIn);
 }

◆ setValue() [2/2]

template<class T >

bool MetaDataDb::setValue	(	const MDLabel	label,
		const T &	valueIn,
		size_t	id
	)

inline

Definition at line 230 of file metadata_db.h.

                                                                     {
         return MetaData::setValue(label, valueIn, id);
     }

◆ setValueCol() [1/2]

bool MetaDataDb::setValueCol ( const MDObject & mdValueIn )

overridevirtual

Set the value of all objects in an specified column (both value and column are specified in mdValueIn)

Implements MetaData.

Definition at line 102 of file metadata_db.cpp.

 {
     //add label if not exists, this is checked in addlabel
     addLabel(mdValueIn.label);
     return myMDSql->setObjectValue(mdValueIn);
 }

◆ setValueCol() [2/2]

template<class T >

bool MetaDataDb::setValueCol	(	const MDLabel	label,
		const T &	valueIn
	)

inline

Definition at line 215 of file metadata_db.h.

                                                             {
         return MetaData::setValueCol(label, valueIn);
     }

◆ size()

size_t MetaDataDb::size ( ) const

overridevirtual

Number of objects contained in the metadata.

Implements MetaData.

Definition at line 549 of file metadata_db.cpp.

 {
     return myMDSql->size();
 }

◆ sort() [1/2]

void MetaDataDb::sort	(	MetaDataDb &	MDin,
		const MDLabel	sortLabel,
		bool	asc = `true`,
		int	limit = `-1`,
		int	offset = `0`
	)

Definition at line 1460 of file metadata_db.cpp.

 {
     if (MDin.containsLabel(sortLabel))
     {
         init(MDin._activeLabels);
         copyInfo(MDin);
         //if you sort just once the index will not help much
         addIndex(sortLabel);
         MDQuery query(limit, offset, sortLabel,asc);
         MDin.myMDSql->copyObjects(this, &query);
     }
     else
         *this=MDin;
 }

◆ sort() [2/2]

void MetaDataDb::sort	(	MetaDataDb &	MDin,
		const String &	sortLabel,
		bool	asc = `true`,
		int	limit = `-1`,
		int	offset = `0`
	)

Definition at line 1475 of file metadata_db.cpp.

 {
     // Check if the label has semicolon
     size_t ipos=sortLabel.find(':');
     MDLabelType type = MDL::labelType(sortLabel);
     if (ipos!=String::npos || type == LABEL_VECTOR_DOUBLE || type == LABEL_VECTOR_SIZET)
     {
         if(limit != -1 || offset != 0)
             REPORT_ERROR(ERR_ARG_INCORRECT,"Limit and Offset are not implemented for vector sorting.");
 
         MDLabel label;
         size_t column;
         if (ipos!=String::npos)
         {
             // Check that the label is a vector field
             std::vector< String > results;
             splitString(sortLabel,":",results);
             column=textToInteger(results[1]);
             MDLabelType type = MDL::labelType(results[0]);
             if (type != LABEL_VECTOR_DOUBLE || type != LABEL_VECTOR_SIZET)
                 REPORT_ERROR(ERR_ARG_INCORRECT,"Column specifications cannot be used with non-vector labels");
             label = MDL::str2Label(results[0]);
         }
         else
         {
             label = MDL::str2Label(sortLabel);
             column = 0;
         }
 
         // Get the column values
         MultidimArray<double> v;
         v.resizeNoCopy(MDin.size());
         std::vector<double> vectorValues;
         int i = 0;
         for (size_t id : MDin.ids())
         {
             MDin.getValue(label, vectorValues, id);
             if (column >= vectorValues.size())
                 REPORT_ERROR(ERR_MULTIDIM_SIZE,"Trying to access to inexistent column in vector");
             DIRECT_A1D_ELEM(v, i) = vectorValues[column];
             i++;
         }
 
         // Sort
         MultidimArray<int> idx;
         v.indexSort(idx);
 
         // Construct output Metadata
         init(MDin._activeLabels);
         copyInfo(MDin);
         size_t id;
         FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY1D(idx)
         {
             MDRowSql row;
             MDin.getRow(row,DIRECT_A1D_ELEM(idx,i));
             id = addObject();
             setRow(row, id);
         }
     }
     else
     {
         sort(MDin, MDL::str2Label(sortLabel),asc, limit, offset);
     }
 }

◆ split()

void MetaDataDb::split	(	size_t	n,
		std::vector< MetaDataDb > &	results,
		const MDLabel	sortLabel = `MDL_OBJID`
	)

Split Metadata in several Metadatas. The Metadata will be divided in 'n' almost equally parts and the result will be a vector of Metadatas. The "calling" Metadata will no be modified.

// Divide the images metadata in 10 metadatas.
std::vector<MetaData> imagesGroups;
imageMD.split(10, imagesGroups);

Definition at line 1540 of file metadata_db.cpp.

 {
     size_t mdSize = size();
     if (n > mdSize)
         REPORT_ERROR(ERR_MD, "MetaDataDb::split: Couldn't split a metadata in more parts than its size");
 
     results.clear();
     results.resize(n);
     for (size_t i = 0; i < n; i++)
     {
         MetaDataDb &md = results.at(i);
         md._selectSplitPart(*this, n, i, mdSize, sortLabel);
     }
 }

◆ subtraction()

void MetaDataDb::subtraction	(	const MetaDataDb &	mdIn,
		const MDLabel	label
	)

Subtract two Metadatas. Result in "calling" metadata

Definition at line 1396 of file metadata_db.cpp.

 {
     if(mdIn.isEmpty())
         return;
     _setOperates(mdIn, label, SUBSTRACTION);
 }

◆ unionAll()

void MetaDataDb::unionAll ( const MetaDataDb & mdIn )

Union of all elements in two Metadata, duplicating common elements. Result in calling metadata object Repetition are allowed

Definition at line 1366 of file metadata_db.cpp.

 {
     if(mdIn.isEmpty())
         return;
     _setOperates(mdIn, MDL_UNDEFINED, UNION);//label not needed for unionAll operation
 }

◆ unionDistinct()

void MetaDataDb::unionDistinct	(	const MetaDataDb &	mdIn,
		const MDLabel	label = `MDL_OBJID`
	)

Union of elements in two Metadatas, without duplicating. Result in calling metadata object union is a reserved word so I called this method unionDistinct

Definition at line 1359 of file metadata_db.cpp.

 {
     if(mdIn.isEmpty())
         return;
     _setOperates(mdIn, label, UNION_DISTINCT);
 }

◆ vecToMetadata()

void MetaDataDb::vecToMetadata ( const std::vector< MDRow > & rowMetadata )

Definition at line 1713 of file metadata_db.cpp.

 {
     const MDRowSql row;
 
     for (size_t i=0;i<rowMetadata.size();i++)
         this->addRow(rowMetadata[i]);
 }

◆ write() [1/2]

void MetaDataDb::write	(	const FileName &	outFile,
		WriteModeMetaData	mode = `MD_OVERWRITE`
	)		const

overridevirtual

Write metadata to disk. Guess blockname from filename

outFilename="first@md1.doc" -> filename = md1.doc, blockname = first

Implements MetaData.

Definition at line 748 of file metadata_db.cpp.

 {
     String blockName;
     FileName outFile;
     FileName extFile;
 
     blockName=_outFile.getBlockName();
     if (blockName.empty())
         blockName = DEFAULT_BLOCK_NAME;
     outFile = _outFile.removeBlockName();
     extFile = _outFile.getExtension();
 
     if (extFile=="xml")
     {
         writeXML(outFile, blockName, mode);
     }
     else if(extFile=="sqlite")
     {
         writeDB(outFile, blockName, mode);
     }
     else
     {
         writeStar(outFile, blockName, mode);
     }
 }

◆ write() [2/2]

void MetaDataDb::write	(	std::ostream &	os,
		const String &	blockName = `""`,
		WriteModeMetaData	mode = `MD_OVERWRITE`
	)		const

overridevirtual

Implements MetaData.

Definition at line 795 of file metadata_db.cpp.

 {
     if(mode==MD_OVERWRITE)
         os << FileNameVersion << " * "// << (isColumnFormat ? "column" : "row")
         << '\n' //write which type of format (column or row) and the path;
         << WordWrap(this->_comment, line_max);     //write md comment in the 2nd comment line of header
     //write data block
     String _szBlockName("data_");
     _szBlockName += blockName;
 
     if (this->isColumnFormat())
     {
         //write md columns in 3rd comment line of the header
         os << _szBlockName << '\n';
         os << "loop_" << '\n';
         auto sortedLabels = this->_activeLabels;
         std::sort(sortedLabels.begin(), sortedLabels.end());
         for (size_t i = 0; i < sortedLabels.size(); i++)
         {
             const auto &label = sortedLabels.at(i);
             if (label != MDL_STAR_COMMENT)
             {
                 os << " _" << MDL::label2Str(label) << '\n';
             }
         }
         _writeRows(os);
 
         //Put the activeObject to the first, if exists
     }
     else //rowFormat
     {
         os << _szBlockName << '\n';
 
         // Get first object. In this case (row format) there is a single object
         size_t id = firstRowId();
 
         if (id != BAD_OBJID)
         {
             auto sortedLabels = this->_activeLabels;
             std::sort(sortedLabels.begin(), sortedLabels.end());
             for (size_t i = 0; i < sortedLabels.size(); i++)
             {
                 const auto &label = sortedLabels.at(i);
                 if (label != MDL_STAR_COMMENT)
                 {
                     MDObject mdValue(label);
                     os << " _" << MDL::label2Str(label) << " ";
                     myMDSql->getObjectValue(id, mdValue);
                     mdValue.toStream(os);
                     os << '\n';
                 }
             }
         }
 
     }
 }//write

◆ writeDB()

void MetaDataDb::writeDB	(	const FileName	fn,
		const FileName	blockname,
		WriteModeMetaData	mode
	)		const

Write metadata in sqlite3 file.

Definition at line 1656 of file metadata_db.cpp.

 {
     if(mode==MD_OVERWRITE)
         unlink(fn.c_str());
     myMDSql->copyTableToFileDB(blockname,fn);
 }

◆ writeText()

void MetaDataDb::writeText	(	const FileName	fn,
		const std::vector< MDLabel > *	desiredLabels
	)		const

overridevirtual

Write metadata in text file as plain data without header.

Implements MetaData.

Definition at line 1692 of file metadata_db.cpp.

 {
     std::ofstream ofs(fn.c_str(), std::ios_base::trunc|std::ios_base::out);
 
     if (desiredLabels != NULL)
     {
         MetaDataDb mdAux(*this);
         mdAux._activeLabels = *desiredLabels;
         mdAux._writeRows(ofs);
     }
     else
         _writeRows(ofs);
     ofs.close();
 }

◆ writeXML()

void MetaDataDb::writeXML	(	const FileName	fn,
		const FileName	blockname,
		WriteModeMetaData	mode
	)		const

overridevirtual

Export medatada to xml file.

Implements MetaData.

Definition at line 1663 of file metadata_db.cpp.

 {
     //fixme
     if(mode!=MD_OVERWRITE)
         REPORT_ERROR(ERR_NOT_IMPLEMENTED,"XML is only implemented for overwrite mode");
     std::ofstream ofs(fn.c_str(), std::ios_base::out|std::ios_base::trunc);
     size_t size = this->_activeLabels.size();
     ofs <<  "<" << blockname << ">"<< '\n';
     for (size_t id : this->ids())
     {
         ofs <<  "<ROW ";
         for (size_t i = 0; i < size; i++)
         {
             if (this->_activeLabels[i] != MDL_STAR_COMMENT)
             {
                 ofs << MDL::label2Str(this->_activeLabels[i]) << "=\"";
                 MDObject mdValue(this->_activeLabels[i]);
                 //ofs.width(1);
                 myMDSql->getObjectValue(id, mdValue);
                 mdValue.toStream(ofs, true);
                 ofs << "\" ";
             }
         }
         ofs <<  " />" << '\n';
     }
     ofs <<  "</" << blockname << ">"<< '\n';
 }

Friends And Related Function Documentation

◆ MDSql

friend class MDSql

friend

This variables should only be used by MDSql for handling db status of metadataThe table id to do db operations

Definition at line 61 of file metadata_db.h.

Member Data Documentation

◆ _activeLabels

std::vector<MDLabel> MetaDataDb::_activeLabels

protected

What labels have been read from a docfile/metadata file and/or will be stored on a new metadata file when "save" is called

Definition at line 68 of file metadata_db.h.

◆ myMDSql

MDSql* MetaDataDb::myMDSql

protected

Definition at line 62 of file metadata_db.h.

The documentation for this class was generated from the following files:

xmippCore/core/metadata_db.h
xmippCore/core/metadata_db.cpp

Classes

Public Member Functions

Protected Member Functions

Protected Attributes

Friends

Additional Inherited Members

Detailed Description

Notes

Constructor & Destructor Documentation

◆ MetaDataDb() [1/5]

◆ MetaDataDb() [2/5]

◆ MetaDataDb() [3/5]

◆ MetaDataDb() [4/5]

◆ MetaDataDb() [5/5]

◆ ~MetaDataDb()

Member Function Documentation

◆ _clear()

◆ _importObjectsDb()

◆ _importObjectsGeneral()

◆ _parseObjects()

◆ _readRowFormat()

◆ _readRowsStar()

◆ _selectPart()

◆ _selectRandomSubset()

◆ _selectSplitPart() [1/2]

◆ _selectSplitPart() [2/2]

◆ _setOperates() [1/3]

◆ _setOperates() [2/3]

◆ _setOperates() [3/3]

◆ _setOperatesLabel()

◆ _writeRows()

◆ addIndex() [1/2]

◆ addIndex() [2/2]

◆ addItemId()

◆ addLabel()

◆ addMissingLabels()

◆ addObject()

◆ addPlain()

◆ addRow()

◆ addRow2()

◆ addRowOpt()

◆ addRows()

◆ aggregate() [1/2]

◆ aggregate() [2/2]

◆ aggregateGroupBy()

◆ aggregateSingle()

◆ aggregateSingleInt()

◆ aggregateSingleSizeT()

◆ begin() [1/2]

◆ begin() [2/2]

◆ bindValue()

◆ clear()

◆ containsLabel()

◆ containsObject() [1/2]

◆ containsObject() [2/2]

◆ copyColumn()

◆ copyColumnTo()

◆ copyMetadata()

◆ countObjects()

◆ distinct()

◆ end() [1/2]

◆ end() [2/2]

◆ execAddRow()

◆ execGetRow()

◆ existsBlock()

◆ fillConstant()

◆ fillExpand()

◆ fillLinear()

◆ fillRandom()

◆ finalizeAddRow()

◆ finalizeGetRow()

◆ findObjects() [1/2]

◆ findObjects() [2/2]

◆ firstObject()

◆ firstRowId()

◆ getActiveLabels()

◆ getAllRows()

◆ getColumnMax()

◆ getColumnMin()

◆ getColumnValues() [1/3]