metadata_db.h

Go to the documentation of this file.

/***************************************************************************
 *
 * Authors:     J.M. de la Rosa Trevin (jmdelarosa@cnb.csic.es)
 *              Jan Horacek (xhorace4@fi.muni.cz)
 *
 * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
 * 02111-1307  USA
 *
 *  All comments concerning this program package may be sent to the
 *  e-mail address 'xmipp@cnb.csic.es'
 ***************************************************************************/

#ifndef CORE_METADATA_DB_H
#define CORE_METADATA_DB_H

#include <regex.h>
#include <cmath>
#include "utils/memory_utils.h"
#include "metadata_base.h"
#include "metadata_label.h"
#include "metadata_object.h"
#include "metadata_row_base.h"
#include "metadata_static.h"
#include "metadata_sql.h"
#include "metadata_sql_operations.h"
#include "utils/sql_utils.h"
#include "xmipp_error.h"
#include "xmipp_filename.h"
#include "metadata_writemode.h"


class MetaDataDb : public MetaData {
protected:
    friend class MDSql;
    MDSql * myMDSql;

    std::vector<MDLabel> _activeLabels;

    void init(const std::vector<MDLabel> &labelsVector);

    void copyMetadata(const MetaDataDb &md, bool copyObjects = true);

    void _selectSplitPart(const MetaDataDb &mdIn,
                          int n, int part, size_t mdSize,
                          const MDLabel sortLabel);

    void _selectSplitPart(const MetaDataDb &mdIn,
                          size_t n, size_t part,
                          const MDLabel sortLabel=MDL_OBJID);

    void _selectRandomSubset(const MetaDataDb &mdIn, size_t numberOfObjects, const MDLabel sortLabel=MDL_OBJID);

    void _selectPart(const MetaDataDb &mdIn, size_t startPosition, size_t numberOfObjects,
                     const MDLabel sortLabel=MDL_OBJID);

    void _setOperates(const MetaDataDb &mdIn, const MDLabel label, SetOperation operation);
    void _setOperates(const MetaDataDb &mdIn, const std::vector<MDLabel> &labels, SetOperation operation);
    void _setOperates(const MetaDataDb &mdInLeft,
                      const MetaDataDb &mdInRight,
                      const std::vector<MDLabel> &labelsLeft,
                      const std::vector<MDLabel> &labelsRight,
                      SetOperation operation);
    void _setOperatesLabel(const MetaDataDb &mdIn, const MDLabel label, SetOperation operation);
    void _clear(bool onlyData=false);

    void _readRowsStar(mdBlock &block, std::vector<MDObject*> & columnValues,
                       const std::vector<MDLabel> *desiredLabels) override;

    void _readRowFormat(std::istream& is);

    void _parseObjects(std::istream &is, std::vector<MDObject*> & columnValues,
                       const std::vector<MDLabel> *desiredLabels, bool firstTime) override;

    std::vector<MDObject> getObjectsForActiveLabels() const;

    void _importObjectsDb(const MetaDataDb &md, const MDQuery &query, bool doClear=true);
    void _importObjectsGeneral(const MetaData &md, const MDQuery &query, bool doClear=true);

public:
    MetaDataDb();
    MetaDataDb(const std::vector<MDLabel> &labelsVector);
    MetaDataDb(const MetaData &md);

    MetaDataDb(const FileName &fileName, const std::vector<MDLabel> &desiredLabels = {});

    MetaDataDb(const MetaDataDb &md);

    MetaDataDb& operator=(const MetaDataDb &md);

    virtual ~MetaDataDb();

    void clear() override;
    MDSql * getDatabase() { return myMDSql; }

    void writeXML(const FileName fn, const FileName blockname, WriteModeMetaData mode) const override;

    void writeDB(const FileName fn, const FileName blockname, WriteModeMetaData mode) const;

    void writeText(const FileName fn,  const std::vector<MDLabel>* desiredLabels) const override;

    int getMaxStringLength( const MDLabel thisLabel) const override;

    bool setValueCol(const MDObject &mdValueIn) override;

    template<class T>
    bool setValueCol(const MDLabel label, const T &valueIn) {
        return MetaData::setValueCol(label, valueIn);
    }

    //private:
    bool setValue(const MDObject &mdValueIn, size_t id) override;


    template<class T>
    bool setValue(const MDLabel label, const T &valueIn, size_t id) {
        return MetaData::setValue(label, valueIn, id);
    }

    bool getValue(MDObject &mdValueOut, size_t id) const override;
    bool getRowValues(size_t id, std::vector<MDObject> &values) const override;

    template<class T>
    bool getValue(const MDLabel label, T &valueOut, size_t id) const {
        return MetaData::getValue(label, valueOut, id);
    }

    void getColumnValues(const MDLabel label, std::vector<MDObject> &valuesOut) const override;

    template<class T>
    std::vector<T> getColumnValues(const MDLabel label) const {
        return MetaData::getColumnValues<T>(label);
    }

    template<class T>
    void getColumnValues(const MDLabel label, std::vector<T> &valuesOut) const {
        return MetaData::getColumnValues(label, valuesOut);
    }

    template<typename T>
    bool getColumnValuesOpt(const MDLabel label, std::vector<T> &values) const;

    void setColumnValues(const std::vector<MDObject> &valuesIn) override;

    template<class T>
    void setColumnValues(const MDLabel label, const std::vector<T> &valuesIn) {
        return MetaData::setColumnValues(label, valuesIn);
    }

    bool bindValue(size_t id) const;

    bool initGetRow(bool addWhereClause) const;
    bool execGetRow(MDRow &row) const;
    void finalizeGetRow(void) const;

    std::unique_ptr<MDRow> getRow(size_t id) override;
    std::unique_ptr<const MDRow> getRow(size_t id) const override;

    MDRowSql getRowSql(size_t id);
    const MDRowSql getRowSql(size_t id) const;

    bool getRow(MDRowSql &row, size_t id) const; // FIXME: deprecated, use getRow above

    bool getAllRows(std::vector<MDRowSql> &rows) const;
    bool getRow2(MDRow &row, size_t id) const;

    bool setRow(const MDRow &row, size_t id);

    bool initAddRow(const MDRow &row);
    bool execAddRow(const MDRow &row);
    void finalizeAddRow(void);
    size_t addRow(const MDRow &row) override;
    void addRowOpt(const MDRowSql &row);
    void addRows(const std::vector<MDRowSql> &rows);
    void addMissingLabels(const MDRow &row);
    size_t addRow2(const MDRow &row);

    size_t size() const override;

    bool containsLabel(const MDLabel label) const override {
        return vectorContainsLabel(this->_activeLabels, label);
    }

    std::vector<MDLabel> getActiveLabels() const override {
        return this->_activeLabels;
    }

    bool addLabel(const MDLabel label, int pos = -1) override;

    bool removeLabel(const MDLabel label) override;

    size_t addObject() override;

    void importObject(const MetaData &md, const size_t id, bool doClear=true) override;
    void importObjects(const MetaData &md, const std::vector<size_t> &objectsToAdd, bool doClear=true) override;
    void importObjects(const MetaData &md, const MDQuery &query, bool doClear=true) override;

    bool removeObject(size_t id) override;

    void removeObjects(const std::vector<size_t> &toRemove) override;

    int removeObjects(const MDQuery &query) override;
    int removeObjects() override;

    void addIndex(MDLabel label) const;
    void addIndex(const std::vector<MDLabel> &desiredLabels) const;
    void removeIndex(MDLabel label);
    void removeIndex(const std::vector<MDLabel> &desiredLabels);

    void addItemId();

    void removeItemId();

    size_t firstRowId() const override;
    size_t firstObject(const MDQuery&) const override;
    size_t lastRowId() const override;

    void findObjects(std::vector<size_t> &objectsOut, const MDQuery &query) const override;
    void findObjects(std::vector<size_t> &objectsOut, int limit = -1) const override;

    size_t countObjects(const MDQuery &query) const override;
    bool containsObject(size_t objectId) const override;
    bool containsObject(const MDQuery &query) const override;

    void _writeRows(std::ostream &os) const override;

    void write(const FileName &outFile, WriteModeMetaData mode=MD_OVERWRITE) const override;
    void write(std::ostream &os, const String & blockName="",WriteModeMetaData mode=MD_OVERWRITE) const override;

    bool existsBlock(const FileName &_inFile);

    void readXML(const FileName &inFile,
                 const std::vector<MDLabel> *desiredLabels= NULL,
                 const String & blockRegExp=DEFAULT_BLOCK_NAME,
                 bool decomposeStack=true);

    void readDB(const FileName &inFile,
                const std::vector<MDLabel> *desiredLabels= NULL,
                const String & blockRegExp=DEFAULT_BLOCK_NAME,
                bool decomposeStack=true);

    void read(const FileName &inFile, const std::vector<MDLabel> *desiredLabels = NULL, bool decomposeStack=true) override;
    void readPlain(const FileName &inFile, const String &labelsString, const String &separator = " ");
    void addPlain(const FileName &inFile, const String &labelsString, const String &separator=" ");

    void aggregate(const MetaDataDb &mdIn, AggregateOperation op,
                   MDLabel aggregateLabel, MDLabel operateLabel, MDLabel resultLabel);
    void aggregate(const MetaDataDb &mdIn, const std::vector<AggregateOperation> &ops,
                   const std::vector<MDLabel> &operateLabels, const std::vector<MDLabel> &resultLabels);
    void aggregateGroupBy(const MetaDataDb &mdIn,
                          AggregateOperation op,
                          const std::vector<MDLabel> &groupByLabels,
                          MDLabel operateLabel,
                          MDLabel resultLabel);
    void aggregateSingle(MDObject &mdValueOut, AggregateOperation op,
                         MDLabel aggregateLabel);
    void aggregateSingleInt(MDObject &mdValueOut, AggregateOperation op,
                            MDLabel aggregateLabel);
    void aggregateSingleSizeT(MDObject &mdValueOut, AggregateOperation op,
                              MDLabel aggregateLabel);



    double getColumnMax(MDLabel column);

    double getColumnMin(MDLabel column);


    void unionDistinct(const MetaDataDb &mdIn, const MDLabel label=MDL_OBJID);

    void unionAll(const MetaDataDb &mdIn);

    void merge(const MetaData &md2);

    void intersection(const MetaDataDb &mdIn, const MDLabel label);

    void subtraction(const MetaDataDb &mdIn, const MDLabel label);

    void distinct(MetaDataDb &MDin, MDLabel label);

    void join1(const MetaDataDb &mdInLeft, const MetaDataDb &mdInRight, const MDLabel label, JoinType type=LEFT);

    void join2(const MetaDataDb &mdInLeft, const MetaDataDb &mdInRight, const MDLabel labelLeft, const MDLabel labelRight , JoinType type=LEFT);

    void join1(const MetaDataDb &mdInLeft, const MetaDataDb &mdInRight, const std::vector<MDLabel> &labels, JoinType type=LEFT);

    void join2(const MetaDataDb &mdInLeft, const MetaDataDb &mdInRight, const std::vector<MDLabel> &labelsLeft, const std::vector<MDLabel> &labelsRight,
            JoinType type=LEFT);

    void joinNatural(const MetaDataDb &mdInLeft, const MetaDataDb &mdInRight);

    void operate(const String &expression);

    void replace(const MDLabel label, const String &oldStr, const String &newStr);

    void randomize(const MetaDataDb &MDin);

    void removeDuplicates(MetaDataDb &MDin, MDLabel label=MDL_UNDEFINED);

    /*
    * Sort a Metadata by a label.
    * Sort the content of MDin comparing
    * the label supplied, the result will
    * be in the "calling" MetaData.
    * Limit fixes the maximum number of returned rows
    * Offset skips the first N rows
    */
    void sort(MetaDataDb &MDin,
              const MDLabel sortLabel,
              bool asc=true,
              int limit=-1,
              int offset=0);


    /*
    * Sort a Metadata by a label.
    * Sort the content of MDin comparing
    * the label supplied, the result will
    * be in the "calling" MetaData.
    * If the input label is a vector field,
    * you may supply label:col, to sort by that column,
    * e.g., NMADisplacements:0
    * Limit fixes the maximum number of returned rows
    * Offset skips the first N rows
    *
    */
    void sort(MetaDataDb &MDin, const String &sortLabel, bool asc=true, int limit=-1, int offset=0);

    void split(size_t n, std::vector<MetaDataDb> &results,
               const MDLabel sortLabel=MDL_OBJID);

    void selectSplitPart(const MetaData &mdIn,
                         size_t n, size_t part,
                         const MDLabel sortLabel=MDL_OBJID);

    void selectRandomSubset(const MetaData &mdIn, size_t numberOfObjects, const MDLabel sortLabel=MDL_OBJID) override;

    void selectPart(const MetaData &mdIn, size_t startPosition, size_t numberOfObjects,
                    const MDLabel sortLabel=MDL_OBJID) override;

    void makeAbsPath(const MDLabel label=MDL_IMAGE);

    template <bool IsConst>
    struct MDDbRowIterator : public MDBaseRowIterator<IsConst> {
    private:
        typename TypeHelpers::choose<IsConst, const MetaDataDb&, MetaDataDb&>::type _mdd;
        typename TypeHelpers::choose<IsConst, MDRowSql, MDRowSql>::type _row;
        std::vector<size_t> _ids;
        size_t _i;
        bool _finalized = false;

    public:
        MDDbRowIterator(typename TypeHelpers::choose<IsConst, const MetaDataDb&, MetaDataDb&>::type &mdd, size_t _i)
            : _mdd(mdd), _i(_i) {
            mdd.myMDSql->selectObjects(_ids);
            if (this->_i >= _ids.size())
                return;

            _mdd.initGetRow(false);

            _mdd.execGetRow(this->_row);
            this->_row.set_id(this->_ids[this->_i]);
            if (this->_i+1 == _ids.size()) {
                _mdd.finalizeGetRow();
                _finalized = true;
            }
        }

        virtual ~MDDbRowIterator() {
            if (!_finalized)
                _mdd.finalizeGetRow();
        }

        std::unique_ptr<MDBaseRowIterator<IsConst>> clone() override {
            return memoryUtils::make_unique<MDDbRowIterator<IsConst>>(_mdd, _i);
        }

        void increment() override {
            if (this->_i >= _ids.size())
                return;

            this->_i++;

            if (this->_i >= _ids.size())
                return;

            this->_mdd.execGetRow(this->_row);
            this->_row.set_id(this->_ids[this->_i]);

            if (this->_i == _ids.size()) {
                _mdd.finalizeGetRow();
                _finalized = true;
            }
        }

        bool operator==(const MDBaseRowIterator<IsConst>& other) const override {
            const MDDbRowIterator<IsConst>* dri = dynamic_cast<const MDDbRowIterator<IsConst>*>(&other);
            if (dri != nullptr)
                return this->_i == dri->_i;
            return false;
        }

        typename TypeHelpers::choose<IsConst, const MDRowSql&, MDRowSql&>::type operator*() override { return _row; }
    };

    iterator begin() override {
        return {memoryUtils::make_unique<MDDbRowIterator<false>>(*this, 0)};
    }
    iterator end() override {
        return {memoryUtils::make_unique<MDDbRowIterator<false>>(*this, this->size())};
    }

    const_iterator begin() const override {
        return {memoryUtils::make_unique<MDDbRowIterator<true>>(*this, 0)};
    }
    const_iterator end() const override {
        return {memoryUtils::make_unique<MDDbRowIterator<true>>(*this, this->size())};
    }


    template <bool IsConst>
    struct MDDbIdIterator : public MDBaseIdIterator<IsConst> {
    private:
        const MetaDataDb& _mdd;
        bool _last;
        const MDQuery* _pQuery;
        std::vector<size_t> _ids;
        size_t _i;

    public:
        MDDbIdIterator(const MetaDataDb& mdd, bool last = false, const MDQuery* pQuery = nullptr)
            : _mdd(mdd), _last(last), _pQuery(pQuery) {
            mdd.myMDSql->selectObjects(_ids, pQuery);
            _i = last ? this->_ids.size() : 0;
        }

        bool operator==(const MDBaseIdIterator<IsConst>& other) const override {
            const MDDbIdIterator<IsConst>* dri = dynamic_cast<const MDDbIdIterator<IsConst>*>(&other);
            if (dri != nullptr)
                return this->_i == dri->_i;
            return false;
        }

        size_t operator*() override { return _ids[_i]; }

        void increment() override { this->_i++; }

        std::unique_ptr<MDBaseIdIterator<IsConst>> clone() override {
            return memoryUtils::make_unique<MDDbIdIterator<IsConst>>(_mdd, _last, _pQuery);
        }
    };

    id_iterator id_begin() override {
        return {memoryUtils::make_unique<MDDbIdIterator<false>>(*this)};
    }

    id_iterator id_end() override {
        return {memoryUtils::make_unique<MDDbIdIterator<false>>(*this, true)};
    }

    id_const_iterator id_begin() const override {
        return {memoryUtils::make_unique<MDDbIdIterator<true>>(*this)};
    }

    id_const_iterator id_end() const override {
        return {memoryUtils::make_unique<MDDbIdIterator<true>>(*this, true)};
    }

    void fillExpand(MDLabel label);

    void fillConstant(MDLabel label, const String &value) override;
    void fillRandom(MDLabel label, const String &mode, double op1, double op2, double op3=0.) override;
    void fillLinear(MDLabel label, double initial, double step) override;

    void copyColumn(MDLabel labelDest, MDLabel labelSrc) override;
    void copyColumnTo(MetaData& md, MDLabel labelDest, MDLabel labelSrc) override;

    void renameColumn(MDLabel oldLabel, MDLabel newLabel) override;
    void renameColumn(const std::vector<MDLabel> &oldLabel,
            const std::vector<MDLabel> &newLabel) override;

    void metadataToVec(std::vector<MDRowSql> &vd);
    void vecToMetadata(const std::vector<MDRow> &rowMetadata);

    bool operator==(const MetaDataDb& op) const;
}
;//class MetaData

std::ostream& operator<<(std::ostream& o, const MetaData & mD);

WriteModeMetaData metadataModeConvert (String mode);

template<typename T>
bool MetaDataDb::getColumnValuesOpt(const MDLabel label, std::vector<T> &values) const {
    if (!containsLabel(label))
            return false;
    return sqlUtils::select(label,
            myMDSql->db,
            myMDSql->tableName(myMDSql->tableId),
            values);
}

#endif