Xmipp  v3.23.11-Nereus
Public Member Functions | Protected Member Functions | List of all members
ProgMetadataUtilities Class Reference
Inheritance diagram for ProgMetadataUtilities:
Inheritance graph
[legend]
Collaboration diagram for ProgMetadataUtilities:
Collaboration graph
[legend]

Public Member Functions

void run ()
 
- Public Member Functions inherited from XmippProgram
const char * getParam (const char *param, int arg=0)
 
const char * getParam (const char *param, const char *subparam, int arg=0)
 
int getIntParam (const char *param, int arg=0)
 
int getIntParam (const char *param, const char *subparam, int arg=0)
 
double getDoubleParam (const char *param, int arg=0)
 
double getDoubleParam (const char *param, const char *subparam, int arg=0)
 
float getFloatParam (const char *param, int arg=0)
 
float getFloatParam (const char *param, const char *subparam, int arg=0)
 
void getListParam (const char *param, StringVector &list)
 
int getCountParam (const char *param)
 
bool checkParam (const char *param)
 
bool existsParam (const char *param)
 
void addParamsLine (const String &line)
 
void addParamsLine (const char *line)
 
ParamDefgetParamDef (const char *param) const
 
virtual void quit (int exit_code=0) const
 
virtual int tryRun ()
 
void initProgress (size_t total, size_t stepBin=60)
 
void setProgress (size_t value=0)
 
void endProgress ()
 
void processDefaultComment (const char *param, const char *left)
 
void setDefaultComment (const char *param, const char *comment)
 
virtual void initComments ()
 
void setProgramName (const char *name)
 
void addUsageLine (const char *line, bool verbatim=false)
 
void clearUsage ()
 
void addExampleLine (const char *example, bool verbatim=true)
 
void addSeeAlsoLine (const char *seeAlso)
 
void addKeywords (const char *keywords)
 
const char * name () const
 
virtual void usage (int verb=0) const
 
virtual void usage (const String &param, int verb=2)
 
int version () const
 
virtual void show () const
 
virtual void read (int argc, const char **argv, bool reportErrors=true)
 
virtual void read (int argc, char **argv, bool reportErrors=true)
 
void read (const String &argumentsLine)
 
 XmippProgram ()
 
 XmippProgram (int argc, const char **argv)
 
virtual ~XmippProgram ()
 

Protected Member Functions

void defineParams ()
 
void readParams ()
 
void doSet ()
 
void doOperate ()
 
void doFill ()
 
void doQuery ()
 
void doFile ()
 
- Protected Member Functions inherited from XmippProgram
void defineCommons ()
 

Additional Inherited Members

- Public Attributes inherited from XmippProgram
bool doRun
 
bool runWithoutArgs
 
int verbose
 Verbosity level. More...
 
int debug
 
- Protected Attributes inherited from XmippProgram
int errorCode
 
ProgramDefprogDef
 Program definition and arguments parser. More...
 
std::map< String, CommentListdefaultComments
 
int argc
 Original command line arguments. More...
 
const char ** argv
 

Detailed Description

Definition at line 33 of file metadata_utilities.cpp.

Member Function Documentation

◆ defineParams()

void ProgMetadataUtilities::defineParams ( )
inlineprotectedvirtual

Function in which the param of each Program are defined.

Reimplemented from XmippProgram.

Definition at line 45 of file metadata_utilities.cpp.

46  {
47  addUsageLine("Perform several operations on metadata files. ");
48  addUsageLine("If the -o option is not used the original metadata will be modified.");
49  addUsageLine("+ Also you can use the --print option just to print out the result metadata to screen.");
50  addUsageLine("+ The combination of -i and -o without other operations can serve to extract data blocks");
51  addUsageLine("+ inside a metadata and write to an independent one.");
52  addSeeAlsoLine("metadata_import");
53 
54  addParamsLine(" -i <metadata> : Input metadata file");
55  addParamsLine(" [-o <metadata>] : Output metadata file, if not provided result will overwrite input file");
56 
57  addParamsLine(" [--set <set_operation> <md2_file> <label=image> <label2=image2>] : Set operations");
58  addParamsLine(" where <set_operation>");
59  addParamsLine(" union : Union with metadata md2, duplicated values only will appear once");
60  addParamsLine(" union_all : Union with metadata md2, will repeat duplicated values");
61  addParamsLine(" intersection : Intersection with metadata md2");
62  addParamsLine(" subtraction : Subtraction with metadata md2");
63  addParamsLine(" join : Inner join with md2 using label l1");
64  addParamsLine(" natural_join : Natural join with md2 using all common labels");
65  addParamsLine(" inner_join : Inner join with label=label2. No label may be repeated");
66  addParamsLine(" merge : Merge columns with md2, label is ignored");
67  addParamsLine(" : Both metadatas should have same size, and elements should be in same order,");
68  addParamsLine(" : if not, you should use 'join' instead, but this constrain having a common label");
69  addParamsLine(" alias -s; ");
70 
71  addParamsLine("or --operate <operation> : Operations on the metadata structure");
72  addParamsLine(" where <operation>");
73  addParamsLine(" sort <label=image> <order=asc>: Sort metadata using a label as identifier");
74  addParamsLine(" : for sorting according to a component of a vector label");
75  addParamsLine(" : use label:col, e.g., NMADisplacements:0");
76  addParamsLine(" : The first column is column number 0.");
77  addParamsLine(" : order can be asc (ascending) or desc (descending)");
78  addParamsLine(" percentile <labelIn> <labelOut> : Fill a column with the percentile (between 0 and 1) of another column");
79  addParamsLine(" :+The metadata is sorted by labelIn");
80  addParamsLine(" random_subset <size> : Extract a random subset without replacement of this metadata");
81  addParamsLine(" bootstrap : Extract a bootstrap subset (with replacement) of this metadata");
82  addParamsLine(" randomize : Randomize elements of metadata");
83  addParamsLine(" keep_column <labels> : Keep some columns(label list) from metadata");
84  addParamsLine(" drop_column <labels> : Drop some columns(label list) from metadata");
85  addParamsLine(" remove_duplicates <label> : Remove duplicates on a given label");
86  addParamsLine(" rename_column <labels> : Rename a column");
87  addParamsLine(" modify_values <expression> : Use an SQLite expression to modify the metadata");
88  addParamsLine(" : This option requires knowledge of basic SQL syntax(more specific SQLite");
89  addParamsLine(" expand <factor> : Expand the metadata content by union with himself ");
90  addParamsLine(":+ Some of the function allowed are,");
91  addParamsLine(":+ Math: +, -, *, /, abs");
92  addParamsLine(":+ acos, asin, atan, atn2, atan2, acosh, asinh, atanh,");
93  addParamsLine(":+ difference,");
94  addParamsLine(":+ degrees, radians, cos, sin, tan, cot, cosh, sinh, tanh,");
95  addParamsLine(":+ coth, exp,");
96  addParamsLine(":+ log, log10, power, sign, sqrt, square, ceil, floor, pi.");
97  addParamsLine(":+ String: replicate, charindex, leftstr, rightstr, ltrim, rtrim, trim,");
98  addParamsLine(":+ replace, reverse, proper, padl, padr, padc, strfilter.");
99 
100  addParamsLine(":+ Aggregate: max, min, avg, sum, mstdev, variance, mode, median,");
101  addParamsLine(":+ lower_quartile, upper_quartile.");
102  addParamsLine(" alias -e; ");
103 
104 
105  addParamsLine("or --file <file_operation> : File operations");
106  addParamsLine(" where <file_operation>");
107  addParamsLine(" copy <directory> <label=image> : Copy files in metadata md1 to directory path (file names at label column)");
108  addParamsLine(" move <directory> <label=image> : Move files in metadata md1 to directory path (file names at label column)");
109  addParamsLine(" delete <label=image> : Delete files in metadata md1 (file names at label column)");
110  // addParamsLine(" convert2db : Convert metadata to sqlite database");
111  // addParamsLine(" convert2xml : Convert metadata to xml file");
112  addParamsLine(" import_txt <labels> : Import a text file specifying its columns");
113  addParamsLine(" alias -f; ");
114 
115  addParamsLine("or --query <query_operation> : Query operations");
116  addParamsLine(" where <query_operation>");
117  addParamsLine(" select <expression> : Create new metadata with those entries that satisfy the expression");
118  addParamsLine(" count <label> : for each value of a given label create new metadata with the number of times the value appears");
119  addParamsLine(" sum <label1> <label2> : group metadata by label1 and add quantities in label2");
120  addParamsLine(" size : print Metadata size");
121  addParamsLine(" labels : print Metadata labels");
122  addParamsLine(" blocks : print blocks in file");
123  addParamsLine(" alias -q; ");
124 
125  addParamsLine("or --fill <labels> <fill_mode> : Fill a column values(should be of same type)");
126  addParamsLine(" where <fill_mode>");
127  addParamsLine(" constant <value> : Fill with a constant value");
128  addParamsLine(" lineal <init_value> <step> : Fill with a lineal serie starting at init_value with an step");
129  addParamsLine(" rand_uniform <a=0.> <b=1.> : Follow a uniform distribution between a and b");
130  addParamsLine(" rand_gaussian <mean=0.> <stddev=1.> : Follow a gaussian distribution with mean and stddev");
131  addParamsLine(" rand_student <mean=0.> <stddev=1.> <df=3.> : Follow a student distribution with mean, stddev and df degrees of freedom.");
132  addParamsLine(" expand : Treat the column as the filename of an row metadata and expand values");
133  addParamsLine(" alias -l; ");
134 
135  addParamsLine(" [--print ] : Just print medata to stdout, or if -o is specified written to disk.");
136  addParamsLine(" : this option is useful for extrating data blocks inside a metadata.");
137  addParamsLine(" alias -p; ");
138 
139  addParamsLine(" [--mode <mode=overwrite>] : Metadata writing mode.");
140  addParamsLine(" where <mode>");
141  addParamsLine(" overwrite : Replace the content of the file with the Metadata");
142  addParamsLine(" append : Write the Metadata as a new block, removing the old one");
143 
144  addExampleLine(" Concatenate two metadatas. If label is not provided, by default is 'image'", false);
145  addExampleLine (" xmipp_metadata_utilities -i mD1.doc --set union mD2.doc -o out.doc");
146  addExampleLine(" Intersect two metadatas using label 'order_'", false);
147  addExampleLine (" xmipp_metadata_utilities -i mD1.doc --set intersection mD2.doc order_ -o out.doc");
148  addExampleLine(" Combine columns from two metadatas. Be sure of both have same number of rows and also", false);
149  addExampleLine(" there aren't common columns, in that case second metadata columns will be used", false);
150  addExampleLine (" xmipp_metadata_utilities -i mD1.doc --set merge mD2.doc -o out.doc");
151  addExampleLine(" Sort the elements in metadata (using default label 'image').", false);
152  addExampleLine (" xmipp_metadata_utilities -i mD1.doc --operate sort -o out.doc");
153  addExampleLine(" You can also add columns and 'filling' its values with different options", false);
154  addExampleLine("By example, to add the column 'shiftX' with uniform random value between 0 and 10", false);
155  addExampleLine (" xmipp_metadata_utilities -i mD1.doc --fill shiftX rand_uniform 0 10 -o out.doc");
156  addExampleLine("Or for initialize metadata columns 'shiftX' and 'shiftY' with a constant value of 5", false);
157  addExampleLine (" xmipp_metadata_utilities -i mD1.doc -l \"shiftX shiftY\" constant 5 -o out.doc");
158  addExampleLine("If you have columns that represent the filename of a metadata with other data (ex CTFParams)", false);
159  addExampleLine("you cannot 'expand' the column with the values in that metadata", false);
160  addExampleLine (" xmipp_metadata_utilities -i mD1.doc --fill CTFParams expand -o outExpanded.doc");
161  addExampleLine("For check all options availables for 'filling' mode, use: ", false);
162  addExampleLine (" xmipp_metadata_utilities --help fill");
163  addExampleLine(" write metadata as table in Sqlite3 database. (use xmipp_sqlite3 to visualize results)", false);
164  addExampleLine (" xmipp_metadata_utilities -i blocknameIn@mD1.doc -o blocknameOut@mD1.sqlite");
165  addExampleLine(" write metadata as xml file.", false);
166  addExampleLine (" xmipp_metadata_utilities -i blocknameIn@mD1.doc -o blocknameOut@mD1.xml");
167  addExampleLine(" Copy files in metadata to a location. The metadata will be also copied to new location", false);
168  addExampleLine (" xmipp_metadata_utilities -i mD1.doc --file copy /home/pepe/newLocation");
169  addExampleLine(" Delete files in metadata.", false);
170  addExampleLine (" xmipp_metadata_utilities -i mD1.doc --file delete");
171  addExampleLine(" Select elements in metadata that satisfy a given constrain.", false);
172  addExampleLine (" xmipp_metadata_utilities -i mD1.doc --query select \"angleRot > 10 AND anglePsi < 0.5\" -o out.doc");
173  addExampleLine(" You can also modify your data using SQLite syntax expression", false);
174  addExampleLine(" xmipp_metadata_utilities -i a.doc --operate modify_values \"angleRot=2.*angleRot\" -o b.doc");
175  addExampleLine(" xmipp_metadata_utilities -i a.doc --operate modify_values \"angleRot=radians(angleRot)\" -o b.doc");
176  addExampleLine(" xmipp_metadata_utilities -i a.doc --operate modify_values \"angleRot=sin(radians(angleRot))\" -o b.doc");
177  addExampleLine(" xmipp_metadata_utilities -i a.doc --operate modify_values \"angleRot=sqrt(angleRot)\" -o b.doc");
178  addExampleLine(" xmipp_metadata_utilities -i a.doc --operate modify_values \"image=replace(image, 'xmp','spi')\" -o b.doc");
179  addExampleLine(" xmipp_metadata_utilities -i a.doc --operate modify_values \"image='new_prefix_dir/'||image\" -o b.doc");
180  addExampleLine(" Count number of images per CTF", false);
181  addExampleLine (" xmipp_metadata_utilities -i mD1.doc -q count CTFModel -o out.doc");
182  addExampleLine(" images assigned a ctfgroup", false);
183  addExampleLine (" xmipp_metadata_utilities -i mD1.doc -q sum defocusGroup count -o out.doc");
184  addExampleLine(" Print the metadata Size", false);
185  addExampleLine (" xmipp_metadata_utilities -i mD1.doc --query size");
186  addExampleLine(" Rename Column", false);
187  addExampleLine (" xmipp_metadata_utilities -i mD1.doc --operate rename_column \"weight wRobust\"");
188 
189  }
void addSeeAlsoLine(const char *seeAlso)
void addExampleLine(const char *example, bool verbatim=true)
void addUsageLine(const char *line, bool verbatim=false)
void addParamsLine(const String &line)

◆ doFile()

void ProgMetadataUtilities::doFile ( )
inlineprotected

Definition at line 452 of file metadata_utilities.cpp.

453  {
454  operation = getParam("--file", 0);
455 
456  // if (operation == "convert2db")
457  // {
458  // doWrite = false;
459  // MDSql::dumpToFile(fn_out);
460  // }
461  // else if (operation == "convert2xml")
462  // {
463  // fn_out.re
464  // doWrite = false;
465  // mdIn.writeXML(fn_out);
466  // }
467  if (operation == "import_txt")
468  {
469  mdIn.readPlain(fn_in, getParam("--file", 1));
470  }
471  else
472  {
473  bool doDelete;
474  FileName path, inFnImg, outFnImg, oldOutFnImg="";
475  if (!(doDelete = operation == "delete"))//copy or move
476  {
477  path = getParam("--file", 1);
478  if (!path.exists())
479  if (path.makePath() != 0)
480  REPORT_ERROR(ERR_IO_NOPERM, (String)"Cannot create directory "+ path);
481  }
482  label = MDL::str2Label(getParam("--file", doDelete ? 1 : 2));
483  doWrite = !doDelete;
484 
485  int counter=FIRST_IMAGE;
486  for (size_t objId : mdIn.ids())
487  {
488  mdIn.getValue(label, inFnImg, objId);
489  bool isStack=inFnImg.isInStack();
490  if (doDelete)
491  {
492  if(isStack)
493  REPORT_ERROR(ERR_NOT_IMPLEMENTED,"Cannot delete files from a stack");
494  else
495  remove(inFnImg.c_str());
496  }
497  else
498  {
499  outFnImg = inFnImg.removeDirectories();
500  //outfilename for output sel
501  if (operation == "copy" && isStack)
502  {
503  if(oldOutFnImg != outFnImg && oldOutFnImg !="")
504  {
505  counter = FIRST_IMAGE;
506  }
507  oldOutFnImg = outFnImg;
508  outFnImg.compose(counter, outFnImg);
509  }
510  mdIn.setValue(label, outFnImg, objId);
511  //output file name for copy
512  if (operation == "copy" && isStack)
513  {
514  outFnImg=outFnImg.removePrefixNumber();
515  outFnImg = path + "/" + outFnImg;
516  outFnImg = integerToString(counter)+'@'+outFnImg;
517  }
518  else
519  outFnImg = path + "/" + outFnImg;
520  if (operation == "copy")
521  {
522  copyImage( inFnImg, outFnImg);
523  ++counter;
524  }
525  else if (operation == "move")
526  {
527  if(isStack)
528  REPORT_ERROR(ERR_NOT_IMPLEMENTED,"Cannot move files from a stack");
529  else
530  rename(inFnImg.c_str(), outFnImg.c_str());
531  }
532  }
533  }
534  fn_out = path + "/" + fn_out.removeDirectories();
535  }
536  }
Case or algorithm not implemented yet.
Definition: xmipp_error.h:177
static MDLabel str2Label(const String &labelName)
#define REPORT_ERROR(nerr, ErrormMsg)
Definition: xmipp_error.h:211
void copyImage(const FileName &source, const FileName &target)
bool getValue(MDObject &mdValueOut, size_t id) const override
FileName removePrefixNumber() const
void compose(const String &str, const size_t no, const String &ext="")
FileName removeDirectories(int keep=0) const
String integerToString(int I, int _width, char fill_with)
virtual IdIteratorProxy< false > ids()
void readPlain(const FileName &inFile, const String &labelsString, const String &separator=" ")
const char * getParam(const char *param, int arg=0)
int makePath(mode_t mode=0755) const
bool exists() const
bool setValue(const MDObject &mdValueIn, size_t id) override
Definition: metadata_db.cpp:90
std::string String
Definition: xmipp_strings.h:34
#define FIRST_IMAGE
Insufficient permissions to perform operation.
Definition: xmipp_error.h:138
bool isInStack() const

◆ doFill()

void ProgMetadataUtilities::doFill ( )
inlineprotected

Definition at line 349 of file metadata_utilities.cpp.

350  {
351 
352  MDL::str2LabelVector(getParam("--fill", 0), labels);
353 
354  if (labels.empty())
355  REPORT_ERROR(ERR_PARAM_INCORRECT, "You should provide at least one label to fill out");
356 
357  operation = getParam("--fill", 1);
358  MDValueGenerator * generator=nullptr;
359 
360  // Select which generator to use
361  if (operation == "expand")
362  {
363  mdIn.fillExpand(labels[0]);
364  return;
365  }
366  //generator = new MDExpandGenerator();
367  else if (operation == "constant")
368  generator = new MDConstGenerator(getParam("--fill", 2));
369  else if (operation.find("rand_") == 0)
370  {
371  double op1 = getDoubleParam("--fill", 2);
372  double op2 = getDoubleParam("--fill", 3);
373  double op3 = 0.;
374  String type = findAndReplace(operation, "rand_", "");
375  if (type == "student")
376  op3 = getDoubleParam("--fill", 4);
377  generator = new MDRandGenerator(op1, op2, type, op3);
378  }
379  else if (operation == "lineal")
380  generator = new MDLinealGenerator(getDoubleParam("--fill", 2), getDoubleParam("--fill", 3));
381  else {
382  std::string msg = "Unsupported operation (" + operation +
383  "). If you believe this is an error, please contact developers.";
385  }
386 
387  //Fill columns
388  for (size_t i = 0; i < labels.size(); ++i)
389  {
390  generator->label = labels[i];
391  generator->fill(mdIn);
392  }
393 
394  delete generator;
395  }//end of function doFill
Parameter incorrect.
Definition: xmipp_error.h:181
Case or algorithm not implemented yet.
Definition: xmipp_error.h:177
double getDoubleParam(const char *param, int arg=0)
#define REPORT_ERROR(nerr, ErrormMsg)
Definition: xmipp_error.h:211
void fillExpand(MDLabel label)
#define i
const char * getParam(const char *param, int arg=0)
viol type
String findAndReplace(const String &tInput, const String &tFind, const String &tReplace)
std::string String
Definition: xmipp_strings.h:34
static void str2LabelVector(const String &labelsStr, std::vector< MDLabel > &labels)
void fill(MetaData &md)

◆ doOperate()

void ProgMetadataUtilities::doOperate ( )
inlineprotected

Definition at line 260 of file metadata_utilities.cpp.

261  {
262  operation = getParam("--operate", 0);
263 
264  if ( operation == "keep_column")
265  {
266  MDL::str2LabelVector(getParam("--operate", 1), labels);
267  mdIn.keepLabels(labels);
268  //for (int i = 0; i < labels.size(); ++i)
269  // mdIn.addLabel(labels[i]);//removeLabel(labels[i]);
270  }
271  else if ( operation == "drop_column")
272  {
273  MDL::str2LabelVector(getParam("--operate", 1), labels);
274  for (size_t i = 0; i < labels.size(); ++i)
275  mdIn.removeLabel(labels[i]);
276  }
277  else if ( operation == "remove_duplicates")
278  {
279  MetaDataDb aux;
280  aux.removeDuplicates(mdIn,MDL::str2Label(getParam("--operate", 1)));
281  mdIn=aux;
282  }
283  else if ( operation == "rename_column")
284  {
285  MDL::str2LabelVector(getParam("--operate", 1), labels);
286  mdIn.renameColumn(labels[0],labels[1]);
287  }
288  else if (operation == "modify_values")// modify_values
289  {
290  mdIn.operate(getParam("--operate", 1));
291  }
292  else if (operation == "expand")// modify_values
293  {
294  int factor = getIntParam("--operate", 1);
295  MetaDataDb md;
296  for (int i = 0; i < factor; i++)
297  md.unionAll(mdIn);
298 
299  mdIn = md;
300  }else
301  {
302  MetaDataDb md(mdIn);
303  if (operation == "sort")
304  {
305  String order=getParam("--operate",2);
306  mdIn.sort(md, getParam("--operate", 1),order=="asc");
307  }
308  else if (operation == "percentile")
309  {
310  mdIn.sort(md, getParam("--operate", 1),true);
311  double n=1;
312  double iN=1.0/md.size();
313  MDLabel labelOut=MDL::str2Label(getParam("--operate", 2));
314  for (size_t objId : mdIn.ids())
315  mdIn.setValue(labelOut,(n++)*iN,objId);
316  }
317  else if (operation == "randomize")
318  mdIn.randomize(md);
319  else if (operation == "bootstrap")
320  {
321  std::vector<size_t> objId;
322  objId.resize(md.size());
323  size_t n=0;
324  for (size_t id : md.ids())
325  objId[n++] = id;
326  // md.getColumnValues(MDL_OBJID,objId); COSS: It should work, but it does not
327  int N_1=((int)objId.size())-1;
328  MDRowSql row;
329  MetaDataDb mdAux;
330  for (size_t _ : md.ids())
331  {
332  md.getRow(row, objId[(size_t)rnd_unif(0,N_1)]);
333  mdAux.setRow(row,mdAux.addObject());
334  }
335  mdIn.sort(mdAux,MDL_IMAGE);
336  }
337  else if (operation == "random_subset")
338  {
339  MetaDataDb mdAux, mdAux2;
340  mdAux.randomize(md);
341  md.clear();
342  mdAux2.selectPart(mdAux, 0, getIntParam("--operate", 1));
343  mdAux.clear();
344  mdIn.sort(mdAux2,MDL_IMAGE);
345  }
346  }
347  }//end of function doOperate
static MDLabel str2Label(const String &labelName)
void keepLabels(const std::vector< MDLabel > &labels)
virtual IdIteratorProxy< false > ids()
bool removeLabel(const MDLabel label) override
#define i
double rnd_unif()
std::unique_ptr< MDRow > getRow(size_t id) override
void randomize(const MetaDataDb &MDin)
const char * getParam(const char *param, int arg=0)
size_t addObject() override
void operate(const String &expression)
void clear() override
Definition: metadata_db.cpp:54
void removeDuplicates(MetaDataDb &MDin, MDLabel label=MDL_UNDEFINED)
size_t size() const override
void selectPart(const MetaData &mdIn, size_t startPosition, size_t numberOfObjects, const MDLabel sortLabel=MDL_OBJID) override
bool setRow(const MDRow &row, size_t id)
void sort(MetaDataDb &MDin, const MDLabel sortLabel, bool asc=true, int limit=-1, int offset=0)
bool setValue(const MDObject &mdValueIn, size_t id) override
Definition: metadata_db.cpp:90
std::string String
Definition: xmipp_strings.h:34
void unionAll(const MetaDataDb &mdIn)
static void str2LabelVector(const String &labelsStr, std::vector< MDLabel > &labels)
void renameColumn(MDLabel oldLabel, MDLabel newLabel) override
int getIntParam(const char *param, int arg=0)
int * n
Name of an image (std::string)
MDLabel

◆ doQuery()

void ProgMetadataUtilities::doQuery ( )
inlineprotected

Definition at line 397 of file metadata_utilities.cpp.

398  {
399  if (mdIn.size()==0)
400  {
401  md2.clear();
402  return;
403  }
404 
405  operation = getParam("--query", 0);
406  String expression;
407 
408  if (operation == "count")//note second label is a dummy parameter
409  {
410  label = MDL::str2Label(getParam("--query", 1));
411  md2 = mdIn;
412  mdIn.aggregate(md2, AGGR_COUNT,label,label,MDL_COUNT);
413  }
414  else if (operation == "sum")
415  {
416  label = MDL::str2Label(getParam("--query", 1));
417  MDLabel label2;
418  label2 = MDL::str2Label(getParam("--query", 2));
419  md2 = mdIn;
420  mdIn.aggregate(md2, AGGR_SUM,label,label2,MDL_SUM);
421  }
422  else if (operation == "select")
423  {
424  expression = getParam("--query", 1);
425  md2 = mdIn;
426  mdIn.importObjects(md2, MDExpression(expression));
427  }
428  else if (operation == "size")
429  {
430  doWrite = false;
431  std::cout << fn_in + " size is: " << mdIn.getParsedLines() << std::endl;
432  }
433  else if (operation == "labels")
434  {
435  doWrite = false;
436  std::cout << fn_in + " has labels: " << std::endl;
437  MDLabelVector labels = mdIn.getActiveLabels();
438  for (size_t i = 0; i < labels.size(); ++i)
439  std::cout << " " << MDL::label2Str(labels[i]) << std::endl;
440  }
441  else if (operation == "blocks")
442  {
443  doWrite = false;
444  StringVector blocks;
445  std::cout << "Blocks in " << fn_in << ": " << std::endl;
446  getBlocksInMetaDataFile(fn_in, blocks);
447  for (size_t i = 0; i < blocks.size(); ++i)
448  std::cout << blocks[i] << std::endl;
449  }
450  }//end of function doQuery
std::vector< MDLabel > getActiveLabels() const override
Definition: metadata_db.h:309
static MDLabel str2Label(const String &labelName)
Sum of elements of a given type (double) [this is a genereic type do not use to transfer information ...
void getBlocksInMetaDataFile(const FileName &inFile, StringVector &blockList)
std::vector< String > StringVector
Definition: xmipp_strings.h:35
#define i
void aggregate(const MetaDataDb &mdIn, AggregateOperation op, MDLabel aggregateLabel, MDLabel operateLabel, MDLabel resultLabel)
Number of elements of a type (int) [this is a genereic type do not use to transfer information to ano...
const char * getParam(const char *param, int arg=0)
void clear() override
Definition: metadata_db.cpp:54
std::vector< MDLabel > MDLabelVector
size_t size() const override
virtual size_t getParsedLines()
void importObjects(const MetaData &md, const std::vector< size_t > &objectsToAdd, bool doClear=true) override
std::string String
Definition: xmipp_strings.h:34
static String label2Str(const MDLabel &label)
MDLabel

◆ doSet()

void ProgMetadataUtilities::doSet ( )
inlineprotected

Definition at line 215 of file metadata_utilities.cpp.

216  {
217  operation = getParam("--set", 0);
218  md2.read(getParam("--set", 1));
219  MDLabel label = MDL::str2Label(getParam("--set", 2));
220  MDLabel label2 = MDL::str2Label(getParam("--set", 3));
221 
222  if (operation == "union")
223  {
224  if(mdIn.isEmpty())
225  mdIn = md2;
226  else
227  mdIn.unionDistinct(md2, label);
228  }
229  else if (operation == "union_all")
230  if(mdIn.isEmpty())
231  mdIn = md2;
232  else
233  mdIn.unionAll(md2);
234  else if (operation == "intersection")
235  mdIn.intersection(md2, label);
236  else if (operation == "subtraction")
237  mdIn.subtraction(md2, label);
238  else if (operation == "join")
239  {
240  MetaDataDb md;
241  md.join1(mdIn, md2, label);
242  mdIn = md;
243  }
244  else if (operation == "natural_join")
245  {
246  MetaDataDb md;
247  md.joinNatural(mdIn, md2);
248  mdIn = md;
249  }
250  else if (operation == "inner_join")
251  {
252  MetaDataDb md;
253  md.join2(mdIn, md2, label, label2, INNER);
254  mdIn = md;
255  }
256  else if (operation == "merge")
257  mdIn.merge(md2);
258  }//end of function doSet
void subtraction(const MetaDataDb &mdIn, const MDLabel label)
static MDLabel str2Label(const String &labelName)
void intersection(const MetaDataDb &mdIn, const MDLabel label)
void unionDistinct(const MetaDataDb &mdIn, const MDLabel label=MDL_OBJID)
void joinNatural(const MetaDataDb &mdInLeft, const MetaDataDb &mdInRight)
void join2(const MetaDataDb &mdInLeft, const MetaDataDb &mdInRight, const MDLabel labelLeft, const MDLabel labelRight, JoinType type=LEFT)
const char * getParam(const char *param, int arg=0)
virtual bool isEmpty() const
void merge(const MetaData &md2)
void read(const FileName &inFile, const std::vector< MDLabel > *desiredLabels=NULL, bool decomposeStack=true) override
void join1(const MetaDataDb &mdInLeft, const MetaDataDb &mdInRight, const MDLabel label, JoinType type=LEFT)
void unionAll(const MetaDataDb &mdIn)
MDLabel

◆ readParams()

void ProgMetadataUtilities::readParams ( )
inlineprotectedvirtual

Function in which each program will read parameters that it need. If some error occurs the usage will be printed out.

Reimplemented from XmippProgram.

Definition at line 191 of file metadata_utilities.cpp.

192  {
193  fn_in = getParam("-i");
194  // Prevent from reading the input metadata for some special cases
195  bool readIn = !((checkParam("--file") && STR_EQUAL(getParam("--file"), "import_txt")) || //when importing from .txt files
196  (checkParam("--query") && STR_EQUAL(getParam("--query"), "blocks")));
197 
198  if (checkParam("--query") &&
199  (STR_EQUAL(getParam("--query"), "size") ||
200  STR_EQUAL(getParam("--query"), "labels")))
201  mdIn.setMaxRows(1); //avoid parse the entire metadata
202 
203  if (readIn)
204  mdIn.read(fn_in);
205  doWrite = true;
206  fn_out = checkParam("-o") ? getParam("-o") : fn_in;
207  mode = MD_OVERWRITE;
208  if (checkParam("--mode")
209  && STR_EQUAL(getParam("--mode"), "append"))
210  mode = MD_APPEND;
213  }
virtual void setMaxRows(size_t maxRows=0)
static bool activateMathExtensions(void)
static bool activateRegExtensions(void)
const char * getParam(const char *param, int arg=0)
void mode
#define STR_EQUAL(str1, str2)
Definition: xmipp_strings.h:42
void read(const FileName &inFile, const std::vector< MDLabel > *desiredLabels=NULL, bool decomposeStack=true) override
bool checkParam(const char *param)

◆ run()

void ProgMetadataUtilities::run ( )
inlinevirtual

This function will be start running the program. it also should be implemented by derived classes.

Reimplemented from XmippProgram.

Definition at line 539 of file metadata_utilities.cpp.

540  {
541  if (checkParam("--set"))
542  doSet();
543  else if (mdIn.size()==0) // Only set operationts allow an empty input md file
544  return;
545  else if (checkParam("--operate"))
546  doOperate();
547  else if (checkParam("--file"))
548  doFile();
549  else if (checkParam("--query"))
550  doQuery();
551  else if (checkParam("--fill"))
552  doFill();
553 
554  if (checkParam("--print"))
555  mdIn.write(std::cout);
556  else if (doWrite)
557  mdIn.write(fn_out, mode);
558  }
void mode
void write(const FileName &outFile, WriteModeMetaData mode=MD_OVERWRITE) const override
size_t size() const override
bool checkParam(const char *param)

The documentation for this class was generated from the following file: