ProgRecFourierMpiStarPU Class Reference

#include <mpi_reconstruct_fourier_starpu.h>

Inheritance diagram for ProgRecFourierMpiStarPU:

Collaboration diagram for ProgRecFourierMpiStarPU:

Classes
struct	DistributedBatchProvider

Public Member Functions
void	defineParams () override
void	readParams () override
void	read (int argc, char **argv, bool reportErrors=true) override
int	tryRun () override
void	run () override
	~ProgRecFourierMpiStarPU () override
Public Member Functions inherited from ProgRecFourierStarPU
	ProgRecFourierStarPU ()
void	show () const override
void	setIO (const FileName &fn_in, const FileName &fn_out) override
Public Member Functions inherited from ProgReconsBase
virtual	~ProgReconsBase ()
Public Member Functions inherited from XmippProgram
const char *	getParam (const char *param, int arg=0)
const char *	getParam (const char *param, const char *subparam, int arg=0)
int	getIntParam (const char *param, int arg=0)
int	getIntParam (const char *param, const char *subparam, int arg=0)
double	getDoubleParam (const char *param, int arg=0)
double	getDoubleParam (const char *param, const char *subparam, int arg=0)
float	getFloatParam (const char *param, int arg=0)
float	getFloatParam (const char *param, const char *subparam, int arg=0)
void	getListParam (const char *param, StringVector &list)
int	getCountParam (const char *param)
bool	checkParam (const char *param)
bool	existsParam (const char *param)
void	addParamsLine (const String &line)
void	addParamsLine (const char *line)
ParamDef *	getParamDef (const char *param) const
virtual void	quit (int exit_code=0) const
void	initProgress (size_t total, size_t stepBin=60)
void	setProgress (size_t value=0)
void	endProgress ()
void	processDefaultComment (const char *param, const char *left)
void	setDefaultComment (const char *param, const char *comment)
virtual void	initComments ()
void	setProgramName (const char *name)
void	addUsageLine (const char *line, bool verbatim=false)
void	clearUsage ()
void	addExampleLine (const char *example, bool verbatim=true)
void	addSeeAlsoLine (const char *seeAlso)
void	addKeywords (const char *keywords)
const char *	name () const
virtual void	usage (int verb=0) const
virtual void	usage (const String &param, int verb=2)
int	version () const
virtual void	read (int argc, const char **argv, bool reportErrors=true)
void	read (const String &argumentsLine)
	XmippProgram ()
	XmippProgram (int argc, const char **argv)
virtual	~XmippProgram ()

Additional Inherited Members
Public Attributes inherited from XmippProgram
bool	doRun
bool	runWithoutArgs
int	verbose
	Verbosity level. More...
int	debug
Protected Member Functions inherited from XmippProgram
void	defineCommons ()
Static Protected Member Functions inherited from ProgRecFourierStarPU
static void	prepareMetaData (const FileName &fn_in, MetaDataVec &SF)
static uint32_t	computeBatchCount (const Params &params, const MetaData &SF)
static void	prepareConstants (const Params &params, const MetaData &SF, const FileName &fn_sym, ComputeConstants &constants)
static void	initStarPU ()
static ComputeStarPUResult	computeStarPU (const Params &params, const MetaData &SF, const ComputeConstants &computeConstants, BatchProvider &batches, bool verbose)
static void	shutdownStarPU ()
static void	postProcessAndSave (const Params &params, const ComputeConstants &computeConstants, const FileName &fn_out, std::complex< float > ***tempVolume, float ***tempWeights)
Protected Attributes inherited from ProgRecFourierStarPU
FileName	fn_in
FileName	fn_out
FileName	fn_sym
struct ProgRecFourierStarPU::Params	params
MetaDataVec	SF
struct ProgRecFourierStarPU::ComputeConstants	computeConstants
Protected Attributes inherited from XmippProgram
int	errorCode
ProgramDef *	progDef
	Program definition and arguments parser. More...
std::map< String, CommentList >	defaultComments
int	argc
	Original command line arguments. More...
const char **	argv

Detailed Description

MPI extension of ProgRecFourierStarPU. MPI should already be initialized externally. (This is done by runReconstructFourierMpiStarPU function)

At least two program instances are needed - one master and at least one worker (though any real speedup is expected only with multiple workers). It is expected, but not required, that the master will run on the same machine as one of the workers, as it uses minimal processing power.

All program instances, no matter their role, should have the access to identical input files (through network storage, manual copy, etc.).

First worker (i.e. second global rank) will write out the result.

Definition at line 52 of file mpi_reconstruct_fourier_starpu.h.

Constructor & Destructor Documentation

~ProgRecFourierMpiStarPU()

ProgRecFourierMpiStarPU::~ProgRecFourierMpiStarPU ( )

override

MPI shutdown

Definition at line 439 of file mpi_reconstruct_fourier_starpu.cpp.

                                                  {
    if (mpiInitialized) {
        LOG_MPI("Finalizing MPI");
        CHECK_MPI(MPI_Finalize());
    }
}

Member Function Documentation

defineParams()

void ProgRecFourierMpiStarPU::defineParams ( )

overridevirtual

Specify supported command line arguments

Reimplemented from ProgRecFourierStarPU.

Definition at line 49 of file mpi_reconstruct_fourier_starpu.cpp.

                                           {
    ProgRecFourierStarPU::defineParams();
    addParamsLine("  [--mpiDistribute <pc=0.3>]     : Ratio [0,1] describing how many batches should be initially distributed.");
    addParamsLine("                                 : Use less for more heterogeneous group of computers, more for more homogeneous group.");
    addParamsLine("  [--mpiPrefetch <batches=25>]    : How many batches should be in the processing pipeline at all times.");
    addParamsLine("                                 : Higher numbers could be faster with faster and/or more homogeneous computers.");
    addParamsLine("                                 : Very low numbers will have severe performance implications.");
}

read()

void ProgRecFourierMpiStarPU::read ( int  argc, char **  argv, bool  reportErrors = true  )

overridevirtual

Read with MPI Initialization

Reimplemented from XmippProgram.

Definition at line 409 of file mpi_reconstruct_fourier_starpu.cpp.

                                                                           {
    // Not using XmippMpiProgram, because it is unnecessarily complicated for this use-case
    // and does not initialize MPI with thread support, which is required
    LOG_MPI("Initializing MPI");
    int providedThreadSupport;
    CHECK_MPI(MPI_Init_thread(&argc, &argv, MPI_THREAD_SERIALIZED, &providedThreadSupport));
    mpiInitialized = true;
    LOG_MPI("MPI initialized with thread level " << providedThreadSupport);

    if (providedThreadSupport < MPI_THREAD_SERIALIZED) {
        std::cerr << "MPI implementation does not support MPI_THREAD_SERIALIZED concurrency level, which is required. Provided level is " << providedThreadSupport << '\n';
        REPORT_ERROR(ERR_NOT_IMPLEMENTED, "Could not initialize MPI");
    }

    XmippProgram::read(argc, argv, reportErrors);
}

readParams()

void ProgRecFourierMpiStarPU::readParams ( )

overridevirtual

Read arguments from command line

Reimplemented from ProgRecFourierStarPU.

Definition at line 58 of file mpi_reconstruct_fourier_starpu.cpp.

                                         {
    ProgRecFourierStarPU::readParams();

    percentOfJobsDistributedByDefault = static_cast<float>(getDoubleParam("--mpiDistribute"));
    preferredBatchesInPipeline = static_cast<uint32_t>(getIntParam("--mpiPrefetch"));
}

run()

void ProgRecFourierMpiStarPU::run ( )

overridevirtual

Run the image processing. Method will load data, process them and store result to final destination.

Reimplemented from ProgRecFourierStarPU.

Definition at line 65 of file mpi_reconstruct_fourier_starpu.cpp.

                                  {
    int rank, size;
    CHECK_MPI(MPI_Comm_rank(MPI_COMM_WORLD, &rank));
    CHECK_MPI(MPI_Comm_size(MPI_COMM_WORLD, &size));

    {// Implementation of tip 6 from https://www.open-mpi.org/faq/?category=debugging
        // Set this env variable to the rank which is to be debugged
        char* rankStr = getenv("XMIPP_MPI_DEBUG_RANK");
        int rankToBlock = rankStr == nullptr ? -1 : atoi(rankStr);

        if (rankToBlock == rank) {
            volatile /* so it isn't optimized out */ int i = 0;
            char hostname[256];
            gethostname(hostname, sizeof(hostname));
            printf("PID %d on %s ready for attach\n", getpid(), hostname);
            fflush(stdout);

            // After attaching with debugger, set i to arbitrary non-zero value to continue
            // gdb> set var i = 7
            while (0 == i)
                sleep(1);
        }
    }

    if (size < 2) {
        if (rank == 0) {
            REPORT_ERROR(ERR_ARG_DEPENDENCE, "This MPI implementation needs at least two instances");
        } else {
            // no need to report error from all nodes at the same time
            return;
        }
    }

    MPI_Comm workComm;
    // Rest of the code in this file assumes, that world rank 0 is master and other ranks are simply shifted by 1 to form workComm.
    // This code does that, but any modification must take this requirement into account.
    CHECK_MPI(MPI_Comm_split(MPI_COMM_WORLD, rank == 0 ? MPI_UNDEFINED : 0, 0, &workComm));

    const int workerCount = size - 1;

    if (workComm == MPI_COMM_NULL) {
        runMaster(workerCount);
    } else {
        int workerRank;
        CHECK_MPI(MPI_Comm_rank(workComm, &workerRank));
        runWorker(workComm, workerCount, workerRank);
    }
}

tryRun()

int ProgRecFourierMpiStarPU::tryRun ( )

overridevirtual

try with MPI abort

Reimplemented from XmippProgram.

Definition at line 426 of file mpi_reconstruct_fourier_starpu.cpp.

                                    {
    // Based on XmippMpiProgram::tryRun()
    try {
        if (doRun)
            this->run();
    } catch (XmippError &xe) {
        std::cerr << xe;
        errorCode = xe.__errno;
        MPI_Abort(MPI_COMM_WORLD, xe.__errno);
    }
    return errorCode;
}

---

The documentation for this class was generated from the following files:

• xmipp/libraries/reconstruction_starpu/mpi/mpi_reconstruct_fourier_starpu.h
• xmipp/libraries/reconstruction_starpu/mpi/mpi_reconstruct_fourier_starpu.cpp