diff --git a/AMDiS/src/AdaptInstationary.cc b/AMDiS/src/AdaptInstationary.cc index f41b12605008e3700ab2ad83fcaa7a21a954d113..107db52ad38d723fbefa41ff2c77efeabfa7c361 100644 --- a/AMDiS/src/AdaptInstationary.cc +++ b/AMDiS/src/AdaptInstationary.cc @@ -35,7 +35,6 @@ namespace AMDiS { // MSG("Please use the constructor that uses references instead of pointers!\n"); initConstructor(problemStat, info, initialInfo, initialTimestampSet); - } @@ -52,7 +51,6 @@ namespace AMDiS { FUNCNAME("AdaptInstationary::AdaptInstationary()"); initConstructor(&problemStat, &info, &initialInfo, initialTimestampSet); - } diff --git a/AMDiS/src/Global.cc b/AMDiS/src/Global.cc index 8a6b942fb92502adadfcf0304bfd76b4eada7c5c..167a16c22e912d7e4714f7932b9df56c1f4587d7 100644 --- a/AMDiS/src/Global.cc +++ b/AMDiS/src/Global.cc @@ -24,11 +24,16 @@ namespace AMDiS { const char *funcName = NULL; + +#ifdef HAVE_PARALLEL_DOMAIN_AMDIS + bool Msg::outputMainRank = false; +#endif + const char *Msg::oldFuncName = NULL; std::ostream* Msg::out = NULL; std::ostream* Msg::error = NULL; int Global::dimOfWorld = 0; - std::vector< std::vector< int > > Global::geoIndexTable; + std::vector<std::vector<int> > Global::geoIndexTable; int Msg::msgInfo = 10; bool Msg::msgWait = true; @@ -39,6 +44,7 @@ namespace AMDiS { new Tetrahedron(NULL) }; + void Msg::wait(bool w) { FUNCNAME("Msg::wait()"); @@ -50,6 +56,7 @@ namespace AMDiS { } } + void Msg::change_out(std::ostream *fp) { FUNCNAME("Msg::change_out()"); @@ -67,12 +74,13 @@ namespace AMDiS { } } + void Msg::change_error_out(std::ofstream *fp) { FUNCNAME("Msg::change_error_out()"); if (fp) { - if (error && *error != std::cout && *error != std::cerr) { + if (error && *error != std::cout && *error != std::cerr) { dynamic_cast< std::ofstream*>(error)->close(); delete error; } @@ -84,13 +92,14 @@ namespace AMDiS { } } + void Msg::open_error_file(const char *filename, OPENMODE type) { FUNCNAME("Msg::open_error_file()"); std::ofstream *fp; - if (filename && (fp = new std::ofstream(filename, type))) { - if (error && *error != std::cout && *error != std::cerr) { + if (filename && (fp = new std::ofstream(filename, type))) { + if (error && *error != std::cout && *error != std::cerr) { dynamic_cast< std::ofstream*>(error)->close(); delete error; } @@ -105,8 +114,14 @@ namespace AMDiS { } } + void Msg::print_funcname(const char *funcName) { +#ifdef HAVE_PARALLEL_DOMAIN_AMDIS + if (outputMainRank && MPI::COMM_WORLD.Get_rank() != 0) + return; +#endif + if (!out) out = &std::cout; @@ -120,6 +135,7 @@ namespace AMDiS { oldFuncName = funcName; } + void Msg::print_error_funcname(const char *funcName, const char *file, int line) { static int old_line = -1; @@ -129,12 +145,13 @@ namespace AMDiS { std::stringstream oss; - if (funcName && oldFuncName != funcName) + if (funcName && oldFuncName != funcName) { oss << funcName << ": "; - else if (!funcName) { + } else if (!funcName) { if (line-old_line > 5) oss << "*unknown function*"; } + if (oldFuncName != funcName) { oss << "ERROR in " << file << ", line " << line << std::endl;; oldFuncName = funcName; @@ -145,6 +162,7 @@ namespace AMDiS { old_line = line; } + void Msg::print_error_exit(const char *format, ...) { va_list arg; @@ -161,6 +179,7 @@ namespace AMDiS { exit(1); } + void Msg::print_error(const char *format, ...) { va_list arg; @@ -176,6 +195,7 @@ namespace AMDiS { va_end(arg); } + void Msg::print_warn_funcname(const char *funcName, const char *file, int line) @@ -206,6 +226,7 @@ namespace AMDiS { old_line = line; } + void Msg::print_warn(const char *format, ...) { va_list arg; @@ -223,6 +244,11 @@ namespace AMDiS { void Msg::print(const char *format, ...) { +#ifdef HAVE_PARALLEL_DOMAIN_AMDIS + if (outputMainRank && MPI::COMM_WORLD.Get_rank() != 0) + return; +#endif + va_list arg; char buff[255]; @@ -235,6 +261,7 @@ namespace AMDiS { va_end(arg); } + void Global::init() { int d = -1; @@ -281,6 +308,7 @@ namespace AMDiS { Msg::setMsgWait(!(d == 0)); } + void Global::clear() { delete referenceElement[1]; @@ -288,6 +316,7 @@ namespace AMDiS { delete referenceElement[3]; } + int fac(int i) { if (i <= 1) @@ -296,25 +325,11 @@ namespace AMDiS { return i * fac(i - 1); } + void waitSec(int seconds) { clock_t endwait = clock () + seconds * CLOCKS_PER_SEC; while (clock() < endwait) {} } - std::string memSizeStr(int size) - { - std::string result; -// if (size > 1024) { -// if (size > 1024 * 1024) { -// result << size / (1024 * 1024) << " MByte"; -// } else { -// result << size / 1024 << " KByte"; -// } -// } else { -// result << size << " Byte"; -// } - - return result; - } } diff --git a/AMDiS/src/Global.h b/AMDiS/src/Global.h index 8f340ead6c222488cd0836d6211831bb9fc85227..ebefbcb199178e200ebe798f8485d1e0019db22f 100644 --- a/AMDiS/src/Global.h +++ b/AMDiS/src/Global.h @@ -50,7 +50,7 @@ #include <time.h> #if HAVE_PARALLEL_DOMAIN_AMDIS -#include "mpi.h" +#include <mpi.h> #endif #include "boost/tuple/tuple.hpp" @@ -243,6 +243,14 @@ namespace AMDiS { return error; } + public: +#if HAVE_PARALLEL_DOMAIN_AMDIS + /// In parallel computations, when this variable is true, only the 0 rank will + /// print messages to the output stream. Error messages and warnings are always + /// printed from all ranks. + static bool outputMainRank; +#endif + protected: /// Message stram static std::ostream *out; @@ -306,11 +314,11 @@ namespace AMDiS { /// prints a message, if min(Msg::msgInfo, info) >= noinfo #define INFO(info,noinfo) \ - if (Msg::getMsgInfo()&&(std::min(Msg::getMsgInfo(),(info))>=(noinfo))) MSG + if (Msg::getMsgInfo() && (std::min(Msg::getMsgInfo(), (info)) >= (noinfo))) MSG /// prints a message, if min(Msg::msgInfo, info) >= noinfo #define PRINT_INFO(info,noinfo) \ - if (Msg::getMsgInfo()&&(std::min(Msg::getMsgInfo(),(info))>=(noinfo))) Msg::print + if (Msg::getMsgInfo() && (std::min(Msg::getMsgInfo(), (info)) >= (noinfo))) Msg::print /** \brief @@ -442,8 +450,6 @@ namespace AMDiS { const int RescheduleErrorCode = 23; - std::string memSizeStr(int size); - /** * \ingroup Assembler * \brief diff --git a/AMDiS/src/ProblemInstat.cc b/AMDiS/src/ProblemInstat.cc index 3338ed827527b868eaf5ec401d3cf205d33b21ae..267a0aa273201360e765c0cdd23e4c1ae8bac447 100644 --- a/AMDiS/src/ProblemInstat.cc +++ b/AMDiS/src/ProblemInstat.cc @@ -9,6 +9,9 @@ // // See also license.opensource.txt in the distribution. +#ifndef HAVE_PARALLEL_DOMAIN_AMDIS +#include <mpi.h> +#endif #include "ProblemInstat.h" #include "io/FileWriter.h" @@ -23,11 +26,13 @@ namespace AMDiS { ProblemInstat::~ProblemInstat() {} + void ProblemInstat::initialize(Flag initFlag, ProblemInstat *adoptProblem/* = NULL*/, Flag adoptFlag /* = INIT_NOTHING*/) {} + void ProblemInstat::solveInitialProblem(AdaptInfo *adaptInfo) { AdaptStationary initialAdapt((name + "->initial->adapt").c_str(), @@ -37,6 +42,7 @@ namespace AMDiS { initialAdapt.adapt(); } + void ProblemInstatScal::transferInitialSolution(AdaptInfo *adaptInfo) { TEST_EXIT(adaptInfo->getTime() == adaptInfo->getStartTime()) @@ -44,6 +50,7 @@ namespace AMDiS { problemStat->writeFiles(adaptInfo, true); } + void ProblemInstatVec::transferInitialSolution(AdaptInfo *adaptInfo) { TEST_EXIT(adaptInfo->getTime() == adaptInfo->getStartTime()) @@ -59,12 +66,14 @@ namespace AMDiS { oldSolution(NULL) {} + ProblemInstatScal::ProblemInstatScal(std::string sname, ProblemScal& prob) : ProblemInstat(sname, NULL), problemStat(&prob), oldSolution(NULL) {} + ProblemInstatScal::ProblemInstatScal(std::string sname, ProblemScal& prob, ProblemStatBase& initialProb) : ProblemInstat(sname, &initialProb), @@ -72,11 +81,13 @@ namespace AMDiS { oldSolution(NULL) {} + ProblemInstatScal::~ProblemInstatScal() { delete oldSolution; } + void ProblemInstatScal::initialize(Flag initFlag, ProblemInstat *adoptProblem, Flag adoptFlag) @@ -131,8 +142,15 @@ namespace AMDiS { void ProblemInstatVec::closeTimestep(AdaptInfo *adaptInfo) { + FUNCNAME("ProblemInstatVec::closeTimestep()"); + bool force = (adaptInfo->getTime() >= adaptInfo->getEndTime()); problemStat->writeFiles(adaptInfo, force); + +#ifdef HAVE_PARALLEL_DOMAIN_AMDIS + MSG("Computational time for timestep: %.5f seconds\n", + (MPI::Wtime() - lastTimepoint)); +#endif } @@ -222,6 +240,10 @@ namespace AMDiS { void ProblemInstatVec::initTimestep(AdaptInfo *adaptInfo) { +#ifdef HAVE_PARALLEL_DOMAIN_AMDIS + lastTimepoint = MPI::Wtime(); +#endif + oldSolution->copy(*(problemStat->getSolution())); } diff --git a/AMDiS/src/ProblemInstat.h b/AMDiS/src/ProblemInstat.h index aff82e6daecb08aef4eebfc309052bec3f8be73d..24887b8e9c389e9a39f09c533a3abb39fb926029 100644 --- a/AMDiS/src/ProblemInstat.h +++ b/AMDiS/src/ProblemInstat.h @@ -260,6 +260,12 @@ namespace AMDiS { /// Solution of the last timestep. SystemVector *oldSolution; + + /// In parallel computations, we want to print the overall computational time + /// that is used for one timestep. +#ifdef HAVE_PARALLEL_DOMAIN_AMDIS + double lastTimepoint; +#endif }; } diff --git a/AMDiS/src/ProblemVec.cc b/AMDiS/src/ProblemVec.cc index 10b81bd9840ed48c16ee1ef6f6ab2c785a04758a..42917ac9fd533ce49775556d347517364a02276c 100644 --- a/AMDiS/src/ProblemVec.cc +++ b/AMDiS/src/ProblemVec.cc @@ -537,10 +537,14 @@ namespace AMDiS { { FUNCNAME("ProblemVec::estimate()"); - clock_t first = clock(); - +#ifdef HAVE_PARALLEL_DOMAIN_AMDIS + double first = MPI::Wtime(); +#else #ifdef _OPENMP - double wtime = omp_get_wtime(); + double first = omp_get_wtime(); +#else + clock_t first = clock(); +#endif #endif if (computeExactError) { @@ -565,12 +569,17 @@ namespace AMDiS { } } +#ifdef HAVE_PARALLEL_DOMAIN_AMDIS + INFO(info, 8)("estimation of the error needed %.5f seconds\n", + MPI::Wtime() - first); +#else #ifdef _OPENMP - INFO(info, 8)("estimation of the error needed %.5f seconds system time / %.5f seconds wallclock time\n", - TIME_USED(first, clock()), omp_get_wtime() - wtime); + INFO(info, 8)("estimation of the error needed %.5f seconds\n", + omp_get_wtime() - first); #else INFO(info, 8)("estimation of the error needed %.5f seconds\n", TIME_USED(first, clock())); +#endif #endif } @@ -1081,28 +1090,35 @@ namespace AMDiS { { FUNCNAME("ProblemVec::writeFiles()"); - clock_t first = clock(); - +#ifdef HAVE_PARALLEL_DOMAIN_AMDIS + double first = MPI::Wtime(); +#else #ifdef _OPENMP - double wtime = omp_get_wtime(); + double first = omp_get_wtime(); +#else + clock_t first = clock(); +#endif #endif - int i; int size = static_cast<int>(fileWriters.size()); #ifdef _OPENMP #pragma omp parallel for schedule(static, 1) #endif - for (i = 0; i < size; i++) { + for (int i = 0; i < size; i++) { fileWriters[i]->writeFiles(adaptInfo, force); } - + +#ifdef HAVE_PARALLEL_DOMAIN_AMDIS + INFO(info, 8)("writeFiles needed %.5f seconds\n", + MPI::Wtime() - first); +#else #ifdef _OPENMP - INFO(info, 8)("writeFiles needed %.5f seconds system time / %.5f seconds wallclock time\n", - TIME_USED(first, clock()), - omp_get_wtime() - wtime); + INFO(info, 8)("writeFiles needed %.5f seconds\n", + omp_get_wtime() - first); #else INFO(info, 8)("writeFiles needed %.5f seconds\n", TIME_USED(first, clock())); +#endif #endif } diff --git a/AMDiS/src/StandardProblemIteration.cc b/AMDiS/src/StandardProblemIteration.cc index b8f2b613823be6eeb2e6ac77f44d93e034ab3a3f..04c2ddf5c67e80d98595ac25b4c42bd44b36425a 100644 --- a/AMDiS/src/StandardProblemIteration.cc +++ b/AMDiS/src/StandardProblemIteration.cc @@ -20,6 +20,7 @@ namespace AMDiS { int StandardProblemIteration::info = 10; + void StandardProblemIteration::beginIteration(AdaptInfo *adaptInfo) { FUNCNAME("StandardProblemIteration::beginIteration()"); @@ -29,6 +30,7 @@ namespace AMDiS { INFO(info, 4)("=============================\n"); } + Flag StandardProblemIteration::oneIteration(AdaptInfo *adaptInfo, Flag toDo) { FUNCNAME("StandardProblemIteration::oneIteration()"); @@ -47,6 +49,7 @@ namespace AMDiS { return flag; } + void StandardProblemIteration::endIteration(AdaptInfo *adaptInfo) { FUNCNAME("StandardProblemIteration::endIteration()"); @@ -56,6 +59,7 @@ namespace AMDiS { INFO(info, 4)("=============================\n"); } + Flag StandardProblemIteration::buildAndAdapt(AdaptInfo *adaptInfo, Flag toDo) { FUNCNAME("StandardProblemIteration::buildAndAdapt()"); @@ -90,18 +94,22 @@ namespace AMDiS { return flag; } + std::string StandardProblemIteration::getName() { return problem->getName(); } + void StandardProblemIteration::serialize(std::ostream &out) { problem->serialize(out); } + void StandardProblemIteration::deserialize(std::istream &in) { problem->deserialize(in); } + }; diff --git a/AMDiS/src/parallel/MeshDistributor.cc b/AMDiS/src/parallel/MeshDistributor.cc index 7dbf208172d89a670419c5c1038115f6e7f2ec6a..6235c1680de24b6c8b3b420ee02d6cd366497bb3 100644 --- a/AMDiS/src/parallel/MeshDistributor.cc +++ b/AMDiS/src/parallel/MeshDistributor.cc @@ -85,8 +85,11 @@ namespace AMDiS { repartitioningAllowed = (tmp > 0); GET_PARAMETER(0, name + "->debug output dir", &debugOutputDir); - GET_PARAMETER(0, name + "->repartition ith change", "%d", &repartitionIthChange); + + tmp = 0; + GET_PARAMETER(0, name + "->log main rank", "%d", &tmp); + Msg::outputMainRank = (tmp > 0); } @@ -146,7 +149,10 @@ namespace AMDiS { // and now partition the mesh partitioner->fillCoarsePartitionVec(&oldPartitionVec); - partitioner->partition(elemWeights, INITIAL); + + bool partitioningSucceed = partitioner->partition(elemWeights, INITIAL); + TEST_EXIT(partitioningSucceed)("Initial partitioning does not work!\n"); + partitioner->fillCoarsePartitionVec(&partitionVec); @@ -509,6 +515,8 @@ namespace AMDiS { debug::writeMesh(feSpace, -1, debugOutputDir + "before_check_mesh"); #endif + double first = MPI::Wtime(); + // === If mesh has not been changed on all ranks, return. === int recvAllValues = 0; @@ -520,9 +528,7 @@ namespace AMDiS { // === At least one rank mesh has been changed, so the boundaries must be === // === adapted to the new mesh structure. === - - clock_t first = clock(); - + do { // To check the interior boundaries, the ownership of the boundaries is not // important. Therefore, we add all boundaries to one boundary container. @@ -566,9 +572,6 @@ namespace AMDiS { debug::writeMesh(feSpace, -1, debugOutputDir + "mesh"); #endif - INFO(info, 8)("Parallel mesh adaption needed %.5f seconds\n", - TIME_USED(first, clock())); - // === Because the mesh has been changed, update the DOF numbering and mappings. === updateLocalGlobalNumbering(); @@ -579,6 +582,11 @@ namespace AMDiS { createPeriodicMap(); + + INFO(info, 8)("Parallel mesh adaption needed %.5f seconds\n", + MPI::Wtime() - first); + + // === The mesh has changed, so check if it is required to repartition the mesh. === nTimestepsAfterLastRepartitioning++; @@ -1003,6 +1011,8 @@ namespace AMDiS { if (repartitioning == 0) return; + double timePoint = MPI::Wtime(); + #if (DEBUG != 0) ParallelDebug::testDoubleDofs(mesh); @@ -1016,8 +1026,6 @@ namespace AMDiS { repartCounter++; } - - MSG("USED-SIZE A: %d\n", mesh->getDofAdmin(0).getUsedDofs()); #endif @@ -1031,8 +1039,15 @@ namespace AMDiS { } partitioner->useLocalGlobalDofMap(&mapLocalGlobalDofs); + bool partitioningSucceed = + partitioner->partition(elemWeights, ADAPTIVE_REPART, 1000.0); + + if (!partitioningSucceed) { + MSG("ParMETIS created empty partition!\n"); + return; + } + oldPartitionVec = partitionVec; - partitioner->partition(elemWeights, ADAPTIVE_REPART, 1000.0); // === Create map that maps macro element indices to pointers to the === @@ -1209,8 +1224,6 @@ namespace AMDiS { VtkWriter::writeFile(&tmpa, oss.str()); repartCounter++; - MSG("USED-SIZE B: %d\n", mesh->getDofAdmin(0).getUsedDofs()); - ParallelDebug::testAllElements(*this); ParallelDebug::testDoubleDofs(mesh); #endif @@ -1230,13 +1243,12 @@ namespace AMDiS { ParallelDebug::testAllElements(*this); ParallelDebug::testInteriorBoundary(*this); - - debug::writeMesh(feSpace, -1, debugOutputDir + "macro_mesh"); + ParallelDebug::printBoundaryInfo(*this); MSG("Debug mode tests finished!\n"); #endif - ParallelDebug::printBoundaryInfo(*this); + MSG("Mesh repartitioning needed %.5f seconds\n", MPI::Wtime() - timePoint); } diff --git a/AMDiS/src/parallel/ParMetisPartitioner.cc b/AMDiS/src/parallel/ParMetisPartitioner.cc index 1234739ea2c8e55a9dec391d8b5c9f74325e9f1d..f8790db344a64159a592664bc9ad74143bd35629 100644 --- a/AMDiS/src/parallel/ParMetisPartitioner.cc +++ b/AMDiS/src/parallel/ParMetisPartitioner.cc @@ -252,7 +252,7 @@ namespace AMDiS { } - void ParMetisPartitioner::partition(std::map<int, double> &elemWeights, + bool ParMetisPartitioner::partition(std::map<int, double> &elemWeights, PartitionMode mode, float itr) { @@ -331,22 +331,8 @@ namespace AMDiS { // === Scale element weights. === - double weightSum = 0.0; - - for (int i = 0; i < nElements; i++) { + for (int i = 0; i < nElements; i++) wgts[i] = static_cast<int>(floatWgts[i] * scale); - weightSum += wgts[i]; - } - - mpi::globalAdd(weightSum); - weightSum /= mpiSize; - - for (int i = 0; i < nElements; i++) { - if (wgts[i] > weightSum) { - MSG("ICH HABE EINEN %d > %f\n", wgts[i], weightSum); - wgts[i] = static_cast<int>(weightSum); - } - } // === Start ParMETIS. === @@ -421,7 +407,7 @@ namespace AMDiS { // === Distribute new partition data. === - distributePartitioning(&(part[0])); + return distributePartitioning(&(part[0])); } @@ -468,7 +454,7 @@ namespace AMDiS { } - void ParMetisPartitioner::distributePartitioning(int *part) + bool ParMetisPartitioner::distributePartitioning(int *part) { FUNCNAME("ParMetisPartitioner::distributePartitioning()"); @@ -492,7 +478,15 @@ namespace AMDiS { int *sumPartitionElements = new int[mpiSize]; mpiComm->Allreduce(nPartitionElements, sumPartitionElements, mpiSize, MPI_INT, MPI_SUM); - + + // Test if there exists an empty partition + bool emptyPartition = false; + for (int i = 0; i < mpiSize; i++) + emptyPartition |= (sumPartitionElements[i] == 0); + + if (emptyPartition) + return false; + // prepare distribution (fill partitionElements with AMDiS indices) int *bufferOffset = new int[mpiSize]; bufferOffset[0] = 0; @@ -562,6 +556,8 @@ namespace AMDiS { delete [] partitionPtr; delete [] bufferOffset; delete [] recvBufferOffset; + + return true; } } diff --git a/AMDiS/src/parallel/ParMetisPartitioner.h b/AMDiS/src/parallel/ParMetisPartitioner.h index 8fe6254115f09c09968e15bebdbd47672686d22b..11c713d4307a2eed9f158c4280299f72395a8547 100644 --- a/AMDiS/src/parallel/ParMetisPartitioner.h +++ b/AMDiS/src/parallel/ParMetisPartitioner.h @@ -181,7 +181,7 @@ namespace AMDiS { ~ParMetisPartitioner(); - void partition(std::map<int, double> &elemWeights, + bool partition(std::map<int, double> &elemWeights, PartitionMode mode = INITIAL, float itr = 1000000.0); @@ -215,7 +215,9 @@ namespace AMDiS { } protected: - void distributePartitioning(int *part); + // Returns true, if the mesh partitioning could be distributed. If this is not the + // case, i.e., because there are empty partitions, the function returns false. + bool distributePartitioning(int *part); protected: Mesh *mesh;