diff --git a/AMDiS/src/AMDiS.cc b/AMDiS/src/AMDiS.cc index 017544ee60bdbe83f6ade0704487e0cafa5c2ff2..1448c59b9f7a4ba4510704006110cd25521e1784 100644 --- a/AMDiS/src/AMDiS.cc +++ b/AMDiS/src/AMDiS.cc @@ -56,7 +56,7 @@ namespace AMDiS { #ifdef HAVE_PARALLEL_DOMAIN_AMDIS MeshDistributor::globalMeshDistributor->exitParallelization(); #ifdef HAVE_PARALLEL_MTL4 - if ( mtl_environment ) + if (mtl_environment) delete mtl_environment; #else PetscFinalize(); diff --git a/AMDiS/src/parallel/MeshDistributor.cc b/AMDiS/src/parallel/MeshDistributor.cc index e894e2a5f7b418b454f602c692555c6f3981484a..04cfb71d202fcddb5d2fa0efc2371377cc337f26 100644 --- a/AMDiS/src/parallel/MeshDistributor.cc +++ b/AMDiS/src/parallel/MeshDistributor.cc @@ -906,8 +906,12 @@ namespace AMDiS { stdMpi.send(sendCodes); for (RankToBoundMap::iterator it = allBound.begin(); it != allBound.end(); ++it) stdMpi.recv(it->first); - + + MSG("DA 1\n"); + stdMpi.startCommunication(); + + MSG("DA 2\n"); // === Compare received mesh structure codes. === diff --git a/AMDiS/src/parallel/StdMpi.h b/AMDiS/src/parallel/StdMpi.h index de677c86409731c97120b6a26532c1cafe07a4cf..0dbcb898131a7315051694d67ad43a40f81d17e5 100644 --- a/AMDiS/src/parallel/StdMpi.h +++ b/AMDiS/src/parallel/StdMpi.h @@ -386,12 +386,17 @@ namespace AMDiS { FUNCNAME("StdMpi::commDataSize()"); MPI::Request request[sendData.size() + recvDataSize.size()]; - vector<int> sendBuffers, recvBuffers; + MPI::Status status[sendData.size() + recvDataSize.size()]; + + vector<int> sendBuffers; + sendBuffers.resize(sendDataSize.size()); + int requestCounter = 0; for (typename map<int, int>::iterator sendIt = sendDataSize.begin(); sendIt != sendDataSize.end(); ++sendIt) { - sendBuffers.push_back(sendIt->second); + sendBuffers[requestCounter] = sendIt->second; + request[requestCounter] = mpiComm.Isend(&(sendBuffers[requestCounter]), 1, MPI_INT, sendIt->first, 0); @@ -399,11 +404,18 @@ namespace AMDiS { } for (map<int, int>::iterator recvIt = recvDataSize.begin(); - recvIt != recvDataSize.end(); ++recvIt) - request[requestCounter++] = - mpiComm.Irecv(&(recvIt->second), 1, MPI_INT, recvIt->first, 0); - - MPI::Request::Waitall(requestCounter, request); + recvIt != recvDataSize.end(); ++recvIt) { + request[requestCounter] = + mpiComm.Irecv(&(recvIt->second), 1, MPI_INT, recvIt->first, 0); + + requestCounter++; + } + + MPI::Request::Waitall(requestCounter, request, status); + +/* for (int j = 0; j < requestCounter; j++) */ +/* if (status[j].Get_error() != MPI::SUCCESS) */ +/* MSG("MPI ERROR IN COMM: %d\n", status[j].Get_error()); */ } @@ -421,6 +433,38 @@ namespace AMDiS { if (exchangeDataSize) commDataSize(); + + // === Remove empty data communication. === + + { + map<int, int>::iterator it = sendDataSize.begin(); + while (it != sendDataSize.end()) { + TEST_EXIT_DBG(it->second >= 0)("Should not happen!\n"); + + if (it->second == 0) { + sendData.erase(it->first); + sendDataSize.erase(it++); + } else + ++it; + } + } + + { + map<int, int>::iterator it = recvDataSize.begin(); + while (it != recvDataSize.end()) { + TEST_EXIT_DBG(it->second >= 0)("Should not happen!\n"); + + if (it->second == 0) { + recvData.erase(it->first); + recvDataSize.erase(it++); + } else + ++it; + } + } + + + // === Start communication. === + MPI::Request request[sendData.size() + recvDataSize.size()]; int requestCounter = 0; vector<cppDataType*> sendBuffers, recvBuffers; diff --git a/AMDiS/src/parallel/ZoltanPartitioner.cc b/AMDiS/src/parallel/ZoltanPartitioner.cc index 6053565b9e8030ee779f54eb45f7b5e01b435627..77ba7f9b4f1bd4a3f788d802132c536346d24c07 100644 --- a/AMDiS/src/parallel/ZoltanPartitioner.cc +++ b/AMDiS/src/parallel/ZoltanPartitioner.cc @@ -81,6 +81,7 @@ namespace AMDiS { zoltan.Set_Param("DEBUG_LEVEL", "0"); #endif + int err = zoltan.LB_Partition(changes, nGid, nLid, nImportEls, import_global_ids,