From a2911e8dee0f2d7a626fbac62b5b9a5e6b78e90b Mon Sep 17 00:00:00 2001
From: Thomas Witkowski <thomas.witkowski@gmx.de>
Date: Fri, 18 Jun 2010 11:47:03 +0000
Subject: [PATCH] Several small changed to simplify parallel code.
---
AMDiS/src/ElementFileWriter.cc | 3 -
AMDiS/src/parallel/GlobalMatrixSolver.cc | 17 +++--
AMDiS/src/parallel/ParallelDomainBase.cc | 90 ++++++++----------------
3 files changed, 41 insertions(+), 69 deletions(-)
diff --git a/AMDiS/src/ElementFileWriter.cc b/AMDiS/src/ElementFileWriter.cc
index 188840cc..8b6fb373 100644
--- a/AMDiS/src/ElementFileWriter.cc
+++ b/AMDiS/src/ElementFileWriter.cc
@@ -372,9 +372,6 @@ namespace AMDiS {
while (elInfo) {
// Get element value.
double val = vec[elInfo->getElement()->getIndex()];
- if (vc == 1101) {
- MSG("FOUND EL %d\n", elInfo->getElement()->getIndex());
- }
// Write value for each vertex of each element.
for (int i = 0; i <= dim; i++)
diff --git a/AMDiS/src/parallel/GlobalMatrixSolver.cc b/AMDiS/src/parallel/GlobalMatrixSolver.cc
index 5616bf83..b66e6639 100644
--- a/AMDiS/src/parallel/GlobalMatrixSolver.cc
+++ b/AMDiS/src/parallel/GlobalMatrixSolver.cc
@@ -197,7 +197,8 @@ namespace AMDiS {
DOFVector<double>::Iterator dofIt(vec, USED_DOFS);
for (dofIt.reset(); !dofIt.end(); ++dofIt) {
// Calculate global row index of the dof.
- DegreeOfFreedom globalRow = meshDistributor->mapLocalToGlobal(dofIt.getDOFIndex());
+ DegreeOfFreedom globalRow =
+ meshDistributor->mapLocalToGlobal(dofIt.getDOFIndex());
// Calculate petsc index of the row dof.
int index = globalRow * dispMult + dispAdd;
@@ -384,6 +385,8 @@ namespace AMDiS {
{
FUNCNAME("GlobalMatrixSolver::fillPetscMatrix()");
+ MSG("START FILL PETSC MATRIX!\n");
+
clock_t first = clock();
int nRankRows = meshDistributor->getNumberRankDofs() * nComponents;
int nOverallRows = meshDistributor->getNumberOverallDofs() * nComponents;
@@ -402,7 +405,11 @@ namespace AMDiS {
VecSetSizes(petscTmpVec, nRankRows, nOverallRows);
VecSetType(petscTmpVec, VECMPI);
- if (!d_nnz || meshDistributor->getLastMeshChangeIndex() != lastMeshNnz) {
+ int recvAllValues = 0;
+ int sendValue = static_cast<int>(meshDistributor->getLastMeshChangeIndex() != lastMeshNnz);
+ meshDistributor->getMpiComm().Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM);
+
+ if (!d_nnz || recvAllValues != 0) {
if (d_nnz) {
delete [] d_nnz;
d_nnz = NULL;
@@ -430,6 +437,7 @@ namespace AMDiS {
("Wrong matrix ownership range!\n");
#endif
+
// === Transfer values from DOF matrices to the PETSc matrix. ===
for (int i = 0; i < nComponents; i++)
@@ -442,6 +450,7 @@ namespace AMDiS {
MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(petscMatrix, MAT_FINAL_ASSEMBLY);
+
// === Transfer values from DOF vector to the PETSc vector. ===
for (int i = 0; i < nComponents; i++)
@@ -496,9 +505,9 @@ namespace AMDiS {
int nRankDofs = meshDistributor->getNumberRankDofs();
for (int i = 0; i < nComponents; i++) {
- DOFVector<double> *dofvec = vec.getDOFVector(i);
+ DOFVector<double> &dofvec = *(vec.getDOFVector(i));
for (int j = 0; j < nRankDofs; j++)
- (*dofvec)[meshDistributor->mapLocalToDofIndex(j)] =
+ dofvec[meshDistributor->mapLocalToDofIndex(j)] =
vecPointer[j * nComponents + i];
}
diff --git a/AMDiS/src/parallel/ParallelDomainBase.cc b/AMDiS/src/parallel/ParallelDomainBase.cc
index a7ddc962..91719195 100644
--- a/AMDiS/src/parallel/ParallelDomainBase.cc
+++ b/AMDiS/src/parallel/ParallelDomainBase.cc
@@ -1582,11 +1582,11 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo);
}
- DofContainer rankAllDofs;
- for (DofSet::iterator dofIt = rankDofSet.begin(); dofIt != rankDofSet.end(); ++dofIt)
- rankAllDofs.push_back(*dofIt);
- sort(rankAllDofs.begin(), rankAllDofs.end(), cmpDofsByValue);
- DofContainer rankDofs = rankAllDofs;
+ DofContainer rankDofs;
+ for (DofSet::iterator it = rankDofSet.begin(); it != rankDofSet.end(); ++it)
+ rankDofs.push_back(*it);
+ sort(rankDofs.begin(), rankDofs.end(), cmpDofsByValue);
+ int nRankAllDofs = rankDofs.size();
// === Traverse on interior boundaries and move all not ranked owned DOFs from ===
@@ -1627,20 +1627,8 @@ namespace AMDiS {
it->rankObj.el->getVertexDofs(feSpace, it->rankObj, dofs);
it->rankObj.el->getNonVertexDofs(feSpace, it->rankObj, dofs);
- for (int i = 0; i < static_cast<int>(dofs.size()); i++) {
- // FOR DEBUGGING
- /*
- WorldVector<double> cs;
- mesh->getDofIndexCoords(dofs[i], feSpace, cs);
- MSG("SEND EL %d DOF %d TO %d\n", it->rankObj.elIndex, *(dofs[i]), it.getRank());
- if (cs.getSize() == 2)
- MSG("COORDS-s2: %f %f\n", cs[0], cs[1]);
- else
- MSG("COORDS-s2: %f %f %f\n", cs[0], cs[1], cs[2]);
- */
-
- sendDofs[it.getRank()].push_back(dofs[i]);
- }
+ for (int i = 0; i < static_cast<int>(dofs.size()); i++)
+ sendDofs[it.getRank()].push_back(dofs[i]);
}
@@ -1655,17 +1643,6 @@ namespace AMDiS {
if (eraseIt != rankDofs.end())
rankDofs.erase(eraseIt);
- // FOR DEBUGGING
- /*
- WorldVector<double> cs;
- mesh->getDofIndexCoords(dofs[i], feSpace, cs);
- MSG("RECV EL %d DOF %d FROM %d\n", it->rankObj.elIndex, *(dofs[i]), it.getRank());
- if (cs.getSize() == 2)
- MSG("COORDS-r2: %f %f\n", cs[0], cs[1]);
- else
- MSG("COORDS-r2: %f %f %f\n", cs[0], cs[1], cs[2]);
- */
-
recvDofs[it.getRank()].push_back(dofs[i]);
}
}
@@ -1684,40 +1661,24 @@ namespace AMDiS {
nOverallDofs = 0;
mpiComm.Allreduce(&nRankDofs, &nOverallDofs, 1, MPI_INT, MPI_SUM);
-
- // Do not change the indices now, but create a new indexing and store it here.
- DofIndexMap rankDofsNewLocalIndex;
+ // First, we set all dofs in ranks partition to be owend by the rank. Later,
+ // the dofs in ranks partition that are owned by other rank are set to false.
isRankDof.clear();
- int i = 0;
- for (DofContainer::iterator dofIt = rankAllDofs.begin();
- dofIt != rankAllDofs.end(); ++dofIt) {
- rankDofsNewLocalIndex[*dofIt] = i;
- // First, we set all dofs in ranks partition to be owend by the rank. Later,
- // the dofs in ranks partition that are owned by other rank are set to false.
+ for (int i = 0; i < nRankAllDofs; i++)
isRankDof[i] = true;
- i++;
- }
// Stores for all rank owned dofs a new global index.
DofIndexMap rankDofsNewGlobalIndex;
- // Stores for all rank owned dofs a continues local index.
- DofIndexMap rankOwnedDofsNewLocalIndex;
-
- i = 0;
- for (DofContainer::iterator dofIt = rankDofs.begin();
- dofIt != rankDofs.end(); ++dofIt) {
- rankDofsNewGlobalIndex[*dofIt] = i + rstart;
- rankOwnedDofsNewLocalIndex[*dofIt] = i;
- i++;
- }
-
+ for (int i = 0; i < nRankDofs; i++)
+ rankDofsNewGlobalIndex[rankDofs[i]] = i + rstart;
// === Send new DOF indices. ===
#if (DEBUG != 0)
ParallelDomainDbg::testDofContainerCommunication(*this, sendDofs, recvDofs);
#endif
-
+
+ int i = 0;
StdMpi<std::vector<DegreeOfFreedom> > stdMpi(mpiComm, false);
for (RankToDofContainer::iterator sendIt = sendDofs.begin();
sendIt != sendDofs.end(); ++sendIt, i++) {
@@ -1738,15 +1699,23 @@ namespace AMDiS {
for (DofContainer::iterator dofIt = recvIt->second.begin();
dofIt != recvIt->second.end(); ++dofIt) {
rankDofsNewGlobalIndex[*dofIt] = stdMpi.getRecvData(recvIt->first)[j++];
- isRankDof[rankDofsNewLocalIndex[*dofIt]] = false;
+ isRankDof[**dofIt] = false;
}
}
// === Create now the local to global index and local to dof index mappings. ===
- createLocalMappings(rankDofsNewLocalIndex, rankOwnedDofsNewLocalIndex,
- rankDofsNewGlobalIndex);
+ mapLocalGlobalDofs.clear();
+ mapLocalDofIndex.clear();
+
+ for (DofIndexMap::iterator dofIt = rankDofsNewGlobalIndex.begin();
+ dofIt != rankDofsNewGlobalIndex.end(); ++dofIt)
+ mapLocalGlobalDofs[*(dofIt->first)] = dofIt->second;
+
+ for (int i = 0; i < nRankDofs; i++)
+ mapLocalDofIndex[i] = *(rankDofs[i]);
+
// === Update dof admins due to new number of dofs. ===
@@ -1787,11 +1756,6 @@ namespace AMDiS {
vertexDof[*it],
isRankDof[**it]);
}
- MSG("\n");
- for (DofMapping::iterator it = mapLocalDofIndex.begin();
- it != mapLocalDofIndex.end(); ++it) {
- MSG("mapLocalDofIndex[%d] = %d\n", it->first, it->second);
- }
#endif
#endif
}
@@ -1827,6 +1791,7 @@ namespace AMDiS {
mapLocalGlobalDofs.clear();
mapLocalDofIndex.clear();
+
// Iterate over all DOFs in ranks partition.
for (DofIndexMap::iterator dofIt = rankDofsNewLocalIndex.begin();
dofIt != rankDofsNewLocalIndex.end(); ++dofIt) {
@@ -1837,12 +1802,13 @@ namespace AMDiS {
mapLocalGlobalDofs[newLocalIndex] = newGlobalIndex;
}
+
for (DofIndexMap::iterator dofIt = rankOwnedDofsNewLocalIndex.begin();
dofIt != rankOwnedDofsNewLocalIndex.end(); ++dofIt)
mapLocalDofIndex[dofIt->second] = *(dofIt->first);
}
-
+
void MeshDistributor::createDofMemberInfo(DofToPartitions& partitionDofs,
DofContainer& rankOwnedDofs,
DofContainer& rankAllDofs,
--
GitLab