From b227d4cf9507678f549840bc913e8a391dbe096b Mon Sep 17 00:00:00 2001
From: Sebastian Reuther <sebastian.reuther@tu-dresden.de>
Date: Mon, 16 Nov 2015 09:41:34 +0000
Subject: [PATCH] added more synch methods in MeshDistributor
---
AMDiS/src/parallel/MeshDistributor.h | 141 +++++++++++++++++++++++++++
1 file changed, 141 insertions(+)
diff --git a/AMDiS/src/parallel/MeshDistributor.h b/AMDiS/src/parallel/MeshDistributor.h
index 109e7f54..7e76039f 100644
--- a/AMDiS/src/parallel/MeshDistributor.h
+++ b/AMDiS/src/parallel/MeshDistributor.h
@@ -282,6 +282,147 @@ namespace AMDiS { namespace Parallel {
synchVector(vec);
}
+ /** \brief
+ * This function must be used if the values of a set of DOFVectors must be
+ * synchronized over all ranks. That means, that each rank sends the
+ * values of the DOFs, which are owned by the rank and lie on an interior
+ * boundary, to all other ranks also having these DOFs.
+ *
+ * The synchronization direction is from major to minor rank. This means
+ * that the value of the rank with the higher number sends its value
+ * to the rank with the lower number.
+ */
+ // majorRank => minorRank
+ template<typename T>
+ void synchVector(std::vector<DOFVector<T>*> &vecs)
+ {
+ if (vecs.size() > 0)
+ {
+ // get FE space
+ const FiniteElemSpace *fe = vecs[0]->getFeSpace();
+ // TODO: check equal FE space
+ // The lines below do not work!
+ // for ( typename std::vector<DOFVector<T>*>::iterator vecIt = vecs.begin(); vecIt != vecs.end(); ++vecIt)
+ // TEST_EXIT( (*vecIt)->getFeSpace()->getBasisFcts()->getDegree() == fe->getBasisFcts()->getDegree() )("FE space of vectors to synch not equal!\n");
+
+ MultiLevelDofComm& dofComm = dofComms[fe->getMesh()];
+
+ int nLevels = levelData.getNumberOfLevels();
+ for (int level = nLevels - 1; level >= 0; level--)
+ {
+ StdMpi<std::vector<std::vector<T> > > stdMpi(levelData.getMpiComm(level));
+
+ for (DofComm::Iterator it(dofComm[level].getSendDofs(), fe); !it.end(); it.nextRank())
+ {
+ std::vector<std::vector<T> > dofs;
+ dofs.reserve(it.getDofs().size());
+ for (; !it.endDofIter(); it.nextDof())
+ {
+ std::vector<T> values;
+ values.reserve(vecs.size());
+ for (typename std::vector<DOFVector<T>*>::iterator vecIt = vecs.begin(); vecIt != vecs.end(); ++vecIt )
+ values.push_back((**vecIt)[it.getDofIndex()]);
+ dofs.push_back(values);
+ }
+ stdMpi.send(it.getRank(), dofs);
+ }
+
+ for (DofComm::Iterator it(dofComm[level].getRecvDofs(), fe); !it.end(); it.nextRank())
+ stdMpi.recv(it.getRank());
+
+ stdMpi.startCommunication();
+
+ for (DofComm::Iterator it(dofComm[level].getRecvDofs(), fe); !it.end(); it.nextRank())
+ {
+ for (; !it.endDofIter(); it.nextDof())
+ {
+ std::vector<T> values = stdMpi.getRecvData(it.getRank())[it.getDofCounter()];
+ typename std::vector<DOFVector<T>*>::iterator vecIt = vecs.begin();
+ typename std::vector<T>::iterator valuesIt = values.begin();
+ for (; vecIt != vecs.end(), valuesIt != values.end(); ++vecIt , ++valuesIt)
+ (**vecIt)[it.getDofIndex()] = *valuesIt;
+ }
+ }
+ }
+ }
+ }
+
+ /** \brief
+ * Works quite similar to the function \ref synchVector with an operator/
+ * assigner for the values on the subdomain boundaries of the DOFVector vec.
+ * Additionally, the values stored in additionalVecs are synchronized in
+ * the same way (direction (minor to major or major to minor rank)) as the
+ * DOFs of the variable vec.
+ */
+ template<typename T, typename Operator>
+ void synchVectorSameWay(DOFVector<T> &vec, std::vector<DOFVector<T>*> additionalVecs, Operator op)
+ {
+ // get FE space and check equal FE space
+ const FiniteElemSpace *fe = vec.getFeSpace();
+ // TODO: check equal FE space
+ // The lines below do not work!
+ // for ( typename std::vector<DOFVector<T>*>::iterator vecIt = additionalVecs.begin(); vecIt != additionalVecs.end(); ++vecIt)
+ // TEST_EXIT( (*vecIt)->getFeSpace()->getBasisFcts()->getDegree() == fe->getBasisFcts()->getDegree() )("FE space of vectors to synch not equal!\n");
+
+ MultiLevelDofComm& dofComm = dofComms[fe->getMesh()];
+
+ int nLevels = levelData.getNumberOfLevels();
+ for (int level = nLevels - 1; level >= 0; level--)
+ {
+ StdMpi < std::vector<std::vector<T> > > stdMpi(levelData.getMpiComm(level));
+
+ for (DofComm::Iterator it(dofComm[level].getRecvDofs(), fe); !it.end(); it.nextRank())
+ {
+ std::vector<std::vector<T> > dofs;
+ dofs.reserve(it.getDofs().size());
+ for (; !it.endDofIter(); it.nextDof())
+ {
+ std::vector<T> values;
+ values.reserve(additionalVecs.size() + 1);
+ values.push_back( vec[it.getDofIndex()] );
+ for (typename std::vector<DOFVector<T>*>::iterator vecIt = additionalVecs.begin(); vecIt != additionalVecs.end(); ++vecIt )
+ values.push_back( (**vecIt)[it.getDofIndex()] );
+ dofs.push_back( values );
+ }
+ stdMpi.send(it.getRank(), dofs);
+ }
+
+ for (DofComm::Iterator it(dofComm[level].getSendDofs(), fe); !it.end(); it.nextRank())
+ stdMpi.recv(it.getRank());
+
+ stdMpi.startCommunication();
+
+ for (DofComm::Iterator it(dofComm[level].getSendDofs(), fe); !it.end(); it.nextRank())
+ {
+ for (; !it.endDofIter(); it.nextDof())
+ {
+ DegreeOfFreedom idx = it.getDofIndex();
+ std::vector<T> values = stdMpi.getRecvData(it.getRank())[it.getDofCounter()];
+
+ T minorRankValue = vec[idx];
+ T majorRankValue = values[0];
+ op(vec[idx], values[0]);
+ T synchValue = vec[idx];
+
+ TEST_EXIT(additionalVecs.size() == values.size()-1)("The number of additional vectors and the received values do not match!\n");
+
+ typename std::vector<DOFVector<T>*>::iterator vecIt = additionalVecs.begin();
+ typename std::vector<T>::iterator valuesIt = values.begin();
+ ++valuesIt; // exclude the first one since it belongs to the variable vec
+ for (; vecIt != additionalVecs.end(), valuesIt != values.end(); ++vecIt , ++valuesIt)
+ if (synchValue == majorRankValue)
+ (**vecIt)[idx] = *valuesIt;
+ }
+ }
+ }
+ // call simple sync method
+ std::vector<DOFVector<T>*> allDOFVectors;
+ allDOFVectors.push_back(&vec);
+ for ( int i = 0; i < additionalVecs.size(); ++i )
+ allDOFVectors.push_back(additionalVecs[i]);
+ synchVector(allDOFVectors);
+ }
+
/** \brief
* This function must be used if the values of a DOFVector must be
* synchronised over all ranks. That means, that each rank sends the
--
GitLab