Setup for multilevel parallel dof mapping.

AMDiS/src/parallel/MeshDistributor.cc

@@ -1200,7 +1200,7 @@ namespace AMDiS {
 
     // === Run mesh partitioner to calculate a new mesh partitioning.  ===
 
-    partitioner->setLocalGlobalDofMap(&(dofMap[feSpaces[0]].getMap()));
+    partitioner->setLocalGlobalDofMap(&(dofMap[feSpaces[0]].getMap(0)));
 
     bool partitioningSucceed = 
       partitioner->partition(elemWeights, ADAPTIVE_REPART);
@@ -1979,9 +1979,9 @@ namespace AMDiS {
     MSG("------------- Debug information -------------\n");
     for (unsigned int i = 0; i < feSpaces.size(); i++) {
       MSG("FE space %d:\n", i);
-      MSG("   nRankDofs = %d\n", dofMap[feSpaces[i]].nRankDofs);
-      MSG("   nOverallDofs = %d\n", dofMap[feSpaces[i]].nOverallDofs);
-      MSG("   rStartDofs = %d\n", dofMap[feSpaces[i]].rStartDofs);
+      MSG("   nRankDofs = %d\n", dofMap[feSpaces[i]].nRankDofs[0]);
+      MSG("   nOverallDofs = %d\n", dofMap[feSpaces[i]].nOverallDofs[0]);
+      MSG("   rStartDofs = %d\n", dofMap[feSpaces[i]].rStartDofs[0]);
     }
 
     stringstream oss;
@@ -2259,20 +2259,6 @@ namespace AMDiS {
   }
 
 
-  DegreeOfFreedom MeshDistributor::mapGlobalToLocal(const FiniteElemSpace *feSpace,
-						    DegreeOfFreedom dof)
-  {
-    FUNCNAME("MeshDistributor::mapGlobalToLocal()");
-
-    for (DofMap::iterator it = dofMap[feSpace].getMap().begin();
-	 it != dofMap[feSpace].getMap().end(); ++it) 
-      if (it->second.global == dof)
-	return it->first;
-
-    return -1;
-  }
-
-
   void MeshDistributor::serialize(ostream &out)
   {
     FUNCNAME("MeshDistributor::serialize()");

AMDiS/src/parallel/MeshDistributor.h

@@ -157,13 +157,6 @@ namespace AMDiS {
       return periodicMap;
     }
 
-    /// Returns for a global index the DOF index in rank's subdomain. As there
-    /// is no direct data structure that stores this information, we have to
-    /// search for it in \ref dofMap. This is not very efficient and this 
-    /// function should thus be used for debugging only.
-    DegreeOfFreedom mapGlobalToLocal(const FiniteElemSpace *feSpace,
-				     DegreeOfFreedom dof);
-
     DofComm& getSendDofs()
     {
       return sendDofs;

AMDiS/src/parallel/ParallelDebug.cc

@@ -634,7 +634,12 @@ namespace AMDiS {
 
 
   void ParallelDebug::printMapLocalGlobal(MeshDistributor &pdb, int rank)
-  {    
+  {   
+    FUNCNAME("ParallelDebug::printMapLocalGlobal()");
+
+    ERROR_EXIT("Rewrite this function!\n");
+
+#if 0
     if (rank == -1 || pdb.mpiRank == rank) {
       const FiniteElemSpace *feSpace = pdb.feSpaces[0];
 
@@ -662,6 +667,7 @@ namespace AMDiS {
 	cout << "------" << endl;
       }
     }
+#endif
   }
 
 

AMDiS/src/parallel/ParallelDofMapping.cc

@@ -20,47 +20,61 @@ namespace AMDiS {
   void FeSpaceDofMap::clear()
   {
     dofMap.clear();
+    dofMap.resize(nLevel);
+
     nonRankDofs.clear();
-    nRankDofs = 0;
-    nLocalDofs = 0;
-    nOverallDofs = 0;
-    rStartDofs = 0;
+    nonRankDofs.resize(nLevel);
+
+    for (int i = 0; i < nLevel; i++) {
+      nRankDofs[i] = 0;
+      nLocalDofs[i] = 0;
+      nOverallDofs[i] = 0;
+      rStartDofs[i] = 0;
+    }
   }
 
 
   void FeSpaceDofMap::update()
   {
-    // === Compute local indices for all rank owned DOFs. ===
+    FUNCNAME("FeSpaceDofMap::update()");
 
-    for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
-      if (it->second.local == -1 && nonRankDofs.count(it->first) == 0)
-	it->second.local = nRankDofs++;
-        
-    // === Compute number of local and global DOFs in the mapping. ===
-
-    nOverallDofs = 0;
-    rStartDofs = 0;
-    mpi::getDofNumbering(mpiComm, nRankDofs, rStartDofs, nOverallDofs);
-    
-    // === If required, compute also the global indices. ===
+    for (int i = 0; i < nLevel; i++) {
 
-    if (needGlobalMapping) {
-      computeGlobalMapping();
+      // === Compute local indices for all rank owned DOFs. ===
       
-      if (hasNonLocalDofs)
-	computeNonLocalIndices();
+      for (DofMap::iterator it = dofMap[i].begin(); it != dofMap[i].end(); ++it)
+	if (it->second.local == -1 && nonRankDofs[i].count(it->first) == 0)
+	  it->second.local = nRankDofs[i]++;
+      
+      // === Compute number of local and global DOFs in the mapping. ===
+      
+      nOverallDofs[i] = 0;
+      rStartDofs[i] = 0;
+      mpi::getDofNumbering(mpiComm, nRankDofs[i], rStartDofs[i], nOverallDofs[i]);
+      
+      // === If required, compute also the global indices. ===
+      
+      if (needGlobalMapping) {
+	computeGlobalMapping(i);
+	
+	if (hasNonLocalDofs)
+	  computeNonLocalIndices(i);
+      }
     }
   }
 
 
-  void FeSpaceDofMap::computeGlobalMapping()
+  void FeSpaceDofMap::computeGlobalMapping(int level)
   {
-    for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
-      it->second.global = it->second.local + rStartDofs;
+    FUNCNAME("FeSpaceDofMap::computeGlobalMapping()");
+
+    for (DofMap::iterator it = dofMap[level].begin(); 
+	 it != dofMap[level].end(); ++it)
+      it->second.global = it->second.local + rStartDofs[level];
   }
 
 
-  void FeSpaceDofMap::computeNonLocalIndices()
+  void FeSpaceDofMap::computeNonLocalIndices(int level)
   {
     FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()");
 
@@ -70,20 +84,22 @@ namespace AMDiS {
     // === other ranks that also include this DOF.                           ===
 
     StdMpi<vector<int> > stdMpi(mpiComm);
-    for (DofComm::Iterator it(*sendDofs, feSpace); !it.end(); it.nextRank())
+    for (DofComm::Iterator it(*sendDofs, level, feSpace); 
+	 !it.end(); it.nextRank())
       for (; !it.endDofIter(); it.nextDof())
-	if (dofMap.count(it.getDofIndex()) && !nonRankDofs.count(it.getDofIndex()))
-	  stdMpi.getSendData(it.getRank()).push_back(dofMap[it.getDofIndex()].global);
+	if (dofMap[level].count(it.getDofIndex()) && !nonRankDofs[level].count(it.getDofIndex()))
+	  stdMpi.getSendData(it.getRank()).push_back(dofMap[level][it.getDofIndex()].global);
 
     stdMpi.updateSendDataSize();
 
 
     // === Check from which ranks this rank must receive some data. ===
 
-    for (DofComm::Iterator it(*recvDofs, feSpace); !it.end(); it.nextRank()) {
+    for (DofComm::Iterator it(*recvDofs, level, feSpace); 
+	 !it.end(); it.nextRank()) {
       bool recvFromRank = false;
       for (; !it.endDofIter(); it.nextDof()) {
-	if (nonRankDofs.count(it.getDofIndex())) {
+	if (nonRankDofs[level].count(it.getDofIndex())) {
 	  recvFromRank = true;
 	  break;
 	}
@@ -101,12 +117,12 @@ namespace AMDiS {
 
     // === And set the global indices for all DOFs that are not owned by rank. ===
     
-    for (DofComm::Iterator it(*recvDofs, feSpace);
+    for (DofComm::Iterator it(*recvDofs, level, feSpace);
 	 !it.end(); it.nextRank()) {
       int i = 0;
       for (; !it.endDofIter(); it.nextDof())
-	if (nonRankDofs.count(it.getDofIndex()))
-	  dofMap[it.getDofIndex()].global = stdMpi.getRecvData(it.getRank())[i++];
+	if (nonRankDofs[level].count(it.getDofIndex()))
+	  dofMap[level][it.getDofIndex()].global = stdMpi.getRecvData(it.getRank())[i++];
     }
   }
 
@@ -144,10 +160,12 @@ namespace AMDiS {
 	 it != feSpacesUnique.end(); ++it)
       data[*it].clear();
 
-    nRankDofs = -1;
-    nLocalDofs = -1;
-    nOverallDofs = -1;
-    rStartDofs = -1;
+    for (int i = 0; i < nLevel; i++) {
+      nRankDofs[i] = -1;
+      nLocalDofs[i] = -1;
+      nOverallDofs[i] = -1;
+      rStartDofs[i] = -1;
+    }
 
     dofToMatIndex.clear();
   }
@@ -180,56 +198,56 @@ namespace AMDiS {
   }    
 
 
-  int ParallelDofMapping::computeRankDofs()
+  int ParallelDofMapping::computeRankDofs(int level)
   {
     FUNCNAME("ParallelDofMapping::computeRankDofs()");
     
     int result = 0;
     for (unsigned int i = 0; i < feSpaces.size(); i++) {
       TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
-      result += data[feSpaces[i]].nRankDofs;
+      result += data[feSpaces[i]].nRankDofs[level];
     }
     
     return result;
   }
 
 
-  int ParallelDofMapping::computeLocalDofs()
+  int ParallelDofMapping::computeLocalDofs(int level)
   {
     FUNCNAME("ParallelDofMapping::computeLocalDofs()");
     
     int result = 0;
     for (unsigned int i = 0; i < feSpaces.size(); i++) {
       TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
-      result += data[feSpaces[i]].nLocalDofs;
+      result += data[feSpaces[i]].nLocalDofs[level];
     }
     
     return result;
   }
 
 
-  int ParallelDofMapping::computeOverallDofs()
+  int ParallelDofMapping::computeOverallDofs(int level)
   {
     FUNCNAME("ParallelDofMapping::computeOverallDofs()");
 
     int result = 0;
     for (unsigned int i = 0; i < feSpaces.size(); i++) {
       TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
-      result += data.find(feSpaces[i])->second.nOverallDofs;
+      result += data.find(feSpaces[i])->second.nOverallDofs[level];
     }
     
     return result;
   }
 
 
-  int ParallelDofMapping::computeStartDofs()
+  int ParallelDofMapping::computeStartDofs(int level)
   {
     FUNCNAME("ParallelDofMapping::computeStartDofs()");
 
     int result = 0;
     for (unsigned int i = 0; i < feSpaces.size(); i++) {
       TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
-      result += data.find(feSpaces[i])->second.rStartDofs;
+      result += data.find(feSpaces[i])->second.rStartDofs[level];
     }
     
     return result;
@@ -245,15 +263,18 @@ namespace AMDiS {
 	 it != feSpacesUnique.end(); ++it)
       data[*it].update();
     
-    // Compute all numbers from this mappings.
-    nRankDofs = computeRankDofs();
-    nLocalDofs = computeLocalDofs();
-    nOverallDofs = computeOverallDofs();
-    rStartDofs = computeStartDofs();
-    
-    // And finally, compute the matrix indices.
-    if (needMatIndex)
-      computeMatIndex(needMatIndexFromGlobal);
+    for (int i = 0; i < nLevel; i++) {
+      // Compute all numbers from this mappings.
+
+      nRankDofs[i] = computeRankDofs(i);
+      nLocalDofs[i] = computeLocalDofs(i);
+      nOverallDofs[i] = computeOverallDofs(i);
+      rStartDofs[i] = computeStartDofs(i);
+      
+      // And finally, compute the matrix indices.
+      if (needMatIndex)
+	computeMatIndex(needMatIndexFromGlobal, i);
+    }
   }
   
 
@@ -272,16 +293,18 @@ namespace AMDiS {
   }
 
   
-  void ParallelDofMapping::computeMatIndex(bool globalIndex)
+  void ParallelDofMapping::computeMatIndex(bool globalIndex, int level)
   {
     FUNCNAME("ParallelDofMapping::computeMatIndex()");
 
+    TEST_EXIT(level == 0)("Not yet implemented for non-zero level!\n");
+
     dofToMatIndex.clear();
     
     // The offset is always added to the local matrix index. The offset for the
     // DOFs in the first FE spaces is the smalled global index of a DOF that is
     // owned by the rank.
-    int offset = rStartDofs;
+    int offset = rStartDofs[level];
     
 
     // === Create the matrix indices for all component FE spaces. ===
@@ -290,7 +313,7 @@ namespace AMDiS {
 
       // Traverse all DOFs of the FE space and create for all rank owned DOFs
       // a matrix index.
-      DofMap& dofMap = data[feSpaces[i]].getMap();
+      DofMap& dofMap = data[feSpaces[i]].getMap(level);
       for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it) {
 	if (data[feSpaces[i]].isRankDof(it->first)) {
 	  int globalMatIndex = it->second.local + offset;
@@ -303,7 +326,7 @@ namespace AMDiS {
       
       // Increase the offset for the next FE space by the number of DOFs owned 
       // by the rank in the current FE space.
-      offset += data[feSpaces[i]].nRankDofs;
+      offset += data[feSpaces[i]].nRankDofs[level];
 	
       // If there are no non local DOFs, continue with the next FE space.
       if (!hasNonLocalDofs)
@@ -316,7 +339,7 @@ namespace AMDiS {
       // === interior boundaries.                                         ===
 
       StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiComm);
-      for (DofComm::Iterator it(*sendDofs, feSpaces[i]); 
+      for (DofComm::Iterator it(*sendDofs, level, feSpaces[i]); 
 	   !it.end(); it.nextRank()) {
 	vector<DegreeOfFreedom> sendGlobalDofs;
 	
@@ -330,14 +353,14 @@ namespace AMDiS {
 	stdMpi.send(it.getRank(), sendGlobalDofs);
       }
       
-      for (DofComm::Iterator it(*recvDofs, feSpaces[i]); 
+      for (DofComm::Iterator it(*recvDofs, level, feSpaces[i]); 
 	   !it.end(); it.nextRank())
 	stdMpi.recv(it.getRank());
       
       stdMpi.startCommunication();
       
       {
-	for (DofComm::Iterator it(*recvDofs, feSpaces[i]); 
+	for (DofComm::Iterator it(*recvDofs, level, feSpaces[i]); 
 	     !it.end(); it.nextRank()) {
 	  int counter = 0;
 	  for (; !it.endDofIter(); it.nextDof()) {

AMDiS/src/parallel/ParallelDofMapping.h

@@ -107,13 +107,17 @@ namespace AMDiS {
 	sendDofs(NULL),
 	recvDofs(NULL),
 	feSpace(NULL),
+	nLevel(1),
+	dofMap(1),
 	needGlobalMapping(false),
 	hasNonLocalDofs(false),
-	nRankDofs(0),
-	nLocalDofs(0),
-	nOverallDofs(0),
-	rStartDofs(0)
-    {}
+	nRankDofs(1),
+	nLocalDofs(1),
+	nOverallDofs(1),
+	rStartDofs(1)
+    {
+      clear();
+    }
 
     /// Clears all data of the mapping.
     void clear();
@@ -122,9 +126,9 @@ namespace AMDiS {
     /// global index must not be set.
     MultiIndex& operator[](DegreeOfFreedom d)
     {
-      TEST_EXIT_DBG(dofMap.count(d))("Should not happen!\n");
+      TEST_EXIT_DBG(dofMap[0].count(d))("Should not happen!\n");
 
-      return dofMap[d];
+      return dofMap[0][d];
     }
     
     /// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
@@ -133,12 +137,12 @@ namespace AMDiS {
     {
       FUNCNAME("FeSpaceDofMap::insertRankDof()");
       
-      TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
+      TEST_EXIT_DBG(dofMap[0].count(dof0) == 0)("Should not happen!\n");
       
-      dofMap[dof0].local = dof1;
-      nLocalDofs++;
+      dofMap[0][dof0].local = dof1;
+      nLocalDofs[0]++;
       if (dof1 != -1)
-	nRankDofs++;
+	nRankDofs[0]++;
     }
     
     /// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
@@ -147,17 +151,17 @@ namespace AMDiS {
     {
       FUNCNAME("FeSpaceDofMap::insert()");
       
-      TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
+      TEST_EXIT_DBG(dofMap[0].count(dof0) == 0)("Should not happen!\n");
       
-      dofMap[dof0].local = dof1;
-      nLocalDofs++;
-      nonRankDofs.insert(dof0);
+      dofMap[0][dof0].local = dof1;
+      nLocalDofs[0]++;
+      nonRankDofs[0].insert(dof0);
     }
     
     /// Checks if a given DOF is in the DOF mapping.
-    bool isSet(DegreeOfFreedom dof)
+    bool isSet(DegreeOfFreedom dof, int level = 0)
     {
-      return static_cast<bool>(dofMap.count(dof));
+      return static_cast<bool>(dofMap[level].count(dof));
     }
 
     /// Checks if a given DOF is a rank owned DOF of the DOF mapping. The DOF must
@@ -165,19 +169,19 @@ namespace AMDiS {
     /// meaningsless.
     bool isRankDof(DegreeOfFreedom dof)
     {
-      return !(static_cast<bool>(nonRankDofs.count(dof)));
+      return !(static_cast<bool>(nonRankDofs[0].count(dof)));
     }
     
     /// Returns number of DOFs in the mapping.
     unsigned int size()
     {
-      return dofMap.size();
+      return dofMap[0].size();
     }
     
     /// Returns the raw data of the mapping.
-    DofMap& getMap()
+    DofMap& getMap(int level)
     {
-      return dofMap;
+      return dofMap[level];
     }
 
     /// Recomputes the mapping.
@@ -211,11 +215,11 @@ namespace AMDiS {
 
   private:
     /// Computes a global mapping from the local one.
-    void computeGlobalMapping();
+    void computeGlobalMapping(int level);
 
     /// Computes the global indices of all DOFs in the mapping that are not owned
     /// by the rank.
-    void computeNonLocalIndices();
+    void computeNonLocalIndices(int level);
 
   private:
     /// MPI communicator object;
@@ -228,11 +232,14 @@ namespace AMDiS {
     /// correct DOF communicator in \ref sendDofs and \ref recvDofs.
     const FiniteElemSpace *feSpace;
 
+    /// Number of mesh levels.
+    int nLevel;
+
     /// Mapping data from DOF indices to local and global indices.
-    DofMap dofMap;
+    vector<DofMap> dofMap;
 
     /// Set of all DOFs that are in mapping but are not owned by the rank.
-    std::set<DegreeOfFreedom> nonRankDofs;
+    vector<std::set<DegreeOfFreedom> > nonRankDofs;
 
     /// If true, a global index mapping will be computed for all DOFs.
     bool needGlobalMapping;
@@ -241,9 +248,10 @@ namespace AMDiS {
     /// by the rank. If the value is false, each rank contains only DOFs that
     /// are also owned by this rank.
     bool hasNonLocalDofs;
+
   public:
     /// 
-    int nRankDofs, nLocalDofs, nOverallDofs, rStartDofs;
+    vector<int> nRankDofs, nLocalDofs, nOverallDofs, rStartDofs;
   };
   
   
@@ -256,18 +264,23 @@ namespace AMDiS {
   {
   public:
     ParallelDofMapping() 
-      : mpiComm(NULL),
-	sendDofs(NULL),
+      : sendDofs(NULL),
 	recvDofs(NULL),
 	hasNonLocalDofs(false),
 	needMatIndex(false),
 	needMatIndexFromGlobal(false),
+	nLevel(1),
 	feSpaces(0),
-	nRankDofs(-1),
-	nLocalDofs(-1),
-	nOverallDofs(-1),
-	rStartDofs(-1)
-    {} 
+	nRankDofs(1),
+	nLocalDofs(1),
+	nOverallDofs(1),
+	rStartDofs(1)
+    {
+      nRankDofs[0] = -1;
+      nLocalDofs[0] = -1;
+      nOverallDofs[0] = -1;
+      rStartDofs[0] = -1;
+    } 
 
     /** \brief Initialize the parallel DOF mapping.
      *
@@ -307,36 +320,36 @@ namespace AMDiS {
     }
 
     /// Returns \ref nRankDofs, thus the number of DOFs owned by the rank.
-    inline int getRankDofs()
+    inline int getRankDofs(int level)
     {
-      TEST_EXIT_DBG(nRankDofs >= 0)("Should not happen!\n");
+      TEST_EXIT_DBG(nRankDofs[level] >= 0)("Should not happen!\n");
 
-      return nRankDofs;
+      return nRankDofs[level];
     }
 
     /// Returns \ref nLocalDofs, thus the number of DOFs in ranks subdomain.
-    inline int getLocalDofs()
+    inline int getLocalDofs(int level)
     {
-      TEST_EXIT_DBG(nLocalDofs >= 0)("Should not happen!\n");
+      TEST_EXIT_DBG(nLocalDofs[level] >= 0)("Should not happen!\n");
 
-      return nLocalDofs;
+      return nLocalDofs[level];
     }
 
     /// Returns \ref nOverallDofs, thus the number of all DOFs in this mapping.
-    inline int getOverallDofs()
+    inline int getOverallDofs(int level)
     {
-      TEST_EXIT_DBG(nOverallDofs >= 0)("Should not happen!\n");
+      TEST_EXIT_DBG(nOverallDofs[level] >= 0)("Should not happen!\n");
 
-      return nOverallDofs;
+      return nOverallDofs[level];
     }
 
     /// Returns \ref rStartDofs, thus the smallest global index of a DOF that is
     /// owned by the rank.
-    inline int getStartDofs()
+    inline int getStartDofs(int level)
     {
-      TEST_EXIT_DBG(rStartDofs >= 0)("Should not happen!\n");
+      TEST_EXIT_DBG(rStartDofs[level] >= 0)("Should not happen!\n");
 
-      return rStartDofs;
+      return rStartDofs[level];
     }
 
     /// Update the mapping.
@@ -361,7 +374,7 @@ namespace AMDiS {
       TEST_EXIT_DBG(data[feSpaces[ithComponent]].isRankDof(d))
 	("Should not happen!\n");
 
-      return dofToMatIndex.get(ithComponent, d) - rStartDofs;
+      return dofToMatIndex