At least, it compiles. Will never work, never, never ....

AMDiS/src/AMDiS_fwd.h

@@ -99,6 +99,7 @@ namespace AMDiS {
 
 #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
   class FeSpaceDofMap;
+  class MeshLevelData;
 #endif
 
   struct BoundaryObject;

AMDiS/src/DOFMatrix.cc

@@ -238,7 +238,7 @@ namespace AMDiS {
 	if (condition->applyBoundaryCondition()) {
 
 #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
- 	  if (dofMap->isRankDof(rowIndices[i])) 
+ 	  if (dofMap->isRankDof(rowIndices[i], 0))
  	    applyDBCs.insert(static_cast<int>(row));
 #else
  	  applyDBCs.insert(static_cast<int>(row));

AMDiS/src/DOFVector.h

@@ -248,7 +248,7 @@ namespace AMDiS {
     {
       TEST_EXIT_DBG(dofMap)("No rank dofs set!\n");
 
-      return dofMap->isRankDof(dof);
+      return dofMap->isRankDof(dof, 0);
     }
 #endif
 

AMDiS/src/io/Spreadsheet.cc

@@ -54,6 +54,8 @@ namespace AMDiS {
 
   void Spreadsheet::read(string filename)
   {
+    FUNCNAME("Spreadsheet::read()");
+
     data.clear();
 
     string line;

AMDiS/src/parallel/DofComm.cc

@@ -41,7 +41,26 @@ namespace AMDiS {
     }
   }
 
-  
+
+  int DofComm::getNumberDofs(int level, const FiniteElemSpace *feSpace)
+  {
+    FUNCNAME("DofComm::getNumberDofs()");
+
+    TEST_EXIT_DBG(level < data.size())("Should not happen!\n");
+
+    DofContainerSet dofs;
+
+    for (DataIter rankIt = data[level].begin(); 
+	 rankIt != data[level].end(); ++rankIt)
+      for (FeMapIter feIt = rankIt->second.begin();
+	   feIt != rankIt->second.end(); ++feIt)
+	if (feIt->first == feSpace)
+	  dofs.insert(feIt->second.begin(), feIt->second.end());
+
+    return static_cast<int>(dofs.size());
+  }
+
+
   bool DofComm::Iterator::setNextFeMap()
   {
     FUNCNAME("DofComm::Iterator::setNextFeMap()");

AMDiS/src/parallel/DofComm.h

@@ -65,6 +65,8 @@ namespace AMDiS {
       return data[level];
     }
 
+    int getNumberDofs(int level, const FiniteElemSpace *feSpace);
+
   protected:
     LevelDataType data;
 

AMDiS/src/parallel/ElementObjectDatabase.cc

@@ -376,9 +376,13 @@ namespace AMDiS {
   }
 
 
-  void ElementObjectDatabase::createRankData(map<int, int>& macroElementRankMap) 
+  void ElementObjectDatabase::createRankData(map<int, int>& macroElementRankMap,
+					     MeshLevelData& levelData)
   {
     FUNCNAME("ElementObjectDatabase::createRankData()");
+    
+    int nLevel = levelData.getLevelNumber();
+    TEST_EXIT_DBG(nLevel > 0)("Should not happen!\n");
 
     vertexOwner.clear();
     vertexInRank.clear();
@@ -391,7 +395,11 @@ namespace AMDiS {
 	if (it2->elIndex > vertexInRank[it->first][elementInRank].elIndex)
 	  vertexInRank[it->first][elementInRank] = *it2;
 
-	vertexOwner[it->first] = std::max(vertexOwner[it->first], elementInRank);
+	for (int level = 0; level < nLevel; level++) {
+	  int levelId = levelData.getLevelId(level, elementInRank);
+	  vertexOwner[it->first][level] = 
+	    std::max(vertexOwner[it->first][level], levelId);
+	}
       }
     }
     
@@ -407,7 +415,11 @@ namespace AMDiS {
 	if (it2->elIndex > edgeInRank[it->first][elementInRank].elIndex)
 	  edgeInRank[it->first][elementInRank] = *it2;
 
-	edgeOwner[it->first] = std::max(edgeOwner[it->first], elementInRank);
+	for (int level = 0; level < nLevel; level++) {
+	  int levelId = levelData.getLevelId(level, elementInRank);
+	  edgeOwner[it->first][level] = 
+	    std::max(edgeOwner[it->first][level], levelId);
+	}
       }
     }
     
@@ -423,7 +435,11 @@ namespace AMDiS {
 	if (it2->elIndex > faceInRank[it->first][elementInRank].elIndex)
 	  faceInRank[it->first][elementInRank] = *it2;
 
-	faceOwner[it->first] = std::max(faceOwner[it->first], elementInRank);
+	for (int level = 0; level < nLevel; level++) {
+	  int levelId = levelData.getLevelId(level, elementInRank);
+	  faceOwner[it->first][level] = 
+	    std::max(faceOwner[it->first][level], levelId);
+	}
       }
     }
   }
@@ -597,11 +613,12 @@ namespace AMDiS {
     }
 
 
-
+    ERROR_EXIT("REWRITE SERIALIZATION!\n");
+    /*
     SerUtil::serialize(out, vertexOwner);
     SerUtil::serialize(out, edgeOwner);
     SerUtil::serialize(out, faceOwner);
-
+    */
 
     nSize = vertexInRank.size();
     SerUtil::serialize(out, nSize);
@@ -737,6 +754,7 @@ namespace AMDiS {
     }
 
 
+    ERROR_EXIT("REWRITE DESERIALIZATION!\n");
     SerUtil::deserialize(in, vertexOwner);
     SerUtil::deserialize(in, edgeOwner);
     SerUtil::deserialize(in, faceOwner);

AMDiS/src/parallel/ElementObjectDatabase.h

@@ -28,6 +28,7 @@
 #include <boost/tuple/tuple.hpp>
 #include <boost/tuple/tuple_comparison.hpp>
 
+#include "AMDiS_fwd.h"
 #include "Containers.h"
 #include "Global.h"
 #include "Boundary.h"
@@ -144,7 +145,8 @@ namespace AMDiS {
      * \param[in]  macroElementRankMap   Maps to each macro element of the mesh
      *                                   the rank that owns this macro element.
      */
-    void createRankData(map<int, int>& macroElementRankMap);
+    void createRankData(map<int, int>& macroElementRankMap,
+			MeshLevelData& levelData);
 
 
     /** \brief
@@ -257,17 +259,17 @@ namespace AMDiS {
 
 
     /// Returns the rank owner of the current iterator position.
-    int getIterateOwner()
+    int getIterateOwner(int level)
     {
       switch (iterGeoPos) {
       case VERTEX:
-	return vertexOwner[vertexIter->first];
+	return vertexOwner[vertexIter->first][level];
 	break;
       case EDGE:
-	return edgeOwner[edgeIter->first];
+	return edgeOwner[edgeIter->first][level];
 	break;
       case FACE:
-	return faceOwner[faceIter->first];
+	return faceOwner[faceIter->first][level];
 	break;
       default:
 	ERROR_EXIT("Should not happen!\n");
@@ -279,21 +281,21 @@ namespace AMDiS {
 
 
     /// Returns the rank owner of a vertex DOF.
-    int getOwner(DegreeOfFreedom vertex)
+    int getOwner(DegreeOfFreedom vertex, int level)
     {
-      return vertexOwner[vertex];
+      return vertexOwner[vertex][level];
     }
 
     /// Returns the rank owner of an edge.
-    int getOwner(DofEdge edge)
+    int getOwner(DofEdge edge, int level)
     {
-      return edgeOwner[edge];
+      return edgeOwner[edge][level];
     }
 
     /// Returns the rank owner of an face.
-    int getOwner(DofFace face)
+    int getOwner(DofFace face, int level)
     {
-      return faceOwner[face];
+      return faceOwner[face][level];
     }
 
 
@@ -540,15 +542,17 @@ namespace AMDiS {
     /// Maps to an element object the corresponding face.
     map<ElementObjectData, DofFace> faceLocalMap;
 
+    /// Maps from level to rank number
+    typedef map<int, int> LevelRank;
 
     /// Defines for all vertex DOFs the rank that ownes this vertex DOF.
-    map<DegreeOfFreedom, int> vertexOwner;
+    map<DegreeOfFreedom, LevelRank> vertexOwner;
 
     /// Defines for all edges the rank that ownes this edge.
-    map<DofEdge, int> edgeOwner;
+    map<DofEdge, LevelRank> edgeOwner;
 
     /// Defines for all faces the rank that ownes this face.
-    map<DofFace, int> faceOwner;
+    map<DofFace, LevelRank> faceOwner;
 
 
     /// Defines to each vertex DOF a map that maps to each rank number the element

AMDiS/src/parallel/InteriorBoundary.cc

@@ -18,10 +18,11 @@
 
 namespace AMDiS {
 
-  AtomicBoundary& InteriorBoundary::getNewAtomic(int rank)
+  AtomicBoundary& InteriorBoundary::getNewAtomic(int level, int rank)
   {
-    boundary[rank].resize(boundary[rank].size() + 1);
-    return boundary[rank][boundary[rank].size() - 1];
+    int size = boundary[level][rank].size();
+    boundary[level][rank].resize(size + 1);
+    return boundary[level][rank][size];
   }
 
 
@@ -29,32 +30,51 @@ namespace AMDiS {
   {
     InteriorBoundary& other2 = const_cast<InteriorBoundary&>(other);
 
-    for (RankToBoundMap::const_iterator it = boundary.begin(); 
-	 it != boundary.end(); ++it) {
-      if (other2.boundary.count(it->first) == 0)
-	return false;     
-
-      if (other2.boundary[it->first].size() != it->second.size())
-	return false;
-            
-      for (unsigned int i = 0; i < it->second.size(); i++) {
-	std::vector<AtomicBoundary>::iterator bIt = 
-	  find(other2.boundary[it->first].begin(),
-	       other2.boundary[it->first].end(), it->second[i]);
+    if (boundary.size() != other2.boundary.size())
+      return false;
+
+    for (unsigned int level = 0; level < boundary.size(); level++) {
+      for (RankToBoundMap::const_iterator it = boundary[level].begin(); 
+	   it != boundary[level].end(); ++it) {
+	if (other2.boundary[level].count(it->first) == 0)
+	  return false;     
+	
+	if (other2.boundary[level][it->first].size() != it->second.size())
+	  return false;
 	
-	if (bIt == other2.boundary[it->first].end())
-	  return false;	
-      }      
+	for (unsigned int i = 0; i < it->second.size(); i++) {
+	  std::vector<AtomicBoundary>::iterator bIt = 
+	    find(other2.boundary[level][it->first].begin(),
+		 other2.boundary[level][it->first].end(), it->second[i]);
+	  
+	  if (bIt == other2.boundary[level][it->first].end())
+	    return false;	
+	}      
+      }
     }
 
     return true;
   }
 
 
+  void InteriorBoundary::reset(int level)
+  { 
+    FUNCNAME("InteriorBoundary::reset()");
+
+    nLevel = level;
+
+    boundary.clear();
+    boundary.resize(nLevel);
+  }
+
+
   void InteriorBoundary::serialize(std::ostream &out)
   {
     FUNCNAME("InteriorBoundary::serialize()");
 
+    ERROR_EXIT("REWRITE TO MULTILEVEL STRUCTURE!\n");
+
+#if 0
     int mSize = boundary.size();
     SerUtil::serialize(out, mSize);
     for (RankToBoundMap::iterator it = boundary.begin(); 
@@ -83,6 +103,7 @@ namespace AMDiS {
 	SerUtil::serialize(out, bound.type);
       }
     }
+#endif
   }
 
 
@@ -91,6 +112,9 @@ namespace AMDiS {
   {
     FUNCNAME("InteriorBoundary::deserialize()");
 
+    ERROR_EXIT("REWRITE TO MULTILEVEL STRUCTURE!\n");
+
+#if 0
     int mSize = 0;
     SerUtil::deserialize(in, mSize);
     for (int i = 0; i < mSize; i++) {
@@ -137,6 +161,7 @@ namespace AMDiS {
 	  bound.neighObj.el = NULL;
       }
     }
+#endif
   }
 
 

AMDiS/src/parallel/InteriorBoundary.h

@@ -47,34 +47,35 @@ namespace AMDiS {
     public:
       iterator(InteriorBoundary &b)
 	: bound(b),
-	  levelData(NULL),
 	  level(0)
       {
 	reset();
       }
 
-      iterator(InteriorBoundary &b, MeshLevelData &levelData, int level)
+      iterator(InteriorBoundary &b, int level)
 	: bound(b),
-	  levelData(&levelData),
 	  level(level)
       {
+	TEST_EXIT_DBG(level < bound.boundary.size())
+	  ("Should not happen!\n");
+
 	reset();
       }
 
       /// Set the iterator to the first position.
       void reset()
       {
-	mapIt = bound.boundary.begin();
+	mapIt = bound.boundary[level].begin();
 	nextNonempty();
 
-	if (mapIt != bound.boundary.end())
+	if (mapIt != bound.boundary[level].end())
 	  vecIt = mapIt->second.begin();	
       }
 
       /// Test if iterator is at the final position.
       bool end() const
       {
-	return (mapIt == bound.boundary.end());
+	return (mapIt == bound.boundary[level].end());
       }
 
       /// Move iterator to the next position.
@@ -85,7 +86,7 @@ namespace AMDiS {
 	  ++mapIt;
 	  nextNonempty();
 
-	  if (mapIt != bound.boundary.end())
+	  if (mapIt != bound.boundary[level].end())
 	    vecIt = mapIt->second.begin();	  
 	}	
       }
@@ -105,22 +106,12 @@ namespace AMDiS {
 	++mapIt;
 	nextNonempty();
 
-	if (mapIt != bound.boundary.end())
+	if (mapIt != bound.boundary[level].end())
 	  vecIt = mapIt->second.begin();	
       }
 
       inline int getRank() 
       {
-	if (level > 0) {
-	  int r = levelData->mapRank(mapIt->first, level - 1, level);
-
-	  TEST_EXIT_DBG(r >= 0)
-	    ("Mapping rank %d from level % to level %d does not work!\n",
-	     mapIt->first, level - 1, level);
-
-	  return r;
-	}
-
 	return mapIt->first;
       }
 
@@ -128,25 +119,13 @@ namespace AMDiS {
 
       inline void nextNonempty()
       {
-	if (mapIt == bound.boundary.end())
+	if (mapIt == bound.boundary[level].end())
 	  return;
 
-	if (level > 0) {
-	  TEST_EXIT_DBG(levelData)("No mesh level data object defined!\n");
-	  TEST_EXIT_DBG(level == 1)("Only 2-level method supported!\n");
-
-	  while (!levelData->rankInSubdomain(mapIt->first, level) ||
-		 mapIt->second.size() == 0) {
-	    ++mapIt;
-	    if (mapIt == bound.boundary.end())
-	      return;	  
-	  }
-	} else {
-	  while (mapIt->second.size() == 0) {
-	    ++mapIt;
-	    if (mapIt == bound.boundary.end())
-	      return;
-	  }
+	while (mapIt->second.size() == 0) {
+	  ++mapIt;
+	  if (mapIt == bound.boundary[level].end())
+	    return;	  
 	}
       }
 
@@ -157,20 +136,19 @@ namespace AMDiS {
 
       InteriorBoundary &bound;
 
-      MeshLevelData *levelData;
-
       int level;
     };
 
   public:
-    InteriorBoundary() {}
-
-    void clear()
+    InteriorBoundary(int l = 1) 
+      : nLevel(l)
     {
-      boundary.clear();
+      boundary.resize(nLevel);
     }
 
-    AtomicBoundary& getNewAtomic(int rank);
+    void reset(int nLevel);
+
+    AtomicBoundary& getNewAtomic(int level, int rank);
 
     /// Writes this object to a file.
     void serialize(ostream &out);
@@ -185,10 +163,13 @@ namespace AMDiS {
   protected:
     void serializeExcludeList(ostream &out, ExcludeList &list);
 
-    void deserializeExcludeList(istream &in, ExcludeList &list);
+    void deserializeExcludeList(istream &in, ExcludeList &list);    
 
   public:
-    RankToBoundMap boundary;
+    vector<RankToBoundMap> boundary;    
+
+  protected:
+    int nLevel;
   };
 }
 

AMDiS/src/parallel/MeshDistributor.cc

@@ -232,9 +232,10 @@ namespace AMDiS {
       }
     }
 
+    // If required, create hierarchical mesh level structure.
+    createMeshLevelStructure();
 
-    // === Create interior boundary information. ===
-
+    // Create interior boundary information.
     createInteriorBoundaryInfo();
 
 #if (DEBUG != 0)    
@@ -294,11 +295,6 @@ namespace AMDiS {
 	 it != mesh->getPeriodicAssociations().end(); ++it)
       const_cast<DOFAdmin&>(mesh->getDofAdmin(0)).removeDOFContainer(dynamic_cast<DOFContainer*>(it->second));
 
-
-    // If required, create hierarchical mesh level structure.
-    createMeshLevelStructure();
-
-
     updateLocalGlobalNumbering();
 
     // === In 3D we have to make some test, if the resulting mesh is valid.  ===
@@ -784,18 +780,73 @@ namespace AMDiS {
   {
     FUNCNAME("MeshDistributor::createMeshLevelStructure()");
 
+    if (mpiSize != 16)
+      return;
+
     std::set<int> neighbours;
-    for (InteriorBoundary::iterator it(rankIntBoundary); !it.end(); ++it)
-      neighbours.insert(it.getRank());
+    switch (mpiRank) {
+    case 0:
+      neighbours.insert(1); neighbours.insert(4); neighbours.insert(5);
+      break;
+    case 1:
+      neighbours.insert(0); neighbours.insert(2); neighbours.insert(4); neighbours.insert(5); neighbours.insert(6);
+      break;
+    case 2:
+      neighbours.insert(1); neighbours.insert(3); neighbours.insert(5); neighbours.insert(6); neighbours.insert(7);
+      break;
+    case 3:
+      neighbours.insert(2); neighbours.insert(6); neighbours.insert(7);
+      break;
+    case 4:
+      neighbours.insert(0); neighbours.insert(1); neighbours.insert(5); neighbours.insert(8); neighbours.insert(9);
+      break;
+    case 5:
+      neighbours.insert(0); neighbours.insert(1); neighbours.insert(2); 
+      neighbours.insert(4); neighbours.insert(6);
+      neighbours.insert(8); neighbours.insert(9); neighbours.insert(10); 
+      break;
+    case 6:
+      neighbours.insert(1); neighbours.insert(2); neighbours.insert(3); 
+      neighbours.insert(5); neighbours.insert(7);
+      neighbours.insert(9); neighbours.insert(10); neighbours.insert(11); 
+      break;
+    case 7:
+      neighbours.insert(2); neighbours.insert(3); neighbours.insert(6);  neighbours.insert(10); neighbours.insert(11);
+      break;
+    case 8:
+      neighbours.insert(4); neighbours.insert(5); neighbours.insert(9);  neighbours.insert(12); neighbours.insert(13);
+      break;
+    case 9:
+      neighbours.insert(4); neighbours.insert(5); neighbours.insert(6); 
+      neighbours.insert(8); neighbours.insert(10);
+      neighbours.insert(12); neighbours.insert(13); neighbours.insert(14); 
+      break;
+    case 10:
+      neighbours.insert(5); neighbours.insert(6); neighbours.insert(7); 
+      neighbours.insert(9); neighbours.insert(11);
+      neighbours.insert(13); neighbours.insert(14); neighbours.insert(15); 
+      break;
+    case 11:
+      neighbours.insert(6); neighbours.insert(7); neighbours.insert(10);  neighbours.insert(14); neighbours.insert(15);
+      break;
+    case 12:
+      neighbours.insert(8); neighbours.insert(9); neighbours.insert(13);
+      break;
+    case 13:
+      neighbours.insert(8); neighbours.insert(9); neighbours.insert(10);  neighbours.insert(12); neighbours.insert(14);
+      break;
+    case 14:
+      neighbours.insert(9); neighbours.insert(10); neighbours.insert(11);  neighbours.insert(13); neighbours.insert(15);
+      break;
+    case 15:
+      neighbours.insert(10); neighbours.insert(11); neighbours.insert(14);
+      break;
+    }    
 
-    for (InteriorBoundary::iterator it(otherIntBoundary); !it.end(); ++it)
-      neighbours.insert(it.getRank());
+    TEST_EXIT(neighbours.size() > 0)("Should not happen!\n");
 
     levelData.init(neighbours);
 
-    if (mpiSize != 16)
-      return;
-
     bool multiLevelTest = false;
     Parameters::get("parallel->multi level test", multiLevelTest);
     if (multiLevelTest) {
@@ -1472,7 +1523,7 @@ namespace AMDiS {
   {
     FUNCNAME("MeshDistributor::updateInteriorBoundaryInfo()");
 
-    elObjDb.createRankData(partitionMap);
+    elObjDb.createRankData(partitionMap, levelData);