diff --git a/AMDiS/src/parallel/ElementObjectData.cc b/AMDiS/src/parallel/ElementObjectData.cc index 705318ed231811d1843f0207d721c4e4c87a185b..00a0e967506590ac7107b1246f5a26a9efd78207 100644 --- a/AMDiS/src/parallel/ElementObjectData.cc +++ b/AMDiS/src/parallel/ElementObjectData.cc @@ -2,36 +2,54 @@ namespace AMDiS { - void ElementObjects::createRankData() + void ElementObjects::createRankData(std::map<int, int>& macroElementRankMap) { + FUNCNAME("ElementObjects::createRankData()"); + + vertexOwner.clear(); + vertexInRank.clear(); for (std::map<DegreeOfFreedom, std::vector<ElementObjectData> >::iterator it = vertexElements.begin(); it != vertexElements.end(); ++it) { - for (std::vector<ElementObjectData>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) { - int elOwner = elementInRank[it2->elIndex]; + for (std::vector<ElementObjectData>::iterator it2 = it->second.begin(); + it2 != it->second.end(); ++it2) { + int elementInRank = macroElementRankMap[it2->elIndex]; - if (it2->elIndex > vertexInRank[it->first][elOwner].elIndex) - vertexInRank[it->first][elOwner] = *it2; + if (it2->elIndex > vertexInRank[it->first][elementInRank].elIndex) + vertexInRank[it->first][elementInRank] = *it2; + + vertexOwner[it->first] = std::max(vertexOwner[it->first], elementInRank); } } - + + edgeOwner.clear(); + edgeInRank.clear(); for (std::map<DofEdge, std::vector<ElementObjectData> >::iterator it = edgeElements.begin(); it != edgeElements.end(); ++it) { - for (std::vector<ElementObjectData>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) { - int elOwner = elementInRank[it2->elIndex]; + for (std::vector<ElementObjectData>::iterator it2 = it->second.begin(); + it2 != it->second.end(); ++it2) { + int elementInRank = macroElementRankMap[it2->elIndex]; - if (it2->elIndex > edgeInRank[it->first][elOwner].elIndex) - edgeInRank[it->first][elOwner] = *it2; + if (it2->elIndex > edgeInRank[it->first][elementInRank].elIndex) + edgeInRank[it->first][elementInRank] = *it2; + + edgeOwner[it->first] = std::max(edgeOwner[it->first], elementInRank); } } + + faceOwner.clear(); + faceInRank.clear(); for (std::map<DofFace, std::vector<ElementObjectData> >::iterator it = faceElements.begin(); it != faceElements.end(); ++it) { - for (std::vector<ElementObjectData>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) { - int elOwner = elementInRank[it2->elIndex]; + for (std::vector<ElementObjectData>::iterator it2 = it->second.begin(); + it2 != it->second.end(); ++it2) { + int elementInRank = macroElementRankMap[it2->elIndex]; - if (it2->elIndex > faceInRank[it->first][elOwner].elIndex) - faceInRank[it->first][elOwner] = *it2; + if (it2->elIndex > faceInRank[it->first][elementInRank].elIndex) + faceInRank[it->first][elementInRank] = *it2; + + faceOwner[it->first] = std::max(faceOwner[it->first], elementInRank); } } } diff --git a/AMDiS/src/parallel/ElementObjectData.h b/AMDiS/src/parallel/ElementObjectData.h index 5607ea20760256d3eedc6c5010a744ba4f5eb119..69f0d9658a08d8af10f18563b8e9e52c3807da8c 100644 --- a/AMDiS/src/parallel/ElementObjectData.h +++ b/AMDiS/src/parallel/ElementObjectData.h @@ -50,33 +50,29 @@ namespace AMDiS { class ElementObjects { public: - ElementObjects(std::map<int, int> vec) - : elementInRank(vec), - iterGeoPos(CENTER) + ElementObjects() + : iterGeoPos(CENTER) {} void addVertex(DegreeOfFreedom vertex, int elIndex, int ith, BoundaryType bound = INTERIOR) { vertexElements[vertex].push_back(ElementObjectData(elIndex, ith, bound)); - vertexOwner[vertex] = std::max(vertexOwner[vertex], elementInRank[elIndex]); } void addEdge(DofEdge edge, int elIndex, int ith, BoundaryType bound = INTERIOR) { edgeElements[edge].push_back(ElementObjectData(elIndex, ith, bound)); - edgeOwner[edge] = std::max(edgeOwner[edge], elementInRank[elIndex]); } void addFace(DofFace face, int elIndex, int ith, BoundaryType bound = INTERIOR) { faceElements[face].push_back(ElementObjectData(elIndex, ith, bound)); - faceOwner[face] = std::max(faceOwner[face], elementInRank[elIndex]); } - void createRankData(); + void createRankData(std::map<int, int>& macroElementRankMap); bool iterate(GeoIndex pos) { @@ -244,8 +240,6 @@ namespace AMDiS { } private: - std::map<int, int> elementInRank; - std::map<DegreeOfFreedom, std::vector<ElementObjectData> > vertexElements; std::map<DofEdge, std::vector<ElementObjectData> > edgeElements; std::map<DofFace, std::vector<ElementObjectData> > faceElements; diff --git a/AMDiS/src/parallel/InteriorBoundary.h b/AMDiS/src/parallel/InteriorBoundary.h index af64284bb02bf167df5234f0ca23719eb8764d06..7cd8938895c9de78b7781acc7e334e9d06a7ca8d 100644 --- a/AMDiS/src/parallel/InteriorBoundary.h +++ b/AMDiS/src/parallel/InteriorBoundary.h @@ -216,6 +216,11 @@ namespace AMDiS { public: InteriorBoundary() {} + void clear() + { + boundary.clear(); + } + AtomicBoundary& getNewAtomic(int rank); /// Writes this object to a file. diff --git a/AMDiS/src/parallel/MeshDistributor.cc b/AMDiS/src/parallel/MeshDistributor.cc index 9262d3e168279df05895a9a89daca6cc05ba084c..767151d28c2c0defd44d802d435af1e69998b6d7 100644 --- a/AMDiS/src/parallel/MeshDistributor.cc +++ b/AMDiS/src/parallel/MeshDistributor.cc @@ -141,7 +141,7 @@ namespace AMDiS { // === Create new global and local DOF numbering. === - createLocalGlobalNumbering(); + // createLocalGlobalNumbering(); // === Remove all macro elements that are not part of the rank partition. === @@ -150,6 +150,8 @@ namespace AMDiS { macroElementStructureConsisten = true; + updateLocalGlobalNumbering(true); + // === Reset all DOFAdmins of the mesh. === updateDofAdmins(); @@ -187,7 +189,7 @@ namespace AMDiS { #endif mesh->dofCompress(); - updateLocalGlobalNumbering(); + updateLocalGlobalNumbering(false); // === Update periodic mapping, if there are periodic boundaries. === @@ -563,7 +565,7 @@ namespace AMDiS { // === Because the mesh has been changed, update the DOF numbering and mappings. === mesh->dofCompress(); - updateLocalGlobalNumbering(); + updateLocalGlobalNumbering(false); // === Update periodic mapping, if there are periodic boundaries. === @@ -1142,6 +1144,8 @@ namespace AMDiS { #endif partitioner->fillCoarsePartitionVec(&partitionVec); + + updateInteriorBoundaryInfo(); } @@ -1149,127 +1153,28 @@ namespace AMDiS { { FUNCNAME("MeshDistributor::createInteriorBoundaryInfo()"); - createBoundaryDataStructure(); - - // === Once we have this information, we must care about the order of the atomic === - // === bounds in the three boundary handling object. Eventually all the bound- === - // === aries have to be in the same order on both ranks that share the bounday. === - - StdMpi<std::vector<AtomicBoundary> > stdMpi(mpiComm); - stdMpi.send(myIntBoundary.boundary); - stdMpi.recv(otherIntBoundary.boundary); - stdMpi.startCommunication<int>(MPI_INT); - - // === The information about all neighbouring boundaries has been received. So === - // === the rank tests if its own atomic boundaries are in the same order. If === - // === not, the atomic boundaries are swaped to the correct order. === - - for (RankToBoundMap::iterator rankIt = otherIntBoundary.boundary.begin(); - rankIt != otherIntBoundary.boundary.end(); ++rankIt) { - - // === We have received from rank "rankIt->first" the ordered list of element === - // === indices. Now, we have to sort the corresponding list in this rank to === - // === get the same order. === - - for (unsigned int j = 0; j < rankIt->second.size(); j++) { - - // If the expected object is not at place, search for it. - - BoundaryObject &recvedBound = stdMpi.getRecvData()[rankIt->first][j].rankObj; - - if ((rankIt->second)[j].neighObj != recvedBound) { - unsigned int k = j + 1; - - for (; k < rankIt->second.size(); k++) - if ((rankIt->second)[k].neighObj == recvedBound) - break; - - // The element must always be found, because the list is just in another order. - TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n"); - - // Swap the current with the found element. - AtomicBoundary tmpBound = (rankIt->second)[k]; - (rankIt->second)[k] = (rankIt->second)[j]; - (rankIt->second)[j] = tmpBound; - } - } - } - - - // === Do the same for the periodic boundaries. === - - if (periodicBoundary.boundary.size() > 0) { - stdMpi.clear(); - - InteriorBoundary sendBounds, recvBounds; - for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin(); - rankIt != periodicBoundary.boundary.end(); ++rankIt) { - - TEST_EXIT_DBG(rankIt->first != mpiRank) - ("It is not allowed to have an interior boundary within a rank partition!\n"); - - if (rankIt->first < mpiRank) - sendBounds.boundary[rankIt->first] = rankIt->second; - else - recvBounds.boundary[rankIt->first] = rankIt->second; - } + createMeshElementData(); - stdMpi.send(sendBounds.boundary); - stdMpi.recv(recvBounds.boundary); - stdMpi.startCommunication<int>(MPI_INT); + createBoundaryData(); - for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin(); - rankIt != periodicBoundary.boundary.end(); ++rankIt) { - if (rankIt->first <= mpiRank) - continue; - - for (unsigned int j = 0; j < rankIt->second.size(); j++) { - - BoundaryObject &recvedBound = stdMpi.getRecvData()[rankIt->first][j].rankObj; - - if (periodicBoundary.boundary[rankIt->first][j].neighObj != recvedBound) { - unsigned int k = j + 1; - for (; k < rankIt->second.size(); k++) - if (periodicBoundary.boundary[rankIt->first][k].neighObj == recvedBound) - break; - - // The element must always be found, because the list is just in - // another order. - TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n"); - - // Swap the current with the found element. - AtomicBoundary tmpBound = (rankIt->second)[k]; - (rankIt->second)[k] = (rankIt->second)[j]; - (rankIt->second)[j] = tmpBound; - } - } - } - } // periodicBoundary.boundary.size() > 0 } - void MeshDistributor::createBoundaryDataStructure() + void MeshDistributor::updateInteriorBoundaryInfo() { - FUNCNAME("MeshDistributor::createBoundaryDataStructure()"); + FUNCNAME("MeshDistributor::updateInteriorBoundaryInfo()"); - // Data structure to store all sub-objects of all elements of the mesh. - ElementObjects elObjects(partitionVec); - // Maps to each element index a pointer to the corresponding element. - std::map<int, Element*> elIndexMap; - // Maps to each element index the type of this element. - std::map<int, int> elIndexTypeMap; + elObjects.createRankData(partitionVec); - // The following three data structures store periodic DOFs, edges and faces, - // i.e., - std::map<std::pair<DegreeOfFreedom, DegreeOfFreedom>, BoundaryType> periodicDofs; - std::map<std::pair<DofEdge, DofEdge>, BoundaryType> periodicEdges; - std::map<std::pair<DofFace, DofFace>, BoundaryType> periodicFaces; + createBoundaryData(); + } - // Stores to each DOF all its periodic associations. - std::map<DegreeOfFreedom, std::set<BoundaryType> > periodicDofAssoc; + void MeshDistributor::createMeshElementData() + { + FUNCNAME("MeshDistributor::createMeshElementData()"); - // === Phase 1, fills the data structures defined above. === + // === Fills macro element data structures. === TraverseStack stack; ElInfo *elInfo = @@ -1279,8 +1184,8 @@ namespace AMDiS { TEST_EXIT_DBG(elInfo->getLevel() == 0)("Should not happen!\n"); Element *el = elInfo->getElement(); - elIndexMap[el->getIndex()] = el; - elIndexTypeMap[el->getIndex()] = elInfo->getType(); + macroElIndexMap[el->getIndex()] = el; + macroElIndexTypeMap[el->getIndex()] = elInfo->getType(); // === Add all sub object of the element to the variable elObjects. === @@ -1295,7 +1200,7 @@ namespace AMDiS { elObjects.addFace(el->getFace(i), el->getIndex(), i); - // === === + // === Get periodic boundary information. === switch (mesh->getDim()) { case 2: @@ -1346,9 +1251,9 @@ namespace AMDiS { } - // === PHASE 2 === + // === Create mesh element data for this rank. === - elObjects.createRankData(); + elObjects.createRankData(partitionVec); // === Search for interectly connected vertices in periodic boundaries. === @@ -1383,9 +1288,22 @@ namespace AMDiS { } } } + } + + void MeshDistributor::createBoundaryData() + { + FUNCNAME("MeshDistributor::createBoundaryData()"); - // === PHASE 3 === + + // === Clear all relevant data structures, === + + myIntBoundary.clear(); + otherIntBoundary.clear(); + periodicBoundary.clear(); + + + // === Create interior boundary data structure. === for (int geoPos = 0; geoPos < mesh->getDim(); geoPos++) { GeoIndex geoIndex = INDEX_OF_DIM(geoPos, mesh->getDim()); @@ -1397,9 +1315,9 @@ namespace AMDiS { ElementObjectData& rankBoundEl = objData[mpiRank]; AtomicBoundary bound; - bound.rankObj.el = elIndexMap[rankBoundEl.elIndex]; + bound.rankObj.el = macroElIndexMap[rankBoundEl.elIndex]; bound.rankObj.elIndex = rankBoundEl.elIndex; - bound.rankObj.elType = elIndexTypeMap[rankBoundEl.elIndex]; + bound.rankObj.elType = macroElIndexTypeMap[rankBoundEl.elIndex]; bound.rankObj.subObj = geoIndex; bound.rankObj.ithObj = rankBoundEl.ithObject; @@ -1419,9 +1337,9 @@ namespace AMDiS { if (it2->first == mpiRank) continue; - bound.neighObj.el = elIndexMap[it2->second.elIndex]; + bound.neighObj.el = macroElIndexMap[it2->second.elIndex]; bound.neighObj.elIndex = it2->second.elIndex; - bound.neighObj.elType = elIndexTypeMap[it2->second.elIndex]; + bound.neighObj.elType = macroElIndexTypeMap[it2->second.elIndex]; bound.neighObj.subObj = geoIndex; bound.neighObj.ithObj = it2->second.ithObject; @@ -1443,7 +1361,7 @@ namespace AMDiS { ElementObjectData& ownerBoundEl = objData[owner]; - bound.neighObj.el = elIndexMap[ownerBoundEl.elIndex]; + bound.neighObj.el = macroElIndexMap[ownerBoundEl.elIndex]; bound.neighObj.elIndex = ownerBoundEl.elIndex; bound.neighObj.elType = -1; bound.neighObj.subObj = geoIndex; @@ -1464,7 +1382,7 @@ namespace AMDiS { } - // === PHASE 4 === + // === Create periodic boundary data structure. === for (std::map<std::pair<DegreeOfFreedom, DegreeOfFreedom>, BoundaryType>::iterator it = periodicDofs.begin(); it != periodicDofs.end(); ++it) { @@ -1479,13 +1397,13 @@ namespace AMDiS { ElementObjectData& perDofEl1 = elIt->second; AtomicBoundary bound; - bound.rankObj.el = elIndexMap[perDofEl0.elIndex]; + bound.rankObj.el = macroElIndexMap[perDofEl0.elIndex]; bound.rankObj.elIndex = perDofEl0.elIndex; - bound.rankObj.elType = elIndexTypeMap[perDofEl0.elIndex]; + bound.rankObj.elType = macroElIndexTypeMap[perDofEl0.elIndex]; bound.rankObj.subObj = VERTEX; bound.rankObj.ithObj = perDofEl0.ithObject; - bound.neighObj.el = elIndexMap[perDofEl1.elIndex]; + bound.neighObj.el = macroElIndexMap[perDofEl1.elIndex]; bound.neighObj.elIndex = perDofEl1.elIndex; bound.neighObj.elType = -1; bound.neighObj.subObj = VERTEX; @@ -1519,13 +1437,13 @@ namespace AMDiS { ElementObjectData& perEdgeEl1 = elObjects.getElements(it->first.second)[0]; AtomicBoundary bound; - bound.rankObj.el = elIndexMap[perEdgeEl0.elIndex]; + bound.rankObj.el = macroElIndexMap[perEdgeEl0.elIndex]; bound.rankObj.elIndex = perEdgeEl0.elIndex; - bound.rankObj.elType = elIndexTypeMap[perEdgeEl0.elIndex]; + bound.rankObj.elType = macroElIndexTypeMap[perEdgeEl0.elIndex]; bound.rankObj.subObj = EDGE; bound.rankObj.ithObj = perEdgeEl0.ithObject; - bound.neighObj.el = elIndexMap[perEdgeEl1.elIndex]; + bound.neighObj.el = macroElIndexMap[perEdgeEl1.elIndex]; bound.neighObj.elIndex = perEdgeEl1.elIndex; bound.neighObj.elType = -1; bound.neighObj.subObj = EDGE; @@ -1542,6 +1460,102 @@ namespace AMDiS { b.neighObj.setReverseMode(b.rankObj, feSpace); } + + + // === Once we have this information, we must care about the order of the atomic === + // === bounds in the three boundary handling object. Eventually all the bound- === + // === aries have to be in the same order on both ranks that share the bounday. === + + StdMpi<std::vector<AtomicBoundary> > stdMpi(mpiComm); + stdMpi.send(myIntBoundary.boundary); + stdMpi.recv(otherIntBoundary.boundary); + stdMpi.startCommunication<int>(MPI_INT); + + // === The information about all neighbouring boundaries has been received. So === + // === the rank tests if its own atomic boundaries are in the same order. If === + // === not, the atomic boundaries are swaped to the correct order. === + + for (RankToBoundMap::iterator rankIt = otherIntBoundary.boundary.begin(); + rankIt != otherIntBoundary.boundary.end(); ++rankIt) { + + // === We have received from rank "rankIt->first" the ordered list of element === + // === indices. Now, we have to sort the corresponding list in this rank to === + // === get the same order. === + + for (unsigned int j = 0; j < rankIt->second.size(); j++) { + + // If the expected object is not at place, search for it. + + BoundaryObject &recvedBound = stdMpi.getRecvData()[rankIt->first][j].rankObj; + + if ((rankIt->second)[j].neighObj != recvedBound) { + unsigned int k = j + 1; + + for (; k < rankIt->second.size(); k++) + if ((rankIt->second)[k].neighObj == recvedBound) + break; + + // The element must always be found, because the list is just in another order. + TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n"); + + // Swap the current with the found element. + AtomicBoundary tmpBound = (rankIt->second)[k]; + (rankIt->second)[k] = (rankIt->second)[j]; + (rankIt->second)[j] = tmpBound; + } + } + } + + + // === Do the same for the periodic boundaries. === + + if (periodicBoundary.boundary.size() > 0) { + stdMpi.clear(); + + InteriorBoundary sendBounds, recvBounds; + for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin(); + rankIt != periodicBoundary.boundary.end(); ++rankIt) { + + TEST_EXIT_DBG(rankIt->first != mpiRank) + ("It is not allowed to have an interior boundary within a rank partition!\n"); + + if (rankIt->first < mpiRank) + sendBounds.boundary[rankIt->first] = rankIt->second; + else + recvBounds.boundary[rankIt->first] = rankIt->second; + } + + stdMpi.send(sendBounds.boundary); + stdMpi.recv(recvBounds.boundary); + stdMpi.startCommunication<int>(MPI_INT); + + for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin(); + rankIt != periodicBoundary.boundary.end(); ++rankIt) { + if (rankIt->first <= mpiRank) + continue; + + for (unsigned int j = 0; j < rankIt->second.size(); j++) { + + BoundaryObject &recvedBound = stdMpi.getRecvData()[rankIt->first][j].rankObj; + + if (periodicBoundary.boundary[rankIt->first][j].neighObj != recvedBound) { + unsigned int k = j + 1; + for (; k < rankIt->second.size(); k++) + if (periodicBoundary.boundary[rankIt->first][k].neighObj == recvedBound) + break; + + // The element must always be found, because the list is just in + // another order. + TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n"); + + // Swap the current with the found element. + AtomicBoundary tmpBound = (rankIt->second)[k]; + (rankIt->second)[k] = (rankIt->second)[j]; + (rankIt->second)[j] = tmpBound; + } + } + } + } // periodicBoundary.boundary.size() > 0 } @@ -1733,7 +1747,7 @@ namespace AMDiS { } - void MeshDistributor::updateLocalGlobalNumbering() + void MeshDistributor::updateLocalGlobalNumbering(bool b) { FUNCNAME("MeshDistributor::updateLocalGlobalNumbering()"); @@ -1767,6 +1781,10 @@ namespace AMDiS { sort(rankDofs.begin(), rankDofs.end(), cmpDofsByValue); int nRankAllDofs = rankDofs.size(); + if (b) + for (unsigned int i = 0; i < rankDofs.size(); i++) + *const_cast<DegreeOfFreedom*>(rankDofs[i]) = i; + // === Traverse on interior boundaries and move all not ranked owned DOFs from === // === rankDofs to boundaryDofs. === diff --git a/AMDiS/src/parallel/MeshDistributor.h b/AMDiS/src/parallel/MeshDistributor.h index 6c3ccec9747d83bc0685dfcf859e206a0617e43c..2ee2238cd1721424f8c680d09e8e7e1d1e264d4a 100644 --- a/AMDiS/src/parallel/MeshDistributor.h +++ b/AMDiS/src/parallel/MeshDistributor.h @@ -28,11 +28,11 @@ #include <vector> #include <mpi.h> +#include "parallel/InteriorBoundary.h" #include "Global.h" #include "ProblemTimeInterface.h" #include "ProblemIterationInterface.h" #include "FiniteElemSpace.h" -#include "parallel/InteriorBoundary.h" #include "Serializer.h" #include "BoundaryManager.h" #include "ElementObjectData.h" @@ -240,7 +240,11 @@ namespace AMDiS { */ void createInteriorBoundaryInfo(); - void createBoundaryDataStructure(); + void updateInteriorBoundaryInfo(); + + void createMeshElementData(); + + void createBoundaryData(); /// Removes all macro elements from the mesh that are not part of ranks partition. void removeMacroElements(); @@ -249,7 +253,7 @@ namespace AMDiS { void createLocalGlobalNumbering(); /// Updates the local and global DOF numbering after the mesh has been changed. - void updateLocalGlobalNumbering(); + void updateLocalGlobalNumbering(bool b); /** \brief * Creates to all dofs in rank's partition that are on a periodic boundary the @@ -498,6 +502,24 @@ namespace AMDiS { /// Number of DOFs in the whole domain. int nOverallDofs; + // Data structure to store all sub-objects of all elements of the macro mesh. + ElementObjects elObjects; + + // Maps to each macro element index a pointer to the corresponding element. + std::map<int, Element*> macroElIndexMap; + + // Maps to each macro element index the type of this element. + std::map<int, int> macroElIndexTypeMap; + + // The following three data structures store periodic DOFs, edges and faces. + std::map<std::pair<DegreeOfFreedom, DegreeOfFreedom>, BoundaryType> periodicDofs; + std::map<std::pair<DofEdge, DofEdge>, BoundaryType> periodicEdges; + std::map<std::pair<DofFace, DofFace>, BoundaryType> periodicFaces; + + // Stores to each DOF all its periodic associations. + std::map<DegreeOfFreedom, std::set<BoundaryType> > periodicDofAssoc; + + /** \brief * Defines the interior boundaries of the domain that result from partitioning * the whole mesh. Contains only the boundaries, which are owned by the rank, i.e.,