Skip to content
GitLab
Commit 2c1211fd authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

So, it compiles and all four parallel tests are fine.

parent 227aa1e9
Hide whitespace changes
Inline Side-by-side
AMDiS/src/BoundaryObject.h
View file @ 2c1211fd
......@@ -104,7 +104,8 @@ namespace AMDiS {
*/
struct AtomicBoundary {
AtomicBoundary()
: type(INTERIOR)
: type(INTERIOR),
maxLevel(0)
{}
bool operator==(const AtomicBoundary& other) const;
......@@ -119,6 +120,8 @@ namespace AMDiS {
/// boundaries. Till now it is used only for periodic boundaries, which are
/// also handles as interior boundaries.
BoundaryType type;
int maxLevel;
};
......
AMDiS/src/parallel/ElementObjectDatabase.cc
View file @ 2c1211fd
......@@ -388,6 +388,10 @@ namespace AMDiS {
vertexInRank.clear();
for (map<DegreeOfFreedom, vector<ElementObjectData> >::iterator it = vertexElements.begin();
it != vertexElements.end(); ++it) {
vector<std::set<int> > ranksInLevel;
ranksInLevel.resize(nLevel);
for (vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
......@@ -400,7 +404,18 @@ namespace AMDiS {
vertexOwner[it->first][level] =
std::max(vertexOwner[it->first][level], levelId);
}
ranksInLevel[0].insert(elementInRank);
for (int level = 1; level < nLevel; level++)
ranksInLevel[level].insert(levelData.getLevelId(level, elementInRank));
}
int maxLevel = 0;
for (int level = 1; level < nLevel; level++)
if (ranksInLevel[level].size() > 1)
maxLevel = level;
vertexMaxLevel[it->first] = maxLevel;
}
......@@ -408,6 +423,10 @@ namespace AMDiS {
edgeInRank.clear();
for (map<DofEdge, vector<ElementObjectData> >::iterator it = edgeElements.begin();
it != edgeElements.end(); ++it) {
vector<std::set<int> > ranksInLevel;
ranksInLevel.resize(nLevel);
for (vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
......@@ -420,7 +439,18 @@ namespace AMDiS {
edgeOwner[it->first][level] =
std::max(edgeOwner[it->first][level], levelId);
}
ranksInLevel[0].insert(elementInRank);
for (int level = 1; level < nLevel; level++)
ranksInLevel[level].insert(levelData.getLevelId(level, elementInRank));
}
int maxLevel = 0;
for (int level = 1; level < nLevel; level++)
if (ranksInLevel[level].size() > 1)
maxLevel = level;
edgeMaxLevel[it->first] = maxLevel;
}
......@@ -428,6 +458,10 @@ namespace AMDiS {
faceInRank.clear();
for (map<DofFace, vector<ElementObjectData> >::iterator it = faceElements.begin();
it != faceElements.end(); ++it) {
vector<std::set<int> > ranksInLevel;
ranksInLevel.resize(nLevel);
for (vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
......@@ -440,7 +474,18 @@ namespace AMDiS {
faceOwner[it->first][level] =
std::max(faceOwner[it->first][level], levelId);
}
ranksInLevel[0].insert(elementInRank);
for (int level = 1; level < nLevel; level++)
ranksInLevel[level].insert(levelData.getLevelId(level, elementInRank));
}
int maxLevel = 0;
for (int level = 1; level < nLevel; level++)
if (ranksInLevel[level].size() > 1)
maxLevel = level;
faceMaxLevel[it->first] = maxLevel;
}
}
......
AMDiS/src/parallel/ElementObjectDatabase.h
View file @ 2c1211fd
......@@ -279,6 +279,26 @@ namespace AMDiS {
}
}
/// Returns the rank owner of the current iterator position.
int getIterateMaxLevel()
{
switch (iterGeoPos) {
case VERTEX:
return vertexMaxLevel[vertexIter->first];
break;
case EDGE:
return edgeMaxLevel[edgeIter->first];
break;
case FACE:
return faceMaxLevel[faceIter->first];
break;
default:
ERROR_EXIT("Should not happen!\n");
// Will never be reached, just to avoid compiler warnings.
return -1;
}
}
/// Returns the rank owner of a vertex DOF.
int getOwner(DegreeOfFreedom vertex, int level)
......@@ -554,6 +574,13 @@ namespace AMDiS {
/// Defines for all faces the rank that ownes this face.
map<DofFace, LevelRank> faceOwner;
map<DegreeOfFreedom, int> vertexMaxLevel;
map<DofEdge, int> edgeMaxLevel;
map<DofFace, int> faceMaxLevel;
/// Defines to each vertex DOF a map that maps to each rank number the element
/// objects that have this vertex DOF in common.
......
AMDiS/src/parallel/InteriorBoundary.cc
View file @ 2c1211fd
......@@ -18,11 +18,11 @@
namespace AMDiS {
AtomicBoundary& InteriorBoundary::getNewAtomic(int level, int rank)
AtomicBoundary& InteriorBoundary::getNewAtomic(int rank)
{
int size = boundary[level][rank].size();
boundary[level][rank].resize(size + 1);
return boundary[level][rank][size];
int size = boundary[rank].size();
boundary[rank].resize(size + 1);
return boundary[rank][size];
}
......@@ -34,20 +34,20 @@ namespace AMDiS {
return false;
for (unsigned int level = 0; level < boundary.size(); level++) {
for (RankToBoundMap::const_iterator it = boundary[level].begin();
it != boundary[level].end(); ++it) {
if (other2.boundary[level].count(it->first) == 0)
for (RankToBoundMap::const_iterator it = boundary.begin();
it != boundary.end(); ++it) {
if (other2.boundary.count(it->first) == 0)
return false;
if (other2.boundary[level][it->first].size() != it->second.size())
if (other2.boundary[it->first].size() != it->second.size())
return false;
for (unsigned int i = 0; i < it->second.size(); i++) {
std::vector<AtomicBoundary>::iterator bIt =
find(other2.boundary[level][it->first].begin(),
other2.boundary[level][it->first].end(), it->second[i]);
find(other2.boundary[it->first].begin(),
other2.boundary[it->first].end(), it->second[i]);
if (bIt == other2.boundary[level][it->first].end())
if (bIt == other2.boundary[it->first].end())
return false;
}
}
......@@ -57,14 +57,11 @@ namespace AMDiS {
}
void InteriorBoundary::reset(int level)
void InteriorBoundary::clear()
{
FUNCNAME("InteriorBoundary::reset()");
nLevel = level;
FUNCNAME("InteriorBoundary::clear()");
boundary.clear();
boundary.resize(nLevel);
}
......
AMDiS/src/parallel/InteriorBoundary.h
View file @ 2c1211fd
......@@ -45,49 +45,36 @@ namespace AMDiS {
/// Iterator for the interior boundary object.
class iterator {
public:
iterator(InteriorBoundary &b)
iterator(InteriorBoundary &b, int traverseLevel = 0)
: bound(b),
level(0)
level(traverseLevel)
{
reset();
}
iterator(InteriorBoundary &b, int level)
: bound(b),
level(level)
{
TEST_EXIT_DBG(level < bound.boundary.size())
("Should not happen!\n");
reset();
}
/// Set the iterator to the first position.
void reset()
{
mapIt = bound.boundary[level].begin();
mapIt = bound.boundary.begin();
nextNonempty();
if (mapIt != bound.boundary[level].end())
vecIt = mapIt->second.begin();
}
/// Test if iterator is at the final position.
bool end() const
{
return (mapIt == bound.boundary[level].end());
return (mapIt == bound.boundary.end());
}
/// Move iterator to the next position.
void operator++()
{
++vecIt;
do {
++vecIt;
} while (vecIt->maxLevel < level && vecIt != mapIt->second.end());
if (vecIt == mapIt->second.end()) {
++mapIt;
nextNonempty();
if (mapIt != bound.boundary[level].end())
vecIt = mapIt->second.begin();
}
}
......@@ -105,9 +92,6 @@ namespace AMDiS {
{
++mapIt;
nextNonempty();
if (mapIt != bound.boundary[level].end())
vecIt = mapIt->second.begin();
}
inline int getRank()
......@@ -119,14 +103,33 @@ namespace AMDiS {
inline void nextNonempty()
{
if (mapIt == bound.boundary[level].end())
return;
while (mapIt->second.size() == 0) {
do {
// Return, we are at the end.
if (mapIt == bound.boundary.end())
return;
// Search for the next non empty boundary map.
while (mapIt->second.size() == 0) {
++mapIt;
if (mapIt == bound.boundary.end())
return;
}
vecIt = mapIt->second.begin();
// Search for the next atomic boundary on the mesh level
while (vecIt->maxLevel < level && vecIt != mapIt->second.end())
++vecIt;
// If vector iterator is not at the end, we have found one and
// can return.
if (vecIt != mapIt->second.end())
return;
// In this case, no boundary on the given level is found, continue
// with next rank.
++mapIt;
if (mapIt == bound.boundary[level].end())
return;
}
} while (true);
}
protected:
......@@ -140,15 +143,9 @@ namespace AMDiS {
};
public:
InteriorBoundary(int l = 1)
: nLevel(l)
{
boundary.resize(nLevel);
}
void clear();
void reset(int nLevel);
AtomicBoundary& getNewAtomic(int level, int rank);
AtomicBoundary& getNewAtomic(int rank);
/// Writes this object to a file.
void serialize(ostream &out);
......@@ -166,10 +163,7 @@ namespace AMDiS {
void deserializeExcludeList(istream &in, ExcludeList &list);
public:
vector<RankToBoundMap> boundary;
protected:
int nLevel;
RankToBoundMap boundary;
};
}
......
AMDiS/src/parallel/MeshDistributor.cc
View file @ 2c1211fd
......@@ -1576,19 +1576,9 @@ namespace AMDiS {
// === Clear all relevant data structures. ===
int nLevel = levelData.getLevelNumber();
rankIntBoundary.reset(nLevel);
otherIntBoundary.reset(nLevel);
periodicBoundary.reset(nLevel);
for (int level = 0; level < nLevel; level++)
createBoundaryData(level);
}
void MeshDistributor::createBoundaryData(int level)
{
FUNCNAME("MeshDistributor::createBoundaryData()");
rankIntBoundary.clear();
otherIntBoundary.clear();
periodicBoundary.clear();
// === Create interior boundary data structure. ===
......@@ -1600,10 +1590,12 @@ namespace AMDiS {
if (!(objData.count(mpiRank) && objData.size() > 1))
continue;
int owner = elObjDb.getIterateOwner(level);
int owner = elObjDb.getIterateOwner(0);
ElementObjectData& rankBoundEl = objData[mpiRank];
AtomicBoundary bound;
AtomicBoundary bound;
bound.maxLevel = elObjDb.getIterateMaxLevel();
bound.rankObj.el = macroElIndexMap[rankBoundEl.elIndex];
bound.rankObj.elIndex = rankBoundEl.elIndex;
bound.rankObj.elType = macroElIndexTypeMap[rankBoundEl.elIndex];
......@@ -1634,7 +1626,7 @@ namespace AMDiS {
bound.type = INTERIOR;
AtomicBoundary& b = rankIntBoundary.getNewAtomic(level, it2->first);
AtomicBoundary& b = rankIntBoundary.getNewAtomic(it2->first);
b = bound;
if (geoIndex == EDGE)
b.neighObj.reverseMode =
......@@ -1658,7 +1650,7 @@ namespace AMDiS {
bound.type = INTERIOR;
AtomicBoundary& b = otherIntBoundary.getNewAtomic(level, owner);
AtomicBoundary& b = otherIntBoundary.getNewAtomic(owner);
b = bound;
if (geoIndex == EDGE)
b.rankObj.reverseMode =
......@@ -1707,11 +1699,11 @@ namespace AMDiS {
bound.rankObj.ithObj == 1 ||
bound.rankObj.elIndex == 78 &&
bound.rankObj.ithObj == 2) {
AtomicBoundary& b = periodicBoundary.getNewAtomic(level, otherElementRank);
AtomicBoundary& b = periodicBoundary.getNewAtomic(otherElementRank);
b = bound;
}
} else {
AtomicBoundary& b = periodicBoundary.getNewAtomic(level, otherElementRank);
AtomicBoundary& b = periodicBoundary.getNewAtomic(otherElementRank);
b = bound;
}
}
......@@ -1746,7 +1738,7 @@ namespace AMDiS {
bound.type = it->second;
AtomicBoundary& b = periodicBoundary.getNewAtomic(level, otherElementRank);
AtomicBoundary& b = periodicBoundary.getNewAtomic(otherElementRank);
b = bound;
if (mpiRank > otherElementRank)
......@@ -1792,7 +1784,7 @@ namespace AMDiS {
bound.type = it->second;
AtomicBoundary& b = periodicBoundary.getNewAtomic(level, otherElementRank);
AtomicBoundary& b = periodicBoundary.getNewAtomic(otherElementRank);
b = bound;
if (mpiRank > otherElementRank)
......@@ -1811,8 +1803,8 @@ namespace AMDiS {
// === share the bounday. ===
StdMpi<vector<AtomicBoundary> > stdMpi(mpiComm);
stdMpi.send(rankIntBoundary.boundary[level]);
stdMpi.recv(otherIntBoundary.boundary[level]);
stdMpi.send(rankIntBoundary.boundary);
stdMpi.recv(otherIntBoundary.boundary);
stdMpi.startCommunication();
......@@ -1821,8 +1813,8 @@ namespace AMDiS {
// === the same order. If not, the atomic boundaries are swaped to the ===
// === correct order. ===
for (RankToBoundMap::iterator rankIt = otherIntBoundary.boundary[level].begin();
rankIt != otherIntBoundary.boundary[level].end(); ++rankIt) {
for (RankToBoundMap::iterator rankIt = otherIntBoundary.boundary.begin();
rankIt != otherIntBoundary.boundary.end(); ++rankIt) {
// === We have received from rank "rankIt->first" the ordered list of ===
// === element indices. Now, we have to sort the corresponding list in ===
......@@ -1856,29 +1848,28 @@ namespace AMDiS {
// === Do the same for the periodic boundaries. ===
if (periodicBoundary.boundary[level].size() > 0) {
if (periodicBoundary.boundary.size() > 0) {
stdMpi.clear();
InteriorBoundary sendBounds(levelData.getLevelNumber());
InteriorBoundary recvBounds(levelData.getLevelNumber());
for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary[level].begin();
rankIt != periodicBoundary.boundary[level].end(); ++rankIt) {
InteriorBoundary sendBounds, recvBounds;
for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin();
rankIt != periodicBoundary.boundary.end(); ++rankIt) {
if (rankIt->first == mpiRank)
continue;
if (rankIt->first < mpiRank)
sendBounds.boundary[level][rankIt->first] = rankIt->second;
sendBounds.boundary[rankIt->first] = rankIt->second;
else
recvBounds.boundary[level][rankIt->first] = rankIt->second;
recvBounds.boundary[rankIt->first] = rankIt->second;
}
stdMpi.send(sendBounds.boundary[level]);
stdMpi.recv(recvBounds.boundary[level]);
stdMpi.send(sendBounds.boundary);
stdMpi.recv(recvBounds.boundary);
stdMpi.startCommunication();
for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary[level].begin();
rankIt != periodicBoundary.boundary[level].end(); ++rankIt) {
for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin();
rankIt != periodicBoundary.boundary.end(); ++rankIt) {
if (rankIt->first <= mpiRank)
continue;
......@@ -1889,12 +1880,12 @@ namespace AMDiS {
BoundaryObject &recvNeighObj =
stdMpi.getRecvData()[rankIt->first][j].neighObj;
if (periodicBoundary.boundary[level][rankIt->first][j].neighObj != recvRankObj ||
periodicBoundary.boundary[level][rankIt->first][j].rankObj != recvNeighObj) {
if (periodicBoundary.boundary[rankIt->first][j].neighObj != recvRankObj ||
periodicBoundary.boundary[rankIt->first][j].rankObj != recvNeighObj) {
unsigned int k = j + 1;
for (; k < rankIt->second.size(); k++)
if (periodicBoundary.boundary[level][rankIt->first][k].neighObj == recvRankObj &&
periodicBoundary.boundary[level][rankIt->first][k].rankObj == recvNeighObj)
if (periodicBoundary.boundary[rankIt->first][k].neighObj == recvRankObj &&
periodicBoundary.boundary[rankIt->first][k].rankObj == recvNeighObj)
break;
// The element must always be found, because the list is just in
......@@ -1908,7 +1899,7 @@ namespace AMDiS {
}
}
}
} // periodicBoundary.boundary[level].size() > 0
} // periodicBoundary.boundary.size() > 0
}
......@@ -2112,8 +2103,6 @@ namespace AMDiS {
{
FUNCNAME("MeshDistributor::createPeriodicMap()");
TEST_EXIT(levelData.getLevelNumber() == 1)("Not yet implemented for multilevel!\n");
// Clear all periodic DOF mappings calculated before. We do it from scratch.
periodicDofs.init(levelData.getLevelNumber());
periodicMap.clear();
......@@ -2122,7 +2111,7 @@ namespace AMDiS {
// periodicMap must be still cleared before: if we do repartitioning and
// there were periodic boundaries in subdomain before and after repartitioning
// there are no more periodic boundaries.
if (periodicBoundary.boundary[0].size() == 0)
if (periodicBoundary.boundary.size() == 0)
return;
for (unsigned int i = 0; i < feSpaces.size(); i++)
......@@ -2134,8 +2123,6 @@ namespace AMDiS {
{
FUNCNAME("MeshDistributor::createPeriodicMap()");
TEST_EXIT(levelData.getLevelNumber() == 1)("No yet implemented for multilevel stuff!\n");
StdMpi<vector<int> > stdMpi(mpiComm, false);
// === Each rank traverse its periodic boundaries and sends the DOF ===
......@@ -2143,8 +2130,8 @@ namespace AMDiS {
map<int, vector<int> > rankToDofType;
for (RankToBoundMap::iterator it = periodicBoundary.boundary[0].begin();
it != periodicBoundary.boundary[0].end(); ++it) {
for (RankToBoundMap::iterator it = periodicBoundary.boundary.begin();
it != periodicBoundary.boundary.end(); ++it) {
if (it->first == mpiRank) {
// Here we have a periodic boundary within rank's subdomain. So we can
......@@ -2212,8 +2199,8 @@ namespace AMDiS {
// === DOFs from the other ranks. ===
for (RankToBoundMap::iterator it = periodicBoundary.boundary[0].begin();
it != periodicBoundary.boundary[0].end(); ++it) {
for (RankToBoundMap::iterator it = periodicBoundary.boundary.begin();
it != periodicBoundary.boundary.end(); ++it) {
DofContainer& dofs = periodicDofs.getDofContainer(it->first, feSpace);
vector<int>& types = rankToDofType[it->first];
......@@ -2236,8 +2223,8 @@ namespace AMDiS {
StdMpi<PeriodicDofMap> stdMpi2(mpiComm);
for (RankToBoundMap::iterator it = periodicBoundary.boundary[0].begin();
it != periodicBoundary.boundary[0].end(); ++it) {
for (RankToBoundMap::iterator it = periodicBoundary.boundary.begin();
it != periodicBoundary.boundary.end(); ++it) {
if (it->first == mpiRank)
continue;
......
AMDiS/src/parallel/MeshDistributor.h
View file @ 2c1211fd
......@@ -297,8 +297,6 @@ namespace AMDiS {
void createBoundaryData();
void createBoundaryData(int level);
void createBoundaryDofs();
void createBoundaryDofs(const FiniteElemSpace *feSpace, int level);
......
AMDiS/src/parallel/ParallelDebug.cc
View file @ 2c1211fd
......@@ -39,8 +39,8 @@ namespace AMDiS {
// === Send rank's boundary information. ===
for (RankToBoundMap::iterator rankIt = pdb.rankIntBoundary.boundary[0].begin();
rankIt != pdb.rankIntBoundary.boundary[0].end(); ++rankIt) {
for (RankToBoundMap::iterator rankIt = pdb.rankIntBoundary.boundary.begin();
rankIt != pdb.rankIntBoundary.boundary.end(); ++rankIt) {
int nSendInt = rankIt->second.size();
int* buffer = new int[nSendInt];
......@@ -56,8 +56,8 @@ namespace AMDiS {
// === Receive information from other ranks about the interior boundaries. ====
for (RankToBoundMap::iterator rankIt = pdb.otherIntBoundary.boundary[0].begin();
rankIt != pdb.otherIntBoundary.boundary