Skip to content
GitLab
Commit e81c50d2 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Work on multilevel FETI-DP, have to go home ....

parent 3e2d84dc
Hide whitespace changes
Inline Side-by-side
AMDiS/src/parallel/InteriorBoundary.h
View file @ e81c50d2
......@@ -109,8 +109,15 @@ namespace AMDiS {
vecIt = mapIt->second.begin();
}
int getRank()
inline int getRank()
{
if (level > 0) {
int r = levelData->mapRank(mapIt->first, level - 1, level);
TEST_EXIT_DBG(r >= 0)
("Mapping rank %d from level % to level %d does not work!\n",
mapIt->first, level - 1, level);
}
return mapIt->first;
}
......@@ -125,7 +132,7 @@ namespace AMDiS {
TEST_EXIT_DBG(levelData)("No mesh level data object defined!\n");
TEST_EXIT_DBG(level == 1)("Only 2-level method supported!\n");
while (levelData->rankInSubdomain(mapIt->first, level) ||
while (!levelData->rankInSubdomain(mapIt->first, level) ||
mapIt->second.size() == 0) {
++mapIt;
if (mapIt == bound.boundary.end())
......
AMDiS/src/parallel/MeshDistributor.cc
View file @ e81c50d2
......@@ -1967,8 +1967,9 @@ namespace AMDiS {
recvDofs.init(nLevels);
boundaryDofInfo.resize(nLevels);
dofMap.init(mpiComm, feSpaces, feSpaces, true, true);
dofMap.init(levelData, feSpaces, feSpaces, true, true);
dofMap.setDofComm(sendDofs, recvDofs);
dofMap.clear();
for (unsigned int i = 0; i < feSpaces.size(); i++)
updateLocalGlobalNumbering(feSpaces[i]);
......@@ -1977,11 +1978,16 @@ namespace AMDiS {
#if (DEBUG != 0)
MSG("------------- Debug information -------------\n");
for (unsigned int i = 0; i < feSpaces.size(); i++) {
MSG("FE space %d:\n", i);
MSG(" nRankDofs = %d\n", dofMap[feSpaces[i]].nRankDofs[0]);
MSG(" nOverallDofs = %d\n", dofMap[feSpaces[i]].nOverallDofs[0]);
MSG(" rStartDofs = %d\n", dofMap[feSpaces[i]].rStartDofs[0]);
MSG("| number of levels: %d\n", nLevels);
MSG("| number of FE spaces: %d\n", feSpaces.size());
for (int level = 0; level < nLevels; level++) {
for (unsigned int i = 0; i < feSpaces.size(); i++) {
MSG("| level = %d FE space = %d:\n", level, i);
MSG("| nRankDofs = %d\n", dofMap[feSpaces[i]].nRankDofs[level]);
MSG("| nOverallDofs = %d\n", dofMap[feSpaces[i]].nOverallDofs[level]);
MSG("| rStartDofs = %d\n", dofMap[feSpaces[i]].rStartDofs[level]);
}
}
stringstream oss;
......@@ -2006,8 +2012,6 @@ namespace AMDiS {
mesh->dofCompress();
dofMap.clear();
// === Get all DOFs in ranks partition. ===
std::set<const DegreeOfFreedom*> rankDofSet;
......@@ -2019,27 +2023,33 @@ namespace AMDiS {
// === Traverse interior boundaries and get all DOFs on them. ===
int nLevels = levelData.getLevelNumber();
for (int i = 0; i < nLevels; i++)
createBoundaryDofs(feSpace, i);
// All DOFs that must be received are DOFs not owned by rank and have
// therefore to be removed from the set 'rankDofs'.
for (DofComm::Iterator it(recvDofs, feSpace); !it.end(); it.nextRank()) {
for (; !it.endDofIter(); it.nextDof()) {
DofContainer::iterator eraseIt =
find(rankDofs.begin(), rankDofs.end(), it.getDof());
if (eraseIt != rankDofs.end())
rankDofs.erase(eraseIt);
for (int level = 0; level < nLevels; level++) {
createBoundaryDofs(feSpace, level);
// All DOFs that must be received are DOFs not owned by rank and have
// therefore to be removed from the set 'rankDofs'.
DofContainerSet nonRankDofs;
for (DofComm::Iterator it(recvDofs, level, feSpace); !it.end(); it.nextRank()) {
for (; !it.endDofIter(); it.nextDof()) {
#if 0
DofContainer::iterator eraseIt =
find(rankDofs.begin(), rankDofs.end(), it.getDof());
if (eraseIt != rankDofs.end())
rankDofs.erase(eraseIt);
#endif
nonRankDofs.insert(it.getDof());
}
}
for (unsigned int i = 0; i < rankDofs.size(); i++)
if (nonRankDofs.count(rankDofs[i]) == 0)
dofMap[feSpace].insertRankDof(level, *(rankDofs[i]));
for (DofComm::Iterator it(recvDofs, level, feSpace); !it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
dofMap[feSpace].insertNonRankDof(level, it.getDofIndex());
}
for (unsigned int i = 0; i < rankDofs.size(); i++)
dofMap[feSpace].insertRankDof(*(rankDofs[i]));
for (DofComm::Iterator it(recvDofs, feSpace); !it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
dofMap[feSpace].insert(it.getDofIndex());
// === Update DOF admins due to new number of DOFs. ===
lastMeshChangeIndex = mesh->getChangeIndex();
......
AMDiS/src/parallel/ParallelDofMapping.cc
View file @ e81c50d2
......@@ -19,6 +19,7 @@ namespace AMDiS {
void FeSpaceDofMap::clear()
{
int nLevel = levelData->getLevelNumber();
dofMap.clear();
dofMap.resize(nLevel);
......@@ -38,6 +39,7 @@ namespace AMDiS {
{
FUNCNAME("FeSpaceDofMap::update()");
int nLevel = levelData->getLevelNumber();
for (int i = 0; i < nLevel; i++) {
// === Compute local indices for all rank owned DOFs. ===
......@@ -50,7 +52,8 @@ namespace AMDiS {
nOverallDofs[i] = 0;
rStartDofs[i] = 0;
mpi::getDofNumbering(mpiComm, nRankDofs[i], rStartDofs[i], nOverallDofs[i]);
mpi::getDofNumbering(levelData->getMpiComm(i),
nRankDofs[i], rStartDofs[i], nOverallDofs[i]);
// === If required, compute also the global indices. ===
......@@ -83,7 +86,7 @@ namespace AMDiS {
// === Send all global indices of DOFs that are owned by the rank to all ===
// === other ranks that also include this DOF. ===
StdMpi<vector<int> > stdMpi(mpiComm);
StdMpi<vector<int> > stdMpi(levelData->getMpiComm(level));
for (DofComm::Iterator it(*sendDofs, level, feSpace);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
......@@ -127,7 +130,7 @@ namespace AMDiS {
}
void ParallelDofMapping::init(MPI::Intracomm m,
void ParallelDofMapping::init(MeshLevelData &ldata,
vector<const FiniteElemSpace*> &fe,
vector<const FiniteElemSpace*> &uniqueFe,
bool needGlobalMapping,
......@@ -135,7 +138,7 @@ namespace AMDiS {
{
FUNCNAME("ParallelDofMapping::init()");
mpiComm = m;
levelData = &ldata;
feSpaces = fe;
feSpacesUnique = uniqueFe;
hasNonLocalDofs = bNonLocalDofs;
......@@ -160,6 +163,12 @@ namespace AMDiS {
it != feSpacesUnique.end(); ++it)
data[*it].clear();
int nLevel = levelData->getLevelNumber();
nRankDofs.resize(nLevel);
nLocalDofs.resize(nLevel);
nOverallDofs.resize(nLevel);
rStartDofs.resize(nLevel);
for (int i = 0; i < nLevel; i++) {
nRankDofs[i] = -1;
nLocalDofs[i] = -1;
......@@ -183,7 +192,7 @@ namespace AMDiS {
it != feSpacesUnique.end(); ++it)
data[*it].setDofComm(pSend, pRecv);
}
void ParallelDofMapping::addFeSpace(const FiniteElemSpace* feSpace)
{
......@@ -192,7 +201,7 @@ namespace AMDiS {
if (data.count(feSpace))
data.find(feSpace)->second.clear();
else
data.insert(make_pair(feSpace, FeSpaceDofMap(mpiComm)));
data.insert(make_pair(feSpace, FeSpaceDofMap(levelData)));
data.find(feSpace)->second.setFeSpace(feSpace);
}
......@@ -263,6 +272,7 @@ namespace AMDiS {
it != feSpacesUnique.end(); ++it)
data[*it].update();
int nLevel = levelData->getLevelNumber();
for (int i = 0; i < nLevel; i++) {
// Compute all numbers from this mappings.
......@@ -338,7 +348,7 @@ namespace AMDiS {
// === Communicate the matrix indices for all DOFs that are on some ===
// === interior boundaries. ===
StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiComm);
StdMpi<vector<DegreeOfFreedom> > stdMpi(levelData->getMpiComm(level));
for (DofComm::Iterator it(*sendDofs, level, feSpaces[i]);
!it.end(); it.nextRank()) {
vector<DegreeOfFreedom> sendGlobalDofs;
......
AMDiS/src/parallel/ParallelDofMapping.h
View file @ e81c50d2
......@@ -24,6 +24,7 @@
#include <map>
#include <set>
#include "parallel/DofComm.h"
#include "parallel/MeshLevelData.h"
#include "parallel/MpiHelper.h"
#include "parallel/ParallelTypes.h"
#include "parallel/StdMpi.h"
......@@ -102,12 +103,11 @@ namespace AMDiS {
}
/// This is the only valid constructur to be used.
FeSpaceDofMap(MPI::Intracomm m)
: mpiComm(m),
FeSpaceDofMap(MeshLevelData* ld)
: levelData(ld),
sendDofs(NULL),
recvDofs(NULL),
feSpace(NULL),
nLevel(1),
dofMap(1),
needGlobalMapping(false),
hasNonLocalDofs(false),
......@@ -133,29 +133,29 @@ namespace AMDiS {
/// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
/// the rank.
void insertRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
void insertRankDof(int level, DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
{
FUNCNAME("FeSpaceDofMap::insertRankDof()");
TEST_EXIT_DBG(dofMap[0].count(dof0) == 0)("Should not happen!\n");
TEST_EXIT_DBG(dofMap[level].count(dof0) == 0)("Should not happen!\n");
dofMap[0][dof0].local = dof1;
nLocalDofs[0]++;
dofMap[level][dof0].local = dof1;
nLocalDofs[level]++;
if (dof1 != -1)
nRankDofs[0]++;
nRankDofs[level]++;
}
/// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
/// but is owned by a different rank, thus it is part of an interior boundary.
void insert(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
void insertNonRankDof(int level, DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
{
FUNCNAME("FeSpaceDofMap::insert()");
FUNCNAME("FeSpaceDofMap::insertNonRankDof()");
TEST_EXIT_DBG(dofMap[0].count(dof0) == 0)("Should not happen!\n");
TEST_EXIT_DBG(dofMap[level].count(dof0) == 0)("Should not happen!\n");
dofMap[0][dof0].local = dof1;
nLocalDofs[0]++;
nonRankDofs[0].insert(dof0);
dofMap[level][dof0].local = dof1;
nLocalDofs[level]++;
nonRankDofs[level].insert(dof0);
}
/// Checks if a given DOF is in the DOF mapping.
......@@ -222,8 +222,7 @@ namespace AMDiS {
void computeNonLocalIndices(int level);
private:
/// MPI communicator object;
MPI::Intracomm mpiComm;
MeshLevelData *levelData;
/// DOF communicators for all DOFs on interior boundaries.
DofComm *sendDofs, *recvDofs;
......@@ -232,9 +231,6 @@ namespace AMDiS {
/// correct DOF communicator in \ref sendDofs and \ref recvDofs.
const FiniteElemSpace *feSpace;
/// Number of mesh levels.
int nLevel;
/// Mapping data from DOF indices to local and global indices.
vector<DofMap> dofMap;
......@@ -264,12 +260,12 @@ namespace AMDiS {
{
public:
ParallelDofMapping()
: sendDofs(NULL),
: levelData(NULL),
sendDofs(NULL),
recvDofs(NULL),
hasNonLocalDofs(false),
needMatIndex(false),
needMatIndexFromGlobal(false),
nLevel(1),
feSpaces(0),
nRankDofs(1),
nLocalDofs(1),
......@@ -295,7 +291,7 @@ namespace AMDiS {
* \param[in] bNonLocalDofs If true, at least one rank's mapping con-
* taines DOFs that are not owend by the rank.
*/
void init(MPI::Intracomm m,
void init(MeshLevelData& mld,
vector<const FiniteElemSpace*> &fe,
vector<const FiniteElemSpace*> &uniqueFe,
bool needGlobalMapping,
......@@ -308,6 +304,12 @@ namespace AMDiS {
/// about DOFs that are on interior boundaries.
void setDofComm(DofComm &pSend, DofComm &pRecv);
void setComputeMatIndex(bool b, bool global = false)
{
needMatIndex = b;
needMatIndexFromGlobal = global;
}
/// Access the DOF mapping for a given FE space.
inline FeSpaceDofMap& operator[](const FiniteElemSpace* feSpace)
{
......@@ -389,12 +391,6 @@ namespace AMDiS {
ERROR_EXIT("MUST BE IMPLEMENTED!\n");
}
void setComputeMatIndex(bool b, bool global = false)
{
needMatIndex = b;
needMatIndexFromGlobal = global;
}
/// Compute local and global matrix indices.
void computeMatIndex(bool globalIndex, int level);
......@@ -415,8 +411,7 @@ namespace AMDiS {
int computeStartDofs(int level);
private:
/// MPI communicator object;
MPI::Intracomm mpiComm;
MeshLevelData *levelData;
/// DOF communicators for all DOFs on interior boundaries.
DofComm *sendDofs, *recvDofs;
......@@ -436,8 +431,6 @@ namespace AMDiS {
/// on global ones, otherwise on local DOF indices.
bool needMatIndexFromGlobal;
int nLevel;
/// Maps from FE space pointers to DOF mappings.
map<const FiniteElemSpace*, FeSpaceDofMap> data;
......
AMDiS/src/parallel/PetscSolverFeti.cc
View file @ e81c50d2
......@@ -228,13 +228,13 @@ namespace AMDiS {
TEST_EXIT_DBG(meshLevel + 1 == meshDistributor->getMeshLevelData().getLevelNumber())
("Mesh hierarchy does not contain %d levels!\n", meshLevel + 1);
MeshLevelData& levelData = meshDistributor->getMeshLevelData();
if (subDomainSolver == NULL) {
if (meshLevel == 0) {
subDomainSolver =
new SubDomainSolver(meshDistributor, mpiComm, mpiSelfComm);
} else {
MeshLevelData& levelData = meshDistributor->getMeshLevelData();
subDomainSolver =
new SubDomainSolver(meshDistributor,
levelData.getMpiComm(meshLevel - 1),
......@@ -243,24 +243,29 @@ namespace AMDiS {
}
}
primalDofMap.init(mpiComm, feSpaces, meshDistributor->getFeSpaces(),
primalDofMap.init(levelData,
feSpaces, meshDistributor->getFeSpaces(),
true, true);
primalDofMap.setComputeMatIndex(true);
dualDofMap.init(mpiComm, feSpaces, meshDistributor->getFeSpaces(),
dualDofMap.init(levelData,
feSpaces, meshDistributor->getFeSpaces(),
false, false);
dualDofMap.setComputeMatIndex(true);
lagrangeMap.init(mpiComm, feSpaces, meshDistributor->getFeSpaces(),
lagrangeMap.init(levelData,
feSpaces, meshDistributor->getFeSpaces(),
true, true);
lagrangeMap.setComputeMatIndex(true);
localDofMap.init(mpiComm, feSpaces, meshDistributor->getFeSpaces(),
localDofMap.init(levelData,
feSpaces, meshDistributor->getFeSpaces(),
false, false);
localDofMap.setComputeMatIndex(true);
if (fetiPreconditioner == FETI_DIRICHLET) {
interiorDofMap.init(mpiComm, feSpaces, meshDistributor->getFeSpaces(),
interiorDofMap.init(levelData,
feSpaces, meshDistributor->getFeSpaces(),
false, false);
interiorDofMap.setComputeMatIndex(true);
}
......@@ -348,9 +353,9 @@ namespace AMDiS {
for (DofIndexSet::iterator it = primals.begin(); it != primals.end(); ++it)
if (meshDistributor->getDofMap()[feSpace].isRankDof(*it))
primalDofMap[feSpace].insertRankDof(*it);
primalDofMap[feSpace].insertRankDof(0, *it);
else
primalDofMap[feSpace].insert(*it);
primalDofMap[feSpace].insertNonRankDof(0, *it);
}
......@@ -366,7 +371,7 @@ namespace AMDiS {
for (DofContainer::iterator it = allBoundaryDofs.begin();
it != allBoundaryDofs.end(); ++it)
if (!isPrimal(feSpace, **it))
dualDofMap[feSpace].insertRankDof(**it);
dualDofMap[feSpace].insertRankDof(0, **it);
}
......@@ -435,11 +440,11 @@ namespace AMDiS {
for (DofMap::iterator it = dualMap.begin(); it != dualMap.end(); ++it) {
if (meshDistributor->getDofMap()[feSpace].isRankDof(it->first)) {
lagrangeMap[feSpace].insertRankDof(it->first, nRankLagrange);
lagrangeMap[feSpace].insertRankDof(0, it->first, nRankLagrange);
int degree = boundaryDofRanks[feSpace][it->first].size();
nRankLagrange += (degree * (degree - 1)) / 2;
} else {
lagrangeMap[feSpace].insert(it->first);
lagrangeMap[feSpace].insertNonRankDof(0, it->first);
}
}
lagrangeMap[feSpace].nRankDofs[0] = nRankLagrange;
......@@ -461,10 +466,10 @@ namespace AMDiS {
if (admin->isDofFree(i) == false &&
isPrimal(feSpace, i) == false &&
dualDofMap[feSpace].isSet(i) == false) {
localDofMap[feSpace].insertRankDof(i, nLocalInterior);
localDofMap[feSpace].insertRankDof(0, i, nLocalInterior);
if (fetiPreconditioner != FETI_NONE)
interiorDofMap[feSpace].insertRankDof(i, nLocalInterior);
interiorDofMap[feSpace].insertRankDof(0, i, nLocalInterior);
nLocalInterior++;
}
......@@ -474,7 +479,7 @@ namespace AMDiS {
for (DofMap::iterator it = dualDofMap[feSpace].getMap(0).begin();
it != dualDofMap[feSpace].getMap(0).end(); ++it)
localDofMap[feSpace].insertRankDof(it->first);
localDofMap[feSpace].insertRankDof(0, it->first);
}
......
AMDiS/src/parallel/SubDomainSolver.cc
View file @ e81c50d2
......@@ -262,6 +262,8 @@ namespace AMDiS {
{
FUNCNAME("SubDomainSolver::solveGlobal()");
ERROR_EXIT("BLUB!\n");
Vec tmp;
VecCreateSeq(PETSC_COMM_SELF, interiorMap->getRankDofs(level), &tmp);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment