From ff250c55cd9a2a6eb4c38b7bb77a0af8939e4f02 Mon Sep 17 00:00:00 2001
From: Thomas Witkowski <thomas.witkowski@gmx.de>
Date: Tue, 30 Oct 2012 11:43:03 +0000
Subject: [PATCH] Some performance issues.
---
AMDiS/src/Global.cc | 13 ++
AMDiS/src/Marker.h | 34 ++----
AMDiS/src/parallel/MeshDistributor.cc | 154 ++++++++++++++----------
AMDiS/src/parallel/MeshDistributor.h | 6 +
AMDiS/src/parallel/ParallelDofMapping.h | 8 +-
5 files changed, 128 insertions(+), 87 deletions(-)
diff --git a/AMDiS/src/Global.cc b/AMDiS/src/Global.cc
index ff24e8d9..c570cd82 100644
--- a/AMDiS/src/Global.cc
+++ b/AMDiS/src/Global.cc
@@ -200,6 +200,13 @@ namespace AMDiS {
const char *file,
int line)
{
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+#if (DEBUG == 0)
+ if (outputMainRank && MPI::COMM_WORLD.Get_rank() != 0)
+ return;
+#endif
+#endif
+
static int old_line = -1;
if (!out)
@@ -229,6 +236,12 @@ namespace AMDiS {
void Msg::print_warn(const char *format, ...)
{
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+#if (DEBUG == 0)
+ if (outputMainRank && MPI::COMM_WORLD.Get_rank() != 0)
+ return;
+#endif
+#endif
va_list arg;
char buff[255];
diff --git a/AMDiS/src/Marker.h b/AMDiS/src/Marker.h
index e7908950..a6fae8e0 100644
--- a/AMDiS/src/Marker.h
+++ b/AMDiS/src/Marker.h
@@ -64,11 +64,9 @@ namespace AMDiS {
/// destructor
virtual ~Marker() {}
- /** \brief
- * Marks element with newMark. If \ref maximumMarking is set, the element
- * is marked only if the new mark is bigger than the old one. The return
- * value specifies whether the element has been marked, or not.
- */
+ /// Marks element with newMark. If \ref maximumMarking is set, the element
+ /// is marked only if the new mark is bigger than the old one. The return
+ /// value specifies whether the element has been marked, or not.
inline void setMark(Element *el, char newMark)
{
char oldMark = el->getMark();
@@ -120,7 +118,7 @@ namespace AMDiS {
return elMarkCoarsen;
}
- // Returns \ref name of the Marker
+ /// Returns \ref name of the Marker
inline std::string getName() const
{
return name;
@@ -133,10 +131,8 @@ namespace AMDiS {
/// Name of the scalar marker.
std::string name;
- /** \brief
- * Equal to -1 for scalar problems. Component number if marker is
- * part of a vector valued marker.
- */
+ /// Equal to -1 for scalar problems. Component number if marker is
+ /// part of a vector valued marker.
int row;
/// estimation sum
@@ -145,22 +141,16 @@ namespace AMDiS {
/// estmation maximum
double estMax;
- /** \brief
- * If true, elements are marked only if the new value is bigger than
- * the current marking.
- */
+ /// If true, elements are marked only if the new value is bigger than
+ /// the current marking.
bool maximumMarking;
- /** \brief
- * Lower limit for error estimation, from which an element is marked for
- * refinement
- */
+ /// Lower limit for error estimation, from which an element is marked for
+ /// refinement
double markRLimit;
- /** \brief
- * Upper limit for error estimation, from which an element is marked for
- * coarsening
- */
+ /// Upper limit for error estimation, from which an element is marked for
+ /// coarsening
double markCLimit;
/// power in estimator norm
diff --git a/AMDiS/src/parallel/MeshDistributor.cc b/AMDiS/src/parallel/MeshDistributor.cc
index 8ca851aa..20eb83d3 100644
--- a/AMDiS/src/parallel/MeshDistributor.cc
+++ b/AMDiS/src/parallel/MeshDistributor.cc
@@ -87,7 +87,8 @@ namespace AMDiS {
createBoundaryDofFlag(0),
boundaryDofInfo(1),
meshAdaptivity(true),
- hasPeriodicBoundary(false)
+ hasPeriodicBoundary(false),
+ printTimings(false)
{
FUNCNAME("MeshDistributor::ParalleDomainBase()");
@@ -99,7 +100,7 @@ namespace AMDiS {
Parameters::get(name + "->debug output dir", debugOutputDir);
Parameters::get(name + "->repartition ith change", repartitionIthChange);
Parameters::get(name + "->mesh adaptivity", meshAdaptivity);
-
+
string partStr = "parmetis";
Parameters::get(name + "->partitioner", partStr);
@@ -124,6 +125,8 @@ namespace AMDiS {
Parameters::get(name + "->box partitioning", tmp);
partitioner->setBoxPartitioning(static_cast<bool>(tmp));
+ Parameters::get(name + "->print timings", printTimings);
+
TEST_EXIT(partitioner)("Could not create partitioner \"%s\"!\n", partStr.c_str());
}
@@ -1584,6 +1587,10 @@ namespace AMDiS {
{
FUNCNAME("MeshDistributor::createBoundaryDofs()");
+ if (printTimings)
+ MPI::COMM_WORLD.Barrier();
+ double first = MPI::Wtime();
+
// === Create DOF communicator. ===
dofComm.init(0, levelData, feSpaces);
@@ -1596,52 +1603,57 @@ namespace AMDiS {
// === If requested, create more information on communication DOFs. ===
- if (!createBoundaryDofFlag.isSet(BOUNDARY_SUBOBJ_SORTED))
- return;
-
- int nLevels = levelData.getLevelNumber();
- boundaryDofInfo.resize(nLevels);
-
- for (unsigned int i = 0; i < feSpaces.size(); i++) {
- const FiniteElemSpace *feSpace = feSpaces[i];
+ if (createBoundaryDofFlag.isSet(BOUNDARY_SUBOBJ_SORTED)) {
- for (int level = 0; level < nLevels; level++) {
-
- // === Clear data. ===
- for (int geo = FACE; geo >= VERTEX; geo--)
- boundaryDofInfo[level][feSpace].geoDofs[static_cast<GeoIndex>(geo)].clear();
+ int nLevels = levelData.getLevelNumber();
+ boundaryDofInfo.resize(nLevels);
+
+ for (unsigned int i = 0; i < feSpaces.size(); i++) {
+ const FiniteElemSpace *feSpace = feSpaces[i];
- // === Create send DOFs. ===
- for (int geo = FACE; geo >= VERTEX; geo--) {
- for (InteriorBoundary::iterator it(intBoundary.getOwn(), level);
- !it.end(); ++it) {
- if (it->rankObj.subObj == geo) {
- DofContainer dofs;
- it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs);
-
- if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_SEND_DOFS))
- boundaryDofInfo[level][feSpace].
- geoDofs[static_cast<GeoIndex>(geo)].insert(dofs.begin(), dofs.end());
+ for (int level = 0; level < nLevels; level++) {
+
+ // === Clear data. ===
+ for (int geo = FACE; geo >= VERTEX; geo--)
+ boundaryDofInfo[level][feSpace].geoDofs[static_cast<GeoIndex>(geo)].clear();
+
+ // === Create send DOFs. ===
+ for (int geo = FACE; geo >= VERTEX; geo--) {
+ for (InteriorBoundary::iterator it(intBoundary.getOwn(), level);
+ !it.end(); ++it) {
+ if (it->rankObj.subObj == geo) {
+ DofContainer dofs;
+ it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs);
+
+ if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_SEND_DOFS))
+ boundaryDofInfo[level][feSpace].
+ geoDofs[static_cast<GeoIndex>(geo)].insert(dofs.begin(), dofs.end());
+ }
}
}
- }
-
- // === Create recv DOFs. ===
- for (int geo = FACE; geo >= VERTEX; geo--) {
- for (InteriorBoundary::iterator it(intBoundary.getOther(), level);
- !it.end(); ++it) {
- if (it->rankObj.subObj == geo) {
- DofContainer dofs;
- it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs);
-
- if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_RECV_DOFS))
- boundaryDofInfo[level][feSpace].
- geoDofs[static_cast<GeoIndex>(geo)].insert(dofs.begin(), dofs.end());
+
+ // === Create recv DOFs. ===
+ for (int geo = FACE; geo >= VERTEX; geo--) {
+ for (InteriorBoundary::iterator it(intBoundary.getOther(), level);
+ !it.end(); ++it) {
+ if (it->rankObj.subObj == geo) {
+ DofContainer dofs;
+ it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs);
+
+ if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_RECV_DOFS))
+ boundaryDofInfo[level][feSpace].
+ geoDofs[static_cast<GeoIndex>(geo)].insert(dofs.begin(), dofs.end());
+ }
}
}
}
}
}
+
+ if (printTimings) {
+ MPI::COMM_WORLD.Barrier();
+ MSG("MESHDIST 01 (createBoundaryDofs) needed %.5f seconds\n", MPI::Wtime() - first);
+ }
}
@@ -1679,22 +1691,8 @@ namespace AMDiS {
// === Update all registered DOF mapping objects. ===
-
- TEST_EXIT(dofMaps.size())("No DOF mapping defined!\n");
- for (int i = 0; i < static_cast<int>(dofMaps.size()); i++) {
- vector<const FiniteElemSpace*>& dofMapSpaces = dofMaps[i]->getFeSpaces();
-
- dofMaps[i]->clear();
-
- for (int j = 0; j < static_cast<int>(dofMapSpaces.size()); j++)
- updateLocalGlobalNumbering(*(dofMaps[i]), dofMapSpaces[j]);
-
- dofMaps[i]->update();
-#if (DEBUG != 0)
- // dofMaps[i]->printInfo();
-#endif
- }
+ updateParallelDofMappings();
// === Create periodic DOF maps, if there are periodic boundaries. ===
@@ -1769,6 +1767,37 @@ namespace AMDiS {
}
+ void MeshDistributor::updateParallelDofMappings()
+ {
+ FUNCNAME("MeshDistributor::updateParallelDofMappings()");
+
+ if (printTimings)
+ MPI::COMM_WORLD.Barrier();
+ double first = MPI::Wtime();
+
+ TEST_EXIT(dofMaps.size())("No DOF mapping defined!\n");
+
+ for (int i = 0; i < static_cast<int>(dofMaps.size()); i++) {
+ vector<const FiniteElemSpace*>& dofMapSpaces = dofMaps[i]->getFeSpaces();
+
+ dofMaps[i]->clear();
+
+ for (int j = 0; j < static_cast<int>(dofMapSpaces.size()); j++)
+ updateLocalGlobalNumbering(*(dofMaps[i]), dofMapSpaces[j]);
+
+ dofMaps[i]->update();
+#if (DEBUG != 0)
+ // dofMaps[i]->printInfo();
+#endif
+ }
+
+ if (printTimings) {
+ MPI::COMM_WORLD.Barrier();
+ MSG("MESHDIST 02 (update DOF mappings) needed %.5f seconds\n", MPI::Wtime() - first);
+ }
+ }
+
+
void MeshDistributor::updateLocalGlobalNumbering(ParallelDofMapping &dofMap,
const FiniteElemSpace *feSpace)
{
@@ -1776,6 +1805,7 @@ namespace AMDiS {
DofComm &dcom = dofMap.getDofComm();
+
// === Get all DOFs in ranks partition. ===
std::set<const DegreeOfFreedom*> rankDofSet;
@@ -1792,14 +1822,14 @@ namespace AMDiS {
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
nonRankDofs.insert(it.getDof());
-
- for (unsigned int i = 0; i < rankDofs.size(); i++)
- if (nonRankDofs.count(rankDofs[i]) == 0)
- dofMap[feSpace].insertRankDof(*(rankDofs[i]));
-
- for (DofContainerSet::iterator it = nonRankDofs.begin();
- it != nonRankDofs.end(); ++it)
- dofMap[feSpace].insertNonRankDof(**it);
+
+ for (DofContainer::iterator it = rankDofs.begin();
+ it != rankDofs.end(); ++it) {
+ if (nonRankDofs.count(*it))
+ dofMap[feSpace].insertNonRankDof(**it);
+ else
+ dofMap[feSpace].insertRankDof(**it);
+ }
}
diff --git a/AMDiS/src/parallel/MeshDistributor.h b/AMDiS/src/parallel/MeshDistributor.h
index e8a5a791..3022c26c 100644
--- a/AMDiS/src/parallel/MeshDistributor.h
+++ b/AMDiS/src/parallel/MeshDistributor.h
@@ -322,6 +322,9 @@ namespace AMDiS {
return levelData;
}
+ /// Updates all registered parallel DOF mappings, see \ref dofMaps.
+ void updateParallelDofMappings();
+
void updateLocalGlobalNumbering();
/// Updates the local and global DOF numbering after the mesh has been
@@ -583,6 +586,9 @@ namespace AMDiS {
/// solver objects and must be updated automatically after mesh change.
vector<ParallelDofMapping*> dofMaps;
+ /// If true, detailed timings for benchmarking are printed.
+ bool printTimings;
+
public:
/// The boundary DOFs are sorted by subobject entities, i.e., first all
/// face DOFs, edge DOFs and to the last vertex DOFs will be set to
diff --git a/AMDiS/src/parallel/ParallelDofMapping.h b/AMDiS/src/parallel/ParallelDofMapping.h
index 0bb64f7e..f2774941 100644
--- a/AMDiS/src/parallel/ParallelDofMapping.h
+++ b/AMDiS/src/parallel/ParallelDofMapping.h
@@ -161,7 +161,8 @@ namespace AMDiS {
/// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
/// the rank.
- void insertRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
+ inline void insertRankDof(DegreeOfFreedom dof0,
+ DegreeOfFreedom dof1 = -1)
{
FUNCNAME("ComponentDofMap::insertRankDof()");
@@ -176,7 +177,8 @@ namespace AMDiS {
/// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
/// but is owned by a different rank, thus it is part of an interior boundary.
- void insertNonRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
+ inline void insertNonRankDof(DegreeOfFreedom dof0,
+ DegreeOfFreedom dof1 = -1)
{
FUNCNAME("ComponentDofMap::insertNonRankDof()");
@@ -281,7 +283,7 @@ namespace AMDiS {
DofMap dofMap;
/// Set of all DOFs that are in mapping but are not owned by the rank.
- boost::container::flat_set<DegreeOfFreedom> nonRankDofs;
+ boost::container::flat_set<DegreeOfFreedom> nonRankDofs;
/// If true, a global index mapping will be computed for all DOFs.
bool globalMapping;
--
GitLab