From 8c7d38605dfd4f22befc9e8bd9795003674e8f6d Mon Sep 17 00:00:00 2001
From: Thomas Witkowski <thomas.witkowski@gmx.de>
Date: Tue, 21 Dec 2010 20:14:10 +0000
Subject: [PATCH] Fixed several very small problems in parallel code.
---
AMDiS/CMakeLists.txt | 1 +
AMDiS/bin/Makefile.am | 1 +
AMDiS/bin/Makefile.in | 11 +++
AMDiS/libtool | 6 +-
AMDiS/src/Element.h | 4 +
AMDiS/src/Global.h | 1 +
AMDiS/src/Mesh.cc | 56 +++++++------
AMDiS/src/Mesh.h | 24 +++---
AMDiS/src/parallel/ElementObjectData.h | 6 +-
AMDiS/src/parallel/MeshDistributor.cc | 5 +-
AMDiS/src/parallel/ParMetisPartitioner.cc | 82 ++++++++-----------
AMDiS/src/parallel/ParMetisPartitioner.h | 2 +-
AMDiS/src/parallel/ParallelProblemStatBase.cc | 82 +++++++++++++++++++
AMDiS/src/parallel/ParallelProblemStatBase.h | 19 ++---
AMDiS/src/parallel/PetscSolver.cc | 26 ++----
15 files changed, 204 insertions(+), 122 deletions(-)
create mode 100644 AMDiS/src/parallel/ParallelProblemStatBase.cc
diff --git a/AMDiS/CMakeLists.txt b/AMDiS/CMakeLists.txt
index 4a012217..d42aa6df 100644
--- a/AMDiS/CMakeLists.txt
+++ b/AMDiS/CMakeLists.txt
@@ -181,6 +181,7 @@ if(ENABLE_PARALLEL_DOMAIN)
${SOURCE_DIR}/parallel/ParallelDebug.cc
${SOURCE_DIR}/parallel/MpiHelper.cc
${SOURCE_DIR}/parallel/ElementObjectData.cc
+ ${SOURCE_DIR}/parallel/ParallelProblemStatBase.cc
${SOURCE_DIR}/parallel/PetscSolver.cc)
SET(COMPILEFLAGS "${COMPILEFLAGS} -DHAVE_PARALLEL_DOMAIN_AMDIS=1")
endif(ENABLE_PARALLEL_DOMAIN)
diff --git a/AMDiS/bin/Makefile.am b/AMDiS/bin/Makefile.am
index 8dfc05fa..e9f356fd 100644
--- a/AMDiS/bin/Makefile.am
+++ b/AMDiS/bin/Makefile.am
@@ -21,6 +21,7 @@ if USE_PARALLEL_DOMAIN_AMDIS
$(SOURCE_DIR)/parallel/MeshManipulation.h $(SOURCE_DIR)/parallel/MeshManipulation.cc \
$(SOURCE_DIR)/parallel/ParallelDebug.h $(SOURCE_DIR)/parallel/ParallelDebug.cc \
$(SOURCE_DIR)/parallel/ParallelProblemStatBase.h \
+ $(SOURCE_DIR)/parallel/ParallelProblemStatBase.cc \
$(SOURCE_DIR)/parallel/PetscSolver.h $(SOURCE_DIR)/parallel/PetscSolver.cc \
$(SOURCE_DIR)/parallel/MpiHelper.h $(SOURCE_DIR)/parallel/MpiHelper.cc \
$(SOURCE_DIR)/parallel/ElementObjectData.h $(SOURCE_DIR)/parallel/ElementObjectData.cc
diff --git a/AMDiS/bin/Makefile.in b/AMDiS/bin/Makefile.in
index 1cb3e3d5..9aa0041a 100644
--- a/AMDiS/bin/Makefile.in
+++ b/AMDiS/bin/Makefile.in
@@ -43,6 +43,7 @@ host_triplet = @host@
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/MeshManipulation.h $(SOURCE_DIR)/parallel/MeshManipulation.cc \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/ParallelDebug.h $(SOURCE_DIR)/parallel/ParallelDebug.cc \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/ParallelProblemStatBase.h \
+@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/ParallelProblemStatBase.cc \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/PetscSolver.h $(SOURCE_DIR)/parallel/PetscSolver.cc \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/MpiHelper.h $(SOURCE_DIR)/parallel/MpiHelper.cc \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/ElementObjectData.h $(SOURCE_DIR)/parallel/ElementObjectData.cc
@@ -89,6 +90,7 @@ am__libamdis_la_SOURCES_DIST = $(SOURCE_DIR)/parallel/StdMpi.h \
$(SOURCE_DIR)/parallel/ParallelDebug.h \
$(SOURCE_DIR)/parallel/ParallelDebug.cc \
$(SOURCE_DIR)/parallel/ParallelProblemStatBase.h \
+ $(SOURCE_DIR)/parallel/ParallelProblemStatBase.cc \
$(SOURCE_DIR)/parallel/PetscSolver.h \
$(SOURCE_DIR)/parallel/PetscSolver.cc \
$(SOURCE_DIR)/parallel/MpiHelper.h \
@@ -252,6 +254,7 @@ am__libamdis_la_SOURCES_DIST = $(SOURCE_DIR)/parallel/StdMpi.h \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-MeshDistributor.lo \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-MeshManipulation.lo \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-ParallelDebug.lo \
+@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-ParallelProblemStatBase.lo \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-PetscSolver.lo \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-MpiHelper.lo \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-ElementObjectData.lo
@@ -803,6 +806,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-OperatorTerm.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParMetisPartitioner.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParallelDebug.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParallelProblemStatBase.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Parameters.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Parametric.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-PeriodicBC.Plo@am__quote@
@@ -916,6 +920,13 @@ libamdis_la-ParallelDebug.lo: $(SOURCE_DIR)/parallel/ParallelDebug.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -c -o libamdis_la-ParallelDebug.lo `test -f '$(SOURCE_DIR)/parallel/ParallelDebug.cc' || echo '$(srcdir)/'`$(SOURCE_DIR)/parallel/ParallelDebug.cc
+libamdis_la-ParallelProblemStatBase.lo: $(SOURCE_DIR)/parallel/ParallelProblemStatBase.cc
+@am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-ParallelProblemStatBase.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-ParallelProblemStatBase.Tpo" -c -o libamdis_la-ParallelProblemStatBase.lo `test -f '$(SOURCE_DIR)/parallel/ParallelProblemStatBase.cc' || echo '$(srcdir)/'`$(SOURCE_DIR)/parallel/ParallelProblemStatBase.cc; \
+@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libamdis_la-ParallelProblemStatBase.Tpo" "$(DEPDIR)/libamdis_la-ParallelProblemStatBase.Plo"; else rm -f "$(DEPDIR)/libamdis_la-ParallelProblemStatBase.Tpo"; exit 1; fi
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$(SOURCE_DIR)/parallel/ParallelProblemStatBase.cc' object='libamdis_la-ParallelProblemStatBase.lo' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -c -o libamdis_la-ParallelProblemStatBase.lo `test -f '$(SOURCE_DIR)/parallel/ParallelProblemStatBase.cc' || echo '$(srcdir)/'`$(SOURCE_DIR)/parallel/ParallelProblemStatBase.cc
+
libamdis_la-PetscSolver.lo: $(SOURCE_DIR)/parallel/PetscSolver.cc
@am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-PetscSolver.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-PetscSolver.Tpo" -c -o libamdis_la-PetscSolver.lo `test -f '$(SOURCE_DIR)/parallel/PetscSolver.cc' || echo '$(srcdir)/'`$(SOURCE_DIR)/parallel/PetscSolver.cc; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libamdis_la-PetscSolver.Tpo" "$(DEPDIR)/libamdis_la-PetscSolver.Plo"; else rm -f "$(DEPDIR)/libamdis_la-PetscSolver.Tpo"; exit 1; fi
diff --git a/AMDiS/libtool b/AMDiS/libtool
index 95d2fe89..1a37a1c4 100755
--- a/AMDiS/libtool
+++ b/AMDiS/libtool
@@ -44,7 +44,7 @@ available_tags=" CXX F77"
# ### BEGIN LIBTOOL CONFIG
-# Libtool was configured on host deimos101:
+# Libtool was configured on host deimos102:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
@@ -6760,7 +6760,7 @@ build_old_libs=`case $build_libtool_libs in yes) $echo no;; *) $echo yes;; esac`
# End:
# ### BEGIN LIBTOOL TAG CONFIG: CXX
-# Libtool was configured on host deimos101:
+# Libtool was configured on host deimos102:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
@@ -7065,7 +7065,7 @@ include_expsyms=""
# ### BEGIN LIBTOOL TAG CONFIG: F77
-# Libtool was configured on host deimos101:
+# Libtool was configured on host deimos102:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
diff --git a/AMDiS/src/Element.h b/AMDiS/src/Element.h
index 78f78d32..b2faa2be 100644
--- a/AMDiS/src/Element.h
+++ b/AMDiS/src/Element.h
@@ -215,6 +215,10 @@ namespace AMDiS {
///
virtual DofFace getFace(int localFaceIndex) const = 0;
+ /// Returns either vertex, edge or face of the element.
+ template<typename T>
+ T getObject(int index);
+
/// Returns the number of parts of type i in this element
virtual int getGeo(GeoIndex i) const = 0;
diff --git a/AMDiS/src/Global.h b/AMDiS/src/Global.h
index a1de6272..8f340ead 100644
--- a/AMDiS/src/Global.h
+++ b/AMDiS/src/Global.h
@@ -86,6 +86,7 @@ namespace AMDiS {
/// Defines a tzpe for global face identiication via its DOFs.
typedef boost::tuple<DegreeOfFreedom, DegreeOfFreedom, DegreeOfFreedom> DofFace;
+
/// returns the GeoIndex of d for dimension dim.
#define INDEX_OF_DIM(d, dim) (static_cast<GeoIndex>((d == dim) ? CENTER : d + 1))
diff --git a/AMDiS/src/Mesh.cc b/AMDiS/src/Mesh.cc
index f0f4b1d5..9d08399e 100644
--- a/AMDiS/src/Mesh.cc
+++ b/AMDiS/src/Mesh.cc
@@ -654,7 +654,7 @@ namespace AMDiS {
{
FUNCNAME("Mesh::createNewElInfo()");
- switch(dim) {
+ switch (dim) {
case 1:
return new ElInfo1d(this);
break;
@@ -1176,24 +1176,13 @@ namespace AMDiS {
GET_PARAMETER(0, name + "->preserve coarse dofs", "%d", &preserveCoarseDOFs);
if (macroFilename.length()) {
+ // In parallel computations, check if a finer macro mesh is required.
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
- if (checkParallelMacroFile(macroFilename, periodicFilename, check)) {
- std::string newPeriodicFilename = "";
- if (periodicFilename != "")
- newPeriodicFilename = periodicFilename + ".tmp";
-
- macroFileInfo =
- MacroReader::readMacro(macroFilename + ".tmp", this,
- newPeriodicFilename, check);
- } else {
- macroFileInfo =
- MacroReader::readMacro(macroFilename, this, periodicFilename, check);
- }
-#else
- // In sequentiel computations just read the macro file to the mesh.
+ checkParallelMacroFile(macroFilename, periodicFilename, check);
+#endif
+
macroFileInfo =
MacroReader::readMacro(macroFilename, this, periodicFilename, check);
-#endif
if (!valueFilename.length())
clearMacroFileInfo();
@@ -1204,8 +1193,8 @@ namespace AMDiS {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
- bool Mesh::checkParallelMacroFile(std::string macroFilename,
- std::string periodicFilename,
+ void Mesh::checkParallelMacroFile(std::string ¯oFilename,
+ std::string &periodicFilename,
int check)
{
FUNCNAME("Mesh::checkParallelMacroFile()");
@@ -1293,7 +1282,7 @@ namespace AMDiS {
// === If we do not need to refine the mesh, return back. ===
if (nrRefine == 0)
- return false;
+ return;
// === If macro weights are explicitly given, we cannot change the mesh. ===
@@ -1305,6 +1294,24 @@ namespace AMDiS {
}
+
+
+ // === Create unique file names. ===
+
+ int filenameRandomNumber = 0;
+ if (MPI::COMM_WORLD.Get_rank() == 0) {
+ srand(time(NULL));
+ filenameRandomNumber = rand() % 1000000;
+ }
+ MPI::COMM_WORLD.Bcast(&filenameRandomNumber, 1, MPI_INT, 0);
+
+ std::stringstream newMacroFilename;
+ newMacroFilename << macroFilename << "." << filenameRandomNumber << ".tmp";
+
+ std::stringstream newPeriodicFilename;
+ newPeriodicFilename << periodicFilename << "." << filenameRandomNumber << ".tmp";
+
+
// === Rank 0 creates a new mesh file. ===
if (MPI::COMM_WORLD.Get_rank() == 0) {
@@ -1322,10 +1329,10 @@ namespace AMDiS {
FiniteElemSpace::provideFeSpace(localAdmin, basFcts, &testMesh, "tmp");
DataCollector dc(feSpace);
- MacroWriter::writeMacro(&dc, (macroFilename + ".tmp").c_str());
+ MacroWriter::writeMacro(&dc, newMacroFilename.str().c_str());
if (periodicFilename != "")
- MacroWriter::writePeriodicFile(&dc, (periodicFilename + ".tmp").c_str());
+ MacroWriter::writePeriodicFile(&dc, newPeriodicFilename.str().c_str());
}
@@ -1353,10 +1360,11 @@ namespace AMDiS {
// === Print a note to the screen that another mesh file will be used. ===
MSG("The macro mesh file \"%s\" was refined %d times and stored to file \"%s\".\n",
- macroFilename.c_str(), nrRefine, (macroFilename + ".tmp").c_str());
-
+ macroFilename.c_str(), nrRefine, newMacroFilename.str().c_str());
- return true;
+ macroFilename = newMacroFilename.str();
+ if (periodicFilename != "")
+ periodicFilename = newPeriodicFilename.str();
}
#endif
diff --git a/AMDiS/src/Mesh.h b/AMDiS/src/Mesh.h
index 49fe7d40..93c9b8e7 100644
--- a/AMDiS/src/Mesh.h
+++ b/AMDiS/src/Mesh.h
@@ -688,21 +688,19 @@ namespace AMDiS {
* If this is not the case, that macro mesh is globally refined in an
* apropriate way and is written to a new macro file.
*
- * The function returns true, if a new macro and parallel file were created for
- * the current parallel usage. In this case, a new macro file with the same name
- * plus ".tmp", and if required, a new periodic file with the same name plus
- * ".tmp" are created.
+ * The function overwrittes the macro and periodic filenames, if a new macro
+ * fule was created for the current parallel usage.
*
- * \param[in] macroFilename Name of the macro mesh file.
- * \param[in] periodicFilename If periodic boundaries are used, name of the
- * periodicity file. Otherwise, the string must
- * be empty.
- * \param[in] check If the mesh should be checked to be a correct
- * AMDiS macro mesh, the value must be 1 and 0
- * otherwise.
+ * \param[in/out] macroFilename Name of the macro mesh file.
+ * \param[in/out] periodicFilename If periodic boundaries are used, name of the
+ * periodicity file. Otherwise, the string must
+ * be empty.
+ * \param[in] check If the mesh should be checked to be a correct
+ * AMDiS macro mesh, the value must be 1 and 0
+ * otherwise.
*/
- bool checkParallelMacroFile(std::string macroFilename,
- std::string periodicFilename,
+ void checkParallelMacroFile(std::string ¯oFilename,
+ std::string &periodicFilename,
int check);
#endif
diff --git a/AMDiS/src/parallel/ElementObjectData.h b/AMDiS/src/parallel/ElementObjectData.h
index 883c9a45..6ba200f8 100644
--- a/AMDiS/src/parallel/ElementObjectData.h
+++ b/AMDiS/src/parallel/ElementObjectData.h
@@ -90,7 +90,7 @@ namespace AMDiS {
void addVertex(Element *el, int ith, BoundaryType bound = INTERIOR)
{
- DegreeOfFreedom vertex = el->getDof(ith, 0);
+ DegreeOfFreedom vertex = el->getObject<DegreeOfFreedom>(ith);
int elIndex = el->getIndex();
ElementObjectData elObj(elIndex, ith, bound);
@@ -101,7 +101,7 @@ namespace AMDiS {
void addEdge(Element *el, int ith, BoundaryType bound = INTERIOR)
{
- DofEdge edge = el->getEdge(ith);
+ DofEdge edge = el->getObject<DofEdge>(ith);
int elIndex = el->getIndex();
ElementObjectData elObj(elIndex, ith, bound);
@@ -112,7 +112,7 @@ namespace AMDiS {
void addFace(Element *el, int ith, BoundaryType bound = INTERIOR)
{
- DofFace face = el->getFace(ith);
+ DofFace face = el->getObject<DofFace>(ith);
int elIndex = el->getIndex();
ElementObjectData elObj(elIndex, ith, bound);
diff --git a/AMDiS/src/parallel/MeshDistributor.cc b/AMDiS/src/parallel/MeshDistributor.cc
index a169e2b6..4d546bc9 100644
--- a/AMDiS/src/parallel/MeshDistributor.cc
+++ b/AMDiS/src/parallel/MeshDistributor.cc
@@ -146,7 +146,7 @@ namespace AMDiS {
// and now partition the mesh
partitioner->fillCoarsePartitionVec(&oldPartitionVec);
- partitioner->partition(&elemWeights, INITIAL);
+ partitioner->partition(elemWeights, INITIAL);
partitioner->fillCoarsePartitionVec(&partitionVec);
@@ -1029,10 +1029,9 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo);
}
-
partitioner->useLocalGlobalDofMap(&mapLocalGlobalDofs);
oldPartitionVec = partitionVec;
- partitioner->partition(&elemWeights, ADAPTIVE_REPART, 1000.0);
+ partitioner->partition(elemWeights, ADAPTIVE_REPART, 1000.0);
// === Create map that maps macro element indices to pointers to the ===
diff --git a/AMDiS/src/parallel/ParMetisPartitioner.cc b/AMDiS/src/parallel/ParMetisPartitioner.cc
index c2f61e56..abd56da6 100644
--- a/AMDiS/src/parallel/ParMetisPartitioner.cc
+++ b/AMDiS/src/parallel/ParMetisPartitioner.cc
@@ -207,7 +207,7 @@ namespace AMDiS {
}
- void ParMetisPartitioner::partition(std::map<int, double> *elemWeights,
+ void ParMetisPartitioner::partition(std::map<int, double> &elemWeights,
PartitionMode mode,
float itr)
{
@@ -226,10 +226,10 @@ namespace AMDiS {
int nElements = parMetisMesh->getNumElements();
// === create weight array ===
- int *wgts = elemWeights ? new int[nElements] : NULL;
- float *floatWgts = elemWeights ? new float[nElements] : NULL;
+ std::vector<int> wgts(nElements);
+ std::vector<float> floatWgts(nElements);
+ unsigned int floatWgtsPos = 0;
float maxWgt = 0.0;
- float *ptr_floatWgts = floatWgts;
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, 0, Mesh::CALL_EL_LEVEL);
@@ -238,16 +238,18 @@ namespace AMDiS {
if (elementInRank[index]) {
// get weight
- float wgt = static_cast<float>((*elemWeights)[index]);
+ float wgt = static_cast<float>(elemWeights[index]);
maxWgt = max(wgt, maxWgt);
// write float weight
- *ptr_floatWgts = wgt;
- ptr_floatWgts++;
+ TEST_EXIT_DBG(floatWgtsPos < floatWgts.size())("Should not happen!\n");
+ floatWgts[floatWgtsPos++] = wgt;
}
elInfo = stack.traverseNext(elInfo);
}
+ TEST_EXIT_DBG(floatWgtsPos == floatWgts.size())("Should not happen!\n");
+
float tmp;
mpiComm->Allreduce(&maxWgt, &tmp, 1, MPI_FLOAT, MPI_MAX);
maxWgt = tmp;
@@ -256,27 +258,25 @@ namespace AMDiS {
ParMetisGraph parMetisGraph(parMetisMesh, mpiComm);
// === partitioning of dual graph ===
- int wgtflag = elemWeights ? 2 : 0; // weights at vertices only!
+ int wgtflag = 2; // weights at vertices only!
int numflag = 0; // c numbering style!
- int ncon = elemWeights ? 1 : 0; // one weight at each vertex!
+ int ncon = 1; // one weight at each vertex!
int nparts = mpiSize; // number of partitions
- float *tpwgts = elemWeights ? new float[mpiSize] : NULL;
+ std::vector<float> tpwgts(mpiSize);
float ubvec = 1.05;
- int options[3] = {0, 0, 0}; // default options
+ int options[4] = {0, 0, 15, 1}; // default options
int edgecut = -1;
- int *part = new int[nElements];
-
- if (elemWeights) {
- // set tpwgts
- for (int i = 0; i < mpiSize; i++)
- tpwgts[i] = 1.0 / nparts;
+ std::vector<int> part(nElements);
- float scale = 10000.0 / maxWgt;
- // scale wgts
- for (int i = 0; i < nElements; i++)
- wgts[i] = static_cast<int>(floatWgts[i] * scale);
- }
+ // set tpwgts
+ for (int i = 0; i < mpiSize; i++)
+ tpwgts[i] = 1.0 / nparts;
+
+ float scale = 10000.0 / maxWgt;
+ // scale wgts
+ for (int i = 0; i < nElements; i++)
+ wgts[i] = static_cast<int>(floatWgts[i] * scale);
MPI_Comm tmpComm = MPI_Comm(*mpiComm);
@@ -285,60 +285,59 @@ namespace AMDiS {
ParMETIS_V3_PartKway(parMetisMesh->getElementDist(),
parMetisGraph.getXAdj(),
parMetisGraph.getAdjncy(),
- wgts,
+ &(wgts[0]),
NULL,
&wgtflag,
&numflag,
&ncon,
&nparts,
- tpwgts,
+ &(tpwgts[0]),
&ubvec,
options,
&edgecut,
- part,
+ &(part[0]),
&tmpComm);
break;
case ADAPTIVE_REPART:
{
- int *vsize = new int[nElements];
+ std::vector<int> vsize(nElements);
for (int i = 0; i < nElements; i++)
- vsize[i] = 1;
+ vsize[i] = static_cast<int>(floatWgts[i]);
+
ParMETIS_V3_AdaptiveRepart(parMetisMesh->getElementDist(),
parMetisGraph.getXAdj(),
parMetisGraph.getAdjncy(),
- wgts,
+ &(wgts[0]),
NULL,
- vsize,
+ &(vsize[0]),
&wgtflag,
&numflag,
&ncon,
&nparts,
- tpwgts,
+ &(tpwgts[0]),
&ubvec,
&itr,
options,
&edgecut,
- part,
+ &(part[0]),
&tmpComm);
-
- delete [] vsize;
}
break;
case REFINE_PART:
ParMETIS_V3_RefineKway(parMetisMesh->getElementDist(),
parMetisGraph.getXAdj(),
parMetisGraph.getAdjncy(),
- wgts,
+ &(wgts[0]),
NULL,
&wgtflag,
&numflag,
&ncon,
&nparts,
- tpwgts,
+ &(tpwgts[0]),
&ubvec,
options,
&edgecut,
- part,
+ &(part[0]),
&tmpComm);
break;
@@ -347,16 +346,7 @@ namespace AMDiS {
}
// === distribute new partition data ===
- distributePartitioning(part);
-
- if (floatWgts)
- delete [] floatWgts;
- if (wgts)
- delete [] wgts;
- if (tpwgts)
- delete [] tpwgts;
-
- delete [] part;
+ distributePartitioning(&(part[0]));
}
diff --git a/AMDiS/src/parallel/ParMetisPartitioner.h b/AMDiS/src/parallel/ParMetisPartitioner.h
index 90317aa1..38d5e7b1 100644
--- a/AMDiS/src/parallel/ParMetisPartitioner.h
+++ b/AMDiS/src/parallel/ParMetisPartitioner.h
@@ -177,7 +177,7 @@ namespace AMDiS {
mapLocalGlobal(NULL)
{}
- void partition(std::map<int, double> *elemWeights,
+ void partition(std::map<int, double> &elemWeights,
PartitionMode mode = INITIAL,
float itr = 1000000.0);
diff --git a/AMDiS/src/parallel/ParallelProblemStatBase.cc b/AMDiS/src/parallel/ParallelProblemStatBase.cc
new file mode 100644
index 00000000..015d1727
--- /dev/null
+++ b/AMDiS/src/parallel/ParallelProblemStatBase.cc
@@ -0,0 +1,82 @@
+//
+// Software License for AMDiS
+//
+// Copyright (c) 2010 Dresden University of Technology
+// All rights reserved.
+// Authors: Simon Vey, Thomas Witkowski et al.
+//
+// This file is part of AMDiS
+//
+// See also license.opensource.txt in the distribution.
+
+
+#include "parallel/ParallelProblemStatBase.h"
+#include "parallel/MeshDistributor.h"
+#include "parallel/MpiHelper.h"
+
+namespace AMDiS {
+
+ void ParallelProblemStatBase::buildAfterCoarsen(AdaptInfo *adaptInfo, Flag flag,
+ bool assembleMatrix,
+ bool assembleVector)
+ {
+ FUNCNAME("ParallelProblemStatBase::buildAfterCoarsen()");
+
+ meshDistributor->checkMeshChange();
+ ProblemVec::buildAfterCoarsen(adaptInfo, flag, assembleMatrix, assembleVector);
+
+ double vm, rss;
+ processMemUsage(vm, rss);
+
+ MSG("My memory usage is VM = %f RSS = %f\n", vm, rss);
+
+ mpi::globalAdd(vm);
+ mpi::globalAdd(rss);
+
+ MSG("Overall memory usage is VM = %f RSS = %f\n", vm, rss);
+ }
+
+
+ void ParallelProblemStatBase::addToMeshDistributor(MeshDistributor& m)
+ {
+ meshDistributor = &m;
+ m.addProblemStat(this);
+ }
+
+
+ void ParallelProblemStatBase::processMemUsage(double& vm_usage,
+ double& resident_set)
+ {
+ using std::ios_base;
+ using std::ifstream;
+ using std::string;
+
+ vm_usage = 0.0;
+ resident_set = 0.0;
+
+ // 'file' stat seems to give the most reliable results
+ ifstream stat_stream("/proc/self/stat",ios_base::in);
+
+ // dummy vars for leading entries in stat that we don't care about
+ string pid, comm, state, ppid, pgrp, session, tty_nr;
+ string tpgid, flags, minflt, cminflt, majflt, cmajflt;
+ string utime, stime, cutime, cstime, priority, nice;
+ string O, itrealvalue, starttime;
+
+ // the two fields we want
+ unsigned long vsize;
+ long rss;
+
+ stat_stream >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr
+ >> tpgid >> flags >> minflt >> cminflt >> majflt >> cmajflt
+ >> utime >> stime >> cutime >> cstime >> priority >> nice
+ >> O >> itrealvalue >> starttime >> vsize >> rss;
+
+ // in case x86-64 is configured to use 2MB pages
+ long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024;
+ vm_usage = vsize / 1024.0;
+ resident_set = rss * page_size_kb;
+ }
+
+
+}
diff --git a/AMDiS/src/parallel/ParallelProblemStatBase.h b/AMDiS/src/parallel/ParallelProblemStatBase.h
index cd201e6b..aaac9bb2 100644
--- a/AMDiS/src/parallel/ParallelProblemStatBase.h
+++ b/AMDiS/src/parallel/ParallelProblemStatBase.h
@@ -23,6 +23,7 @@
#ifndef AMDIS_PARALLEL_PROBLEM_STAT_BASE_H
#define AMDIS_PARALLEL_PROBLEM_STAT_BASE_H
+#include "parallel/MeshDistributor.h"
#include "ProblemVec.h"
namespace AMDiS {
@@ -39,18 +40,12 @@ namespace AMDiS {
void buildAfterCoarsen(AdaptInfo *adaptInfo, Flag flag,
bool assembleMatrix,
- bool assembleVector)
- {
- meshDistributor->checkMeshChange();
- ProblemVec::buildAfterCoarsen(adaptInfo, flag, assembleMatrix, assembleVector);
- }
-
-
- void addToMeshDistributor(MeshDistributor& m)
- {
- meshDistributor = &m;
- m.addProblemStat(this);
- }
+ bool assembleVector);
+
+ void addToMeshDistributor(MeshDistributor& m);
+
+ protected:
+ void processMemUsage(double& vm_usage, double& resident_set);
protected:
MeshDistributor *meshDistributor;
diff --git a/AMDiS/src/parallel/PetscSolver.cc b/AMDiS/src/parallel/PetscSolver.cc
index b72fc74b..e8e25b1e 100644
--- a/AMDiS/src/parallel/PetscSolver.cc
+++ b/AMDiS/src/parallel/PetscSolver.cc
@@ -36,22 +36,13 @@ namespace AMDiS {
TEST_EXIT(meshDistributor)("Should not happen!\n");
-#ifdef _OPENMP
- double wtime = omp_get_wtime();
-#endif
- clock_t first = clock();
+ double wtime = MPI::Wtime();
fillPetscMatrix(systemMatrix, rhs);
solvePetscMatrix(*solution, adaptInfo);
-#ifdef _OPENMP
- INFO(info, 8)("solution of discrete system needed %.5f seconds system time / %.5f seconds wallclock time\n",
- TIME_USED(first, clock()),
- omp_get_wtime() - wtime);
-#else
- INFO(info, 8)("solution of discrete system needed %.5f seconds\n",
- TIME_USED(first, clock()));
-#endif
+ INFO(info, 8)("solution of discrete system needed %.5f seconds\n",
+ MPI::Wtime() - wtime);
}
@@ -393,7 +384,7 @@ namespace AMDiS {
{
FUNCNAME("PetscSolver::fillPetscMatrix()");
- clock_t first = clock();
+ double wtime = MPI::Wtime();
int nRankRows = meshDistributor->getNumberRankDofs() * nComponents;
int nOverallRows = meshDistributor->getNumberOverallDofs() * nComponents;
@@ -434,7 +425,7 @@ namespace AMDiS {
0, d_nnz, 0, o_nnz, &petscMatrix);
#if (DEBUG != 0)
- INFO(info, 8)("Fill petsc matrix 1 needed %.5f seconds\n", TIME_USED(first, clock()));
+ INFO(info, 8)("Fill petsc matrix 1 needed %.5f seconds\n", MPI::Wtime() - wtime);
#endif
#if (DEBUG != 0)
@@ -455,14 +446,15 @@ namespace AMDiS {
setDofMatrix((*mat)[i][j], nComponents, i, j);
#if (DEBUG != 0)
- INFO(info, 8)("Fill petsc matrix 2 needed %.5f seconds\n", TIME_USED(first, clock()));
+ INFO(info, 8)("Fill petsc matrix 2 needed %.5f seconds\n", MPI::Wtime() - wtime);
#endif
MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(petscMatrix, MAT_FINAL_ASSEMBLY);
#if (DEBUG != 0)
- INFO(info, 8)("Fill petsc matrix 3 needed %.5f seconds\n", TIME_USED(first, clock()));
+ INFO(info, 8)("Fill petsc matrix 3 needed %.5f seconds\n",
+ TIME_USED(MPI::Wtime(), wtime));
#endif
// === Transfer values from DOF vector to the PETSc vector. ===
@@ -473,7 +465,7 @@ namespace AMDiS {
VecAssemblyBegin(petscRhsVec);
VecAssemblyEnd(petscRhsVec);
- INFO(info, 8)("Fill petsc matrix needed %.5f seconds\n", TIME_USED(first, clock()));
+ INFO(info, 8)("Fill petsc matrix needed %.5f seconds\n", MPI::Wtime() - wtime);
}
--
GitLab