From 30c3b864ad71fdf2287e6fedf2f31a4d6b6a0450 Mon Sep 17 00:00:00 2001
From: Thomas Witkowski <thomas.witkowski@gmx.de>
Date: Tue, 30 Nov 2010 09:07:50 +0000
Subject: [PATCH] Some code refactoring and fixed some small problems in mesh
repartitioning.
---
AMDiS/libtool | 22 +-
AMDiS/src/AMDiS.h | 1 +
AMDiS/src/DOFAdmin.cc | 28 +-
AMDiS/src/DOFAdmin.h | 2 +-
AMDiS/src/DOFContainer.h | 9 +-
AMDiS/src/Element.cc | 15 +-
AMDiS/src/Element.h | 5 +-
AMDiS/src/Mesh.cc | 13 +-
AMDiS/src/Mesh.h | 6 -
AMDiS/src/parallel/MeshDistributor.cc | 394 +++----------------------
AMDiS/src/parallel/MeshDistributor.h | 46 +--
AMDiS/src/parallel/MeshManipulation.cc | 20 +-
AMDiS/src/parallel/ParallelDebug.cc | 2 +-
13 files changed, 103 insertions(+), 460 deletions(-)
diff --git a/AMDiS/libtool b/AMDiS/libtool
index 730f91e3..5085abd4 100755
--- a/AMDiS/libtool
+++ b/AMDiS/libtool
@@ -85,7 +85,7 @@ NM="/usr/bin/nm -B"
LN_S="ln -s"
# What is the maximum length of a command?
-max_cmd_len=1572864
+max_cmd_len=98304
# Object file suffix (normally "o").
objext=o
@@ -128,7 +128,7 @@ old_postinstall_cmds="chmod 644 \$oldlib~\$RANLIB \$oldlib"
old_postuninstall_cmds=""
# A C compiler.
-LTCC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
+LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
@@ -233,10 +233,10 @@ finish_eval=""
hardcode_into_libs=yes
# Compile-time system search path for libraries.
-sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64 /lib64 /usr/x86_64-suse-linux/lib"
+sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64 /lib64 /fastfs/wir/local/lib /usr/x86_64-suse-linux/lib"
# Run-time system search path for libraries.
-sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/lib64/Xaw3d /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/lib/Xaw3d /usr/x86_64-suse-linux/lib /usr/local/lib /opt/kde3/lib /lib64 /lib /usr/lib64 /usr/lib /usr/local/lib64 /opt/kde3/lib64 /usr/lib64/graphviz /usr/lib64/graphviz/sharp /usr/lib64/graphviz/java /usr/lib64/graphviz/perl /usr/lib64/graphviz/php /usr/lib64/graphviz/ocaml /usr/lib64/graphviz/python /usr/lib64/graphviz/lua /usr/lib64/graphviz/tcl /usr/lib64/graphviz/guile /usr/lib64/graphviz/ruby /usr/lib64/octave-3.2.4 "
+sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
# Whether dlopen is supported.
dlopen_support=unknown
@@ -259,7 +259,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
-CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
+CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# Is the compiler the GNU compiler?
with_gcc=yes
@@ -8914,7 +8914,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
-CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicxx"
+CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicxx"
# Is the compiler the GNU compiler?
with_gcc=yes
@@ -9039,17 +9039,17 @@ file_list_spec=""
hardcode_action=immediate
# The directories searched by this compiler when creating a shared library.
-compiler_lib_search_dirs="/usr/lib64/mpi/gcc/openmpi/lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
+compiler_lib_search_dirs="/usr/lib64 /licsoft/libraries/openmpi/1.2.6/64bit/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /fastfs/wir/local/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# Dependencies to place before and after the objects being linked to
# create a shared library.
-predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.5/crtbeginS.o"
-postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crtn.o"
+predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtbeginS.o"
+postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crtn.o"
predeps=""
-postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
+postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking
# a shared library.
-compiler_lib_search_path="-L/usr/lib64/mpi/gcc/openmpi/lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
+compiler_lib_search_path="-L/usr/lib64 -L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/fastfs/wir/local/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# ### END LIBTOOL TAG CONFIG: CXX
diff --git a/AMDiS/src/AMDiS.h b/AMDiS/src/AMDiS.h
index 31531d62..db24bd61 100644
--- a/AMDiS/src/AMDiS.h
+++ b/AMDiS/src/AMDiS.h
@@ -100,6 +100,7 @@
#include "io/MacroWriter.h"
#include "io/PngWriter.h"
#include "io/PovrayWriter.h"
+#include "io/ValueReader.h"
#include "io/ValueWriter.h"
#include "io/VtkWriter.h"
diff --git a/AMDiS/src/DOFAdmin.cc b/AMDiS/src/DOFAdmin.cc
index e66f5d6e..8b96d28e 100644
--- a/AMDiS/src/DOFAdmin.cc
+++ b/AMDiS/src/DOFAdmin.cc
@@ -258,7 +258,7 @@ namespace AMDiS {
}
- void DOFAdmin::compress(std::vector<DegreeOfFreedom> &new_dof)
+ void DOFAdmin::compress(std::vector<DegreeOfFreedom> &newDofIndex)
{
FUNCNAME("DOFAdmin::compress()");
@@ -268,17 +268,17 @@ namespace AMDiS {
// vector to mark used dofs
for (int i = 0; i < size; i++)
- new_dof[i] = -1;
+ newDofIndex[i] = -1;
// mark used dofs
DOFIteratorBase it(this, USED_DOFS);
for (it.reset(); !it.end(); ++it)
- new_dof[it.getDOFIndex()] = 1;
+ newDofIndex[it.getDOFIndex()] = 1;
int n = 0, last = 0;
for (int i = 0; i < size; i++) { /* create a MONOTONE compress */
- if (new_dof[i] == 1) {
- new_dof[i] = n++;
+ if (newDofIndex[i] == 1) {
+ newDofIndex[i] = n++;
last = i;
}
}
@@ -300,21 +300,19 @@ namespace AMDiS {
// get index of first changed dof
int first = last;
for (int i = 0; i < size; i++) {
- if (new_dof[i] < i && new_dof[i] >= 0) {
+ if (newDofIndex[i] < i && newDofIndex[i] >= 0) {
first = i;
break;
}
}
+
+ for (std::list<DOFIndexedBase*>::iterator di = dofIndexedList.begin();
+ di != dofIndexedList.end(); ++di)
+ (*di)->compressDOFIndexed(first, last, newDofIndex);
- std::list<DOFIndexedBase*>::iterator di;
- std::list<DOFIndexedBase*>::iterator end = dofIndexedList.end();
- for (di = dofIndexedList.begin(); di != end; ++di)
- (*di)->compressDOFIndexed(first, last, new_dof);
-
- std::list<DOFContainer*>::iterator dc;
- std::list<DOFContainer*>::iterator endc = dofContainerList.end();
- for (dc = dofContainerList.begin(); dc != endc; ++dc)
- (*dc)->compressDofContainer(n, new_dof);
+ for (std::list<DOFContainer*>::iterator dc = dofContainerList.begin();
+ dc != dofContainerList.end(); ++dc)
+ (*dc)->compressDofContainer(n, newDofIndex);
}
diff --git a/AMDiS/src/DOFAdmin.h b/AMDiS/src/DOFAdmin.h
index 07f9b8e0..737a0a77 100644
--- a/AMDiS/src/DOFAdmin.h
+++ b/AMDiS/src/DOFAdmin.h
@@ -102,7 +102,7 @@ namespace AMDiS {
* compression. This method is usually called after mesh adaption involving
* higher order elements or coarsening.
*/
- void compress(std::vector<DegreeOfFreedom> &new_dof);
+ void compress(std::vector<DegreeOfFreedom> &newDofIndex);
/// Returns an iterator to the begin of \ref dofIndexedList
std::list<DOFIndexedBase*>::iterator beginDOFIndexed()
diff --git a/AMDiS/src/DOFContainer.h b/AMDiS/src/DOFContainer.h
index b4349af9..12e39353 100644
--- a/AMDiS/src/DOFContainer.h
+++ b/AMDiS/src/DOFContainer.h
@@ -56,10 +56,11 @@ namespace AMDiS {
for (int i = 0; i < size; i++) {
int j = newDOF[operator[](i)];
- if (j >= 0)
- operator[](i) = j;
- else
- ERROR_EXIT("Invalid DOF %d in DOF container! (%d %d)\n", j, i, operator[](i));
+
+ TEST_EXIT_DBG(j >= 0)
+ ("Invalid DOF %d in DOF container! (%d %d)\n", j, i, operator[](i));
+
+ operator[](i) = j;
}
}
diff --git a/AMDiS/src/Element.cc b/AMDiS/src/Element.cc
index 7bfdffd4..cb2bf142 100644
--- a/AMDiS/src/Element.cc
+++ b/AMDiS/src/Element.cc
@@ -198,7 +198,7 @@ namespace AMDiS {
/* should be used only at the end of dof_compress()!!!!! */
/****************************************************************************/
- void Element::newDofFct1(const DOFAdmin* admin)
+ void Element::newDofFct1(const DOFAdmin* admin, std::vector<int>& newDofIndex)
{
int n0, nd, nd0;
@@ -207,7 +207,7 @@ namespace AMDiS {
nd0 = admin->getNumberOfPreDOFs(VERTEX);
n0 = admin->getMesh()->getNode(VERTEX);
for (int i = 0; i < vertices; i++)
- changeDofs1(admin, n0, nd0, nd, i);
+ changeDofs1(admin, newDofIndex, n0, nd0, nd, i);
}
if (mesh->getDim() > 1) {
@@ -216,7 +216,7 @@ namespace AMDiS {
nd0 = admin->getNumberOfPreDOFs(EDGE);
n0 = admin->getMesh()->getNode(EDGE);
for (int i = 0; i < edges; i++)
- changeDofs1(admin, n0, nd0, nd, i);
+ changeDofs1(admin, newDofIndex, n0, nd0, nd, i);
}
}
@@ -226,14 +226,14 @@ namespace AMDiS {
nd0 = admin->getNumberOfPreDOFs(FACE);
n0 = admin->getMesh()->getNode(FACE);
for (int i = 0; i < faces; i++)
- changeDofs1(admin, n0, nd0, nd, i);
+ changeDofs1(admin, newDofIndex, n0, nd0, nd, i);
}
}
if ((nd = admin->getNumberOfDofs(CENTER))) {
nd0 = admin->getNumberOfPreDOFs(CENTER);
n0 = admin->getMesh()->getNode(CENTER);
- changeDofs1(admin, n0, nd0, nd, 0);
+ changeDofs1(admin, newDofIndex, n0, nd0, nd, 0);
}
}
@@ -283,13 +283,14 @@ namespace AMDiS {
}
- void Element::changeDofs1(const DOFAdmin* admin, int n0, int nd0, int nd, int pos)
+ void Element::changeDofs1(const DOFAdmin* admin, std::vector<int>& newDofIndex,
+ int n0, int nd0, int nd, int pos)
{
DegreeOfFreedom *ldof = dof[n0 + pos] + nd0;
for (int j = 0; j < nd; j++) {
int k = ldof[j];
if (k >= 0)
- ldof[j] = -admin->getMesh()->newDOF[k] - 1;
+ ldof[j] = -newDofIndex[k] - 1;
}
}
diff --git a/AMDiS/src/Element.h b/AMDiS/src/Element.h
index 9c70dd2c..5d67b8f5 100644
--- a/AMDiS/src/Element.h
+++ b/AMDiS/src/Element.h
@@ -521,13 +521,14 @@ namespace AMDiS {
}
/// Used by friend class Mesh while dofCompress
- void newDofFct1(const DOFAdmin*);
+ void newDofFct1(const DOFAdmin*, std::vector<int>&);
/// Used by friend class Mesh while dofCompress
void newDofFct2(const DOFAdmin*);
/// Changes old dofs to negative new dofs
- void changeDofs1(const DOFAdmin* admin, int n0, int nd0, int nd, int pos);
+ void changeDofs1(const DOFAdmin* admin, std::vector<int>& newDofIndex,
+ int n0, int nd0, int nd, int pos);
/// Changes negative new dofs to positive
void changeDofs2(int n0, int nd0, int nd, int pos);
diff --git a/AMDiS/src/Mesh.cc b/AMDiS/src/Mesh.cc
index e1b7573f..f81a7457 100644
--- a/AMDiS/src/Mesh.cc
+++ b/AMDiS/src/Mesh.cc
@@ -160,7 +160,6 @@ namespace AMDiS {
nDof = m.nDof;
nNodeEl = m.nNodeEl;
node = m.node;
- newDOF = m.newDOF;
elementIndex = m.elementIndex;
initialized = m.initialized;
final_lambda = m.final_lambda;
@@ -271,7 +270,8 @@ namespace AMDiS {
TEST_EXIT(admin.size() == 1)("Not yet implemented for multiple admins!\n");
TEST_EXIT(admin[0])("There is something wrong!\n");
- // === Determine to all dofs the macro elements where the dof is part of. ===
+ // === Determine to all DOFs in mesh the macro elements where the DOF ===
+ // === is part of. ===
// Map that stores for each dof pointer (which may have a list of dofs)
// all macro element indices that own this dof.
@@ -463,9 +463,8 @@ namespace AMDiS {
compressAdmin->getHoleCount() < 1)
continue;
- newDOF.resize(size);
-
- compressAdmin->compress(newDOF);
+ std::vector<int> newDofIndex(size);
+ compressAdmin->compress(newDofIndex);
Flag fill_flag = (preserveCoarseDOFs ?
Mesh::CALL_EVERY_EL_PREORDER | Mesh::FILL_NOTHING :
@@ -473,7 +472,7 @@ namespace AMDiS {
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(this, -1, fill_flag);
while (elInfo) {
- elInfo->getElement()->newDofFct1(compressAdmin);
+ elInfo->getElement()->newDofFct1(compressAdmin, newDofIndex);
elInfo = stack.traverseNext(elInfo);
}
@@ -482,8 +481,6 @@ namespace AMDiS {
elInfo->getElement()->newDofFct2(compressAdmin);
elInfo = stack.traverseNext(elInfo);
}
-
- newDOF.resize(0);
}
}
diff --git a/AMDiS/src/Mesh.h b/AMDiS/src/Mesh.h
index 639b5f60..8107fe88 100644
--- a/AMDiS/src/Mesh.h
+++ b/AMDiS/src/Mesh.h
@@ -808,9 +808,6 @@ namespace AMDiS {
/// List of all MacroElements of this Mesh
std::deque<MacroElement*> macroElements;
- /// Needed during DOF compression (\ref DOFAdmin::compress).
- std::vector<DegreeOfFreedom> newDOF;
-
/// Used by check functions
static std::vector<DegreeOfFreedom> dof_used;
@@ -880,9 +877,6 @@ namespace AMDiS {
friend class MacroWriter;
friend class MacroElement;
friend class Element;
- friend void Element::newDofFct1(const DOFAdmin*);
- friend void Element::newDofFct2(const DOFAdmin*);
- friend void Element::changeDofs1(const DOFAdmin*, int, int, int, int);
};
}
diff --git a/AMDiS/src/parallel/MeshDistributor.cc b/AMDiS/src/parallel/MeshDistributor.cc
index 767151d2..de96eb81 100644
--- a/AMDiS/src/parallel/MeshDistributor.cc
+++ b/AMDiS/src/parallel/MeshDistributor.cc
@@ -56,8 +56,7 @@ namespace AMDiS {
deserialized(false),
writeSerializationFile(false),
repartitioningAllowed(false),
- lastMeshChangeIndex(0),
- macroElementStructureConsisten(false)
+ lastMeshChangeIndex(0)
{
FUNCNAME("MeshDistributor::ParalleDomainBase()");
@@ -139,36 +138,38 @@ namespace AMDiS {
#endif
- // === Create new global and local DOF numbering. ===
-
- // createLocalGlobalNumbering();
-
-
// === Remove all macro elements that are not part of the rank partition. ===
removeMacroElements();
- macroElementStructureConsisten = true;
- updateLocalGlobalNumbering(true);
+ // === Create new global and local DOF numbering. ===
+
- // === Reset all DOFAdmins of the mesh. ===
+ // We have to remove the VertexVectors, which contain periodic assoiciations,
+ // because they are not valid anymore after some macro elements have been removed
+ // and the corresponding DOFs were deleted.
+ for (std::map<BoundaryType, VertexVector*>::iterator it = mesh->getPeriodicAssociations().begin();
+ it != mesh->getPeriodicAssociations().end(); ++it)
+ const_cast<DOFAdmin&>(mesh->getDofAdmin(0)).removeDOFContainer(dynamic_cast<DOFContainer*>(it->second));
- updateDofAdmins();
+ updateLocalGlobalNumbering();
// === If in debug mode, make some tests. ===
#if (DEBUG != 0)
MSG("AMDiS runs in debug mode, so make some test ...\n");
+
ParallelDebug::testAllElements(*this);
debug::testSortedDofs(mesh, elMap);
ParallelDebug::testInteriorBoundary(*this);
ParallelDebug::testCommonDofs(*this, true);
ParallelDebug::testGlobalIndexByCoords(*this);
- MSG("Debug mode tests finished!\n");
debug::writeMesh(feSpace, -1, "macro_mesh");
+
+ MSG("Debug mode tests finished!\n");
#endif
@@ -188,8 +189,7 @@ namespace AMDiS {
debug::writeMesh(feSpace, -1, "gr_mesh");
#endif
- mesh->dofCompress();
- updateLocalGlobalNumbering(false);
+ updateLocalGlobalNumbering();
// === Update periodic mapping, if there are periodic boundaries. ===
@@ -305,30 +305,6 @@ namespace AMDiS {
{}
- void MeshDistributor::updateDofAdmins()
- {
- FUNCNAME("MeshDistributor::updateDofAdmins()");
-
- for (int i = 0; i < mesh->getNumberOfDOFAdmin(); i++) {
- DOFAdmin& admin = const_cast<DOFAdmin&>(mesh->getDofAdmin(i));
-
- // There must be always more allocated DOFs than used DOFs in DOFAdmin. Otherwise,
- // it is not possible to define a first DOF hole.
- if (static_cast<unsigned int>(admin.getSize()) == mapLocalGlobalDofs.size())
- admin.enlargeDOFLists();
-
- for (int j = 0; j < admin.getSize(); j++)
- admin.setDOFFree(j, true);
- for (unsigned int j = 0; j < mapLocalGlobalDofs.size(); j++)
- admin.setDOFFree(j, false);
-
- admin.setUsedSize(mapLocalGlobalDofs.size());
- admin.setUsedCount(mapLocalGlobalDofs.size());
- admin.setFirstHole(mapLocalGlobalDofs.size());
- }
- }
-
-
void MeshDistributor::testForMacroMesh()
{
FUNCNAME("MeshDistributor::testForMacroMesh()");
@@ -475,9 +451,6 @@ namespace AMDiS {
}
// Remove periodic vertex associations
- for (std::map<BoundaryType, VertexVector*>::iterator it = mesh->getPeriodicAssociations().begin();
- it != mesh->getPeriodicAssociations().end(); ++it)
- const_cast<DOFAdmin&>(mesh->getDofAdmin(0)).removeDOFContainer(dynamic_cast<DOFContainer*>(it->second));
mesh->getPeriodicAssociations().clear();
}
@@ -564,8 +537,8 @@ namespace AMDiS {
// === Because the mesh has been changed, update the DOF numbering and mappings. ===
- mesh->dofCompress();
- updateLocalGlobalNumbering(false);
+ updateLocalGlobalNumbering();
+
// === Update periodic mapping, if there are periodic boundaries. ===
@@ -1125,11 +1098,14 @@ namespace AMDiS {
mesh->removeMacroElements(deleteMacroElements, feSpace);
+ if (mpiRank == 0)
+ debug::writeElementIndexMesh(mesh, "elementIndexxxx.vtu");
// === Remove double DOFs. ===
MeshManipulation meshManipulation(mesh);
meshManipulation.deleteDoubleDofs(newMacroEl);
+
mesh->dofCompress();
DOFVector<double> tmp(feSpace, "tmp");
@@ -1137,7 +1113,6 @@ namespace AMDiS {
VtkWriter::writeFile(tmp, "after-repartition.vtu");
MSG("USED-SIZE B: %d\n", mesh->getDofAdmin(0).getUsedDofs());
- exit(0);
#if (DEBUG != 0)
ParallelDebug::testAllElements(*this);
@@ -1146,6 +1121,24 @@ namespace AMDiS {
partitioner->fillCoarsePartitionVec(&partitionVec);
updateInteriorBoundaryInfo();
+
+ updateLocalGlobalNumbering();
+
+ #if (DEBUG != 0)
+ MSG("AMDiS runs in debug mode, so make some test ...\n");
+
+ ParallelDebug::testAllElements(*this);
+ ParallelDebug::testInteriorBoundary(*this);
+ ParallelDebug::testCommonDofs(*this, true);
+ ParallelDebug::testGlobalIndexByCoords(*this);
+
+ debug::writeMesh(feSpace, -1, "macro_mesh");
+
+ MSG("Debug mode tests finished!\n");
+#endif
+
+
+ MSG("DONE\n");
}
@@ -1573,184 +1566,12 @@ namespace AMDiS {
}
- void MeshDistributor::createLocalGlobalNumbering()
- {
- FUNCNAME("MeshDistributor::createLocalGlobalNumbering()");
-
- // === Get rank information about DOFs. ===
-
- // Stores to each DOF pointer the set of ranks the DOF is part of.
- DofToPartitions partitionDofs;
- DofContainer rankDofs, rankAllDofs;
- DofToRank boundaryDofs;
-
- createDofMemberInfo(partitionDofs, rankDofs, rankAllDofs, boundaryDofs);
-
- nRankDofs = rankDofs.size();
- nOverallDofs = partitionDofs.size();
-
-
- // === Get starting position for global rank dof ordering. ====
-
- rstart = 0;
- mpiComm.Scan(&nRankDofs, &rstart, 1, MPI_INT, MPI_SUM);
- rstart -= nRankDofs;
-
- // === Create for all dofs in rank new indices. The new index must start at ===
- // === index 0, must be continues and have the same order as the indices ===
- // === had before. ===
-
- // Do not change the indices now, but create a new indexing and store it here.
- DofIndexMap rankDofsNewLocalIndex;
- isRankDof.clear();
- int i = 0;
- for (DofContainer::iterator dofIt = rankAllDofs.begin();
- dofIt != rankAllDofs.end(); ++dofIt) {
- rankDofsNewLocalIndex[*dofIt] = i;
- // First, we set all dofs in ranks partition to be owend by the rank. Later,
- // the dofs in ranks partition that are owned by other rank are set to false.
- isRankDof[i] = true;
- i++;
- }
-
-
- // === Create for all rank owned dofs a new global indexing. ===
-
- // Stores for dofs in rank a new global index.
- DofIndexMap rankDofsNewGlobalIndex;
- // Stores for all rank owned dofs a continues local index.
- DofIndexMap rankOwnedDofsNewLocalIndex;
-
- i = 0;
- for (DofContainer::iterator dofIt = rankDofs.begin();
- dofIt != rankDofs.end(); ++dofIt) {
- rankDofsNewGlobalIndex[*dofIt] = i + rstart;
- rankOwnedDofsNewLocalIndex[*dofIt] = i;
- i++;
- }
-
-
- // === Create information which dof indices must be send and which must ===
- // === be received to/from other ranks. ===
-
- // Stores to each rank a map from DOF pointers (which are all owned by the rank
- // and lie on an interior boundary) to new global DOF indices.
- std::map<int, DofIndexMap> sendNewDofs;
-
- // Maps to each rank the number of new DOF indices this rank will receive from.
- // All these DOFs are at an interior boundary on this rank, but are owned by
- // another rank.
- std::map<int, int> recvNewDofs;
-
- for (DofToRank::iterator it = boundaryDofs.begin(); it != boundaryDofs.end(); ++it) {
-
- if (it->second == mpiRank) {
- // If the boundary dof is a rank dof, it must be send to other ranks.
-
- // Search for all ranks that have this dof too.
- for (std::set<int>::iterator itRanks = partitionDofs[it->first].begin();
- itRanks != partitionDofs[it->first].end();
- ++itRanks) {
- if (*itRanks != mpiRank) {
- TEST_EXIT_DBG(rankDofsNewGlobalIndex.count(it->first) == 1)
- ("DOF Key not found!\n");
-
- sendNewDofs[*itRanks][it->first] = rankDofsNewGlobalIndex[it->first];
- }
- }
- } else {
- // If the boundary dof is not a rank dof, its new dof index (and later
- // also the dof values) must be received from another rank.
- if (recvNewDofs.find(it->second) == recvNewDofs.end())
- recvNewDofs[it->second] = 1;
- else
- recvNewDofs[it->second]++;
- }
- }
-
-
- // === Send and receive the dof indices at boundary. ===
-
- sendDofs.clear();
- for (std::map<int, DofIndexMap>::iterator sendIt = sendNewDofs.begin();
- sendIt != sendNewDofs.end(); ++sendIt)
- for (DofIndexMap::iterator dofIt = sendIt->second.begin();
- dofIt != sendIt->second.end(); ++dofIt)
- sendDofs[sendIt->first].push_back(dofIt->first);
-
- typedef std::vector<std::pair<DegreeOfFreedom, DegreeOfFreedom> > DofMapVec;
-
- StdMpi<DofIndexMap, DofMapVec> stdMpi(mpiComm);
- stdMpi.send(sendNewDofs);
- for (std::map<int, int>::iterator recvIt = recvNewDofs.begin();
- recvIt != recvNewDofs.end(); ++recvIt, i++)
- stdMpi.recv(recvIt->first, recvIt->second * 2);
- stdMpi.startCommunication<int>(MPI_INT);
- std::map<int, DofMapVec> &dofMap = stdMpi.getRecvData();
-
-
- // === Change global dof indices at boundary from other ranks. ===
-
- // Within this small data structure we track which dof index was already changed.
- // This is used to avoid the following situation: Assume, there are two dof indices
- // a and b in boundaryDofs. Then we have to change index a to b and b to c. When
- // the second rule applies, we have to avoid that not the first b, resulted from
- // changing a to b, is set to c, but the second one. Therefore, after the first
- // rule was applied, the dof pointer is set to false in this data structure and
- // is not allowed to be changed anymore.
- std::map<const DegreeOfFreedom*, bool> dofChanged;
- recvDofs.clear();
-
- for (DofToRank::iterator dofIt = boundaryDofs.begin(); dofIt != boundaryDofs.end();
- ++dofIt)
- dofChanged[dofIt->first] = false;
-
- for (std::map<int, DofMapVec>::iterator rankIt = dofMap.begin();
- rankIt != dofMap.end(); ++rankIt) {
-
- for (DofMapVec::iterator recvDofIt = rankIt->second.begin();
- recvDofIt != rankIt->second.end(); ++recvDofIt) {
-
- DegreeOfFreedom oldDof = recvDofIt->first;
- DegreeOfFreedom newGlobalDof = recvDofIt->second;
-
- bool found = false;
-
- // Iterate over all boundary dofs to find the dof which index we have to change.
-
- for (DofToRank::iterator dofIt = boundaryDofs.begin();
- dofIt != boundaryDofs.end(); ++dofIt) {
-
- if (*(dofIt->first) == oldDof && !dofChanged[dofIt->first]) {
- dofChanged[dofIt->first] = true;
-
- recvDofs[rankIt->first].push_back(dofIt->first);
- rankDofsNewGlobalIndex[dofIt->first] = newGlobalDof;
- isRankDof[rankDofsNewLocalIndex[dofIt->first]] = false;
-
- found = true;
- break;
- }
- }
-
- TEST_EXIT_DBG(found)("Should not happen!\n");
- }
- }
-
-
- // === Create now the local to global index and local to dof index mappings. ===
-
- createLocalMappings(rankDofsNewLocalIndex, rankOwnedDofsNewLocalIndex,
- rankDofsNewGlobalIndex);
-
- lastMeshChangeIndex = mesh->getChangeIndex();
- }
-
-
- void MeshDistributor::updateLocalGlobalNumbering(bool b)
+ void MeshDistributor::updateLocalGlobalNumbering()
{
FUNCNAME("MeshDistributor::updateLocalGlobalNumbering()");
+ mesh->dofCompress();
+
#if (DEBUG != 0)
debug::ElementIdxToDofs elMap;
debug::createSortedDofs(mesh, elMap);
@@ -1781,10 +1602,6 @@ namespace AMDiS {
sort(rankDofs.begin(), rankDofs.end(), cmpDofsByValue);
int nRankAllDofs = rankDofs.size();
- if (b)
- for (unsigned int i = 0; i < rankDofs.size(); i++)
- *const_cast<DegreeOfFreedom*>(rankDofs[i]) = i;
-
// === Traverse on interior boundaries and move all not ranked owned DOFs from ===
// === rankDofs to boundaryDofs. ===
@@ -1795,6 +1612,7 @@ namespace AMDiS {
sendDofs.clear();
recvDofs.clear();
+
for (InteriorBoundary::iterator it(myIntBoundary); !it.end(); ++it) {
DofContainer dofs;
it->rankObj.el->getVertexDofs(feSpace, it->rankObj, dofs);
@@ -1845,6 +1663,7 @@ namespace AMDiS {
for (int i = 0; i < nRankDofs; i++)
rankDofsNewGlobalIndex[rankDofs[i]] = i + rstart;
+
// === Send new DOF indices. ===
#if (DEBUG != 0)
@@ -1892,15 +1711,13 @@ namespace AMDiS {
// === Update dof admins due to new number of dofs. ===
- updateDofAdmins();
-
lastMeshChangeIndex = mesh->getChangeIndex();
#if (DEBUG != 0)
std::stringstream oss;
oss << "elementIndex-" << mpiRank << ".vtu";
- debug::writeElementIndexMesh(mesh, oss.str());
+ debug::writeElementIndexMesh(mesh, oss.str());
debug::testSortedDofs(mesh, elMap);
ParallelDebug::testCommonDofs(*this, true);
ParallelDebug::testGlobalIndexByCoords(*this);
@@ -1932,129 +1749,6 @@ namespace AMDiS {
}
- void MeshDistributor::createLocalMappings(DofIndexMap &rankDofsNewLocalIndex,
- DofIndexMap &rankOwnedDofsNewLocalIndex,
- DofIndexMap &rankDofsNewGlobalIndex)
- {
- FUNCNAME("MeshDistributor::createLocalMappings()");
-
-#if (DEBUG != 0)
- if (macroElementStructureConsisten) {
- std::set<const DegreeOfFreedom*> allDofs;
- debug::getAllDofs(feSpace, allDofs);
-
- for (std::set<const DegreeOfFreedom*>::iterator it = allDofs.begin();
- it != allDofs.end(); ++it) {
- TEST_EXIT(rankDofsNewLocalIndex.count(*it) == 1)
- ("Missing DOF %d in rankDofsNewLocalIndex!\n", **it);
-
- TEST_EXIT(rankDofsNewGlobalIndex.count(*it) == 1)
- ("Missing DOF %d in rankDofsNewGlobalIndex!\n", **it);
-
- if (isRankDof[**it]) {
- TEST_EXIT(rankOwnedDofsNewLocalIndex.count(*it) == 1)
- ("Missing DOF %d in rankOwnedDofsNewLocalIndex!\n", **it);
- }
- }
- }
-#endif
-
- mapLocalGlobalDofs.clear();
- mapLocalDofIndex.clear();
-
-
- // Iterate over all DOFs in ranks partition.
- for (DofIndexMap::iterator dofIt = rankDofsNewLocalIndex.begin();
- dofIt != rankDofsNewLocalIndex.end(); ++dofIt) {
- DegreeOfFreedom newLocalIndex = dofIt->second;
- DegreeOfFreedom newGlobalIndex = rankDofsNewGlobalIndex[dofIt->first];
-
- *const_cast<DegreeOfFreedom*>(dofIt->first) = newLocalIndex;
- mapLocalGlobalDofs[newLocalIndex] = newGlobalIndex;
- }
-
-
- for (DofIndexMap::iterator dofIt = rankOwnedDofsNewLocalIndex.begin();
- dofIt != rankOwnedDofsNewLocalIndex.end(); ++dofIt)
- mapLocalDofIndex[dofIt->second] = *(dofIt->first);
- }
-
-
- void MeshDistributor::createDofMemberInfo(DofToPartitions& partitionDofs,
- DofContainer& rankOwnedDofs,
- DofContainer& rankAllDofs,
- DofToRank& boundaryDofs)
- {
- partitionDofs.clear();
- rankOwnedDofs.clear();
- rankAllDofs.clear();
- boundaryDofs.clear();
-
- // === Determine to each dof the set of partitions the dof belongs to. ===
-
- ElementDofIterator elDofIt(feSpace);
-
- TraverseStack stack;
- ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
- while (elInfo) {
- Element *element = elInfo->getElement();
- elDofIt.reset(element);
- do {
- // Determine to each dof the partition(s) it corresponds to.
- partitionDofs[elDofIt.getDofPtr()].insert(partitionVec[element->getIndex()]);
- } while (elDofIt.next());
-
- elInfo = stack.traverseNext(elInfo);
- }
-
-
- // === Determine the set of ranks dofs and the dofs ownership at the boundary. ===
-
- // iterate over all DOFs
- for (DofToPartitions::iterator it = partitionDofs.begin();
- it != partitionDofs.end(); ++it) {
-
- // iterate over all partition the current DOF is part of.
- for (std::set<int>::iterator itpart1 = it->second.begin();
- itpart1 != it->second.end(); ++itpart1) {
-
- if (*itpart1 == mpiRank) {
- rankAllDofs.push_back(it->first);
-
- if (it->second.size() == 1) {
- rankOwnedDofs.push_back(it->first);
- } else {
- // This dof is at the ranks boundary. It is owned by the rank only if
- // the rank number is the highest of all ranks containing this dof.
-
- bool insert = true;
- int highestRank = mpiRank;
- for (std::set<int>::iterator itpart2 = it->second.begin();
- itpart2 != it->second.end(); ++itpart2) {
- if (*itpart2 > mpiRank)
- insert = false;
-
- if (*itpart2 > highestRank)
- highestRank = *itpart2;
- }
-
- if (insert)
- rankOwnedDofs.push_back(it->first);
-
- boundaryDofs[it->first] = highestRank;
- }
-
- break;
- }
-
- }
- }
-
- sort(rankAllDofs.begin(), rankAllDofs.end(), cmpDofsByValue);
- sort(rankOwnedDofs.begin(), rankOwnedDofs.end(), cmpDofsByValue);
- }
-
-
void MeshDistributor::createPeriodicMap()
{
FUNCNAME("MeshDistributor::createPeriodicMap()");
@@ -2186,7 +1880,6 @@ namespace AMDiS {
serialize(out, periodicDofAssociations);
SerUtil::serialize(out, rstart);
- SerUtil::serialize(out, macroElementStructureConsisten);
}
@@ -2236,7 +1929,6 @@ namespace AMDiS {
deserialize(in, periodicDofAssociations);
SerUtil::deserialize(in, rstart);
- SerUtil::deserialize(in, macroElementStructureConsisten);
deserialized = true;
}
diff --git a/AMDiS/src/parallel/MeshDistributor.h b/AMDiS/src/parallel/MeshDistributor.h
index 2ee2238c..2cd48236 100644
--- a/AMDiS/src/parallel/MeshDistributor.h
+++ b/AMDiS/src/parallel/MeshDistributor.h
@@ -94,8 +94,6 @@ namespace AMDiS {
*/
void checkMeshChange();
- void updateDofAdmins();
-
/** \brief
* Test, if the mesh consists of macro elements only. The mesh partitioning of
* the parallelization works for macro meshes only and would fail, if the mesh
@@ -249,11 +247,8 @@ namespace AMDiS {
/// Removes all macro elements from the mesh that are not part of ranks partition.
void removeMacroElements();
- /// Creates from a macro mesh a correct local and global DOF index numbering.
- void createLocalGlobalNumbering();
-
/// Updates the local and global DOF numbering after the mesh has been changed.
- void updateLocalGlobalNumbering(bool b);
+ void updateLocalGlobalNumbering();
/** \brief
* Creates to all dofs in rank's partition that are on a periodic boundary the
@@ -264,40 +259,6 @@ namespace AMDiS {
void createMacroElementInfo();
- /** \brief
- * This function create new mappings from local to global indices,
- * \ref mapLocalGlobalDofs, and from local to dof indices, \ref mapLocalDofIndex.
- * Furthermore, using the first argument the dof indices in ranks partition are
- * changed.
- *
- * \param[in] rankDofsNewLocalIndex Map from dof pointers of all dofs in rank
- * to new dof indices.
- * \param[in] rankOwnedDofsNewLocalIndex Map from dof pointers of dofs owned by
- * the rank to the new local index.
- * \param[in] rankDofsNewGlobalIndex Map from dof pointers of all dofs in rank
- * to the new global index.
- */
- void createLocalMappings(DofIndexMap &rankDofsNewLocalIndex,
- DofIndexMap &rankOwnedDofsNewLocalIndex,
- DofIndexMap &rankDofsNewGlobalIndex);
-
- /** \brief
- * This function traverses the whole mesh, i.e. before it is really partitioned,
- * and collects information about which DOF corresponds to which rank. Can only
- * be used, if \ref partitionVec is set correctly. This is only the case, when
- * the macro mesh is partitioned.
- *
- * \param[out] partitionDOFs Stores to each DOF pointer the set of ranks the DOF
- * is part of.
- * \param[out] rankDOFs Stores all rank DOFs.
- * \param[out] boundaryDOFs Stores all DOFs in ranks partition that are on an
- * interior boundary but correspond to another rank.
- */
- void createDofMemberInfo(DofToPartitions& partitionDofs,
- DofContainer& rankOwnedDofs,
- DofContainer& rankAllDofs,
- DofToRank& boundaryDofs);
-
/** \brief
* Checks for all given interior boundaries if the elements fit together on both
* sides of the boundaries. If this is not the case, the mesh is adapted. Because
@@ -607,11 +568,6 @@ namespace AMDiS {
*/
long lastMeshChangeIndex;
- /// This variable is true, if the macro elements are consistent with all other
- /// data structures. Within the initialization and during mesh redistribution this
- /// may not be the case.
- bool macroElementStructureConsisten;
-
std::map<int, std::vector<int> > macroElementNeighbours;
friend class ParallelDebug;
diff --git a/AMDiS/src/parallel/MeshManipulation.cc b/AMDiS/src/parallel/MeshManipulation.cc
index 4c04faac..e7516cc7 100644
--- a/AMDiS/src/parallel/MeshManipulation.cc
+++ b/AMDiS/src/parallel/MeshManipulation.cc
@@ -15,8 +15,11 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo);
}
+// MSG("----------------------------------\n");
deleteDoubleDofs(leafInMacroEl, newMacroEl, 0);
+// MSG("----------------------------------\n");
deleteDoubleDofs(leafInMacroEl, newMacroEl, 1);
+// MSG("----------------------------------\n");
}
@@ -54,7 +57,7 @@ namespace AMDiS {
newMacroEl.count(leafInMacroEl[el->getIndex()]) == 0 &&
newMacroEl.count(leafInMacroEl[neigh->getIndex()]) == 1)) {
- // MSG("EL: %d NEIGH: %d\n", el->getIndex(), neigh->getIndex());
+// MSG("EL: %d NEIGH: %d\n", el->getIndex(), neigh->getIndex());
if (el->getEdge(i) != neigh->getEdge(elInfo->getOppVertex(i))) {
std::vector<int> elEdgeIndex(2);
@@ -73,7 +76,8 @@ namespace AMDiS {
neighEdgeDof[0] = neigh->getDof(neighEdgeIndex[0], 0);
neighEdgeDof[1] = neigh->getDof(neighEdgeIndex[1], 0);
- // MSG("TEST DOFs %d %d <-> %d %d\n", elEdgeDof0, elEdgeDof1, neighEdgeDof0, neighEdgeDof1);
+// MSG("TEST DOFs %d %d <-> %d %d\n",
+// elEdgeDof[0], elEdgeDof[1], neighEdgeDof[0], neighEdgeDof[1]);
if ((elEdgeDof[0] < elEdgeDof[1] && neighEdgeDof[0] > neighEdgeDof[1]) ||
(elEdgeDof[0] > elEdgeDof[1] && neighEdgeDof[0] < neighEdgeDof[1])) {
@@ -122,14 +126,14 @@ namespace AMDiS {
} while (changed);
- elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS);
+ elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_EVERY_EL_PREORDER);
while (elInfo) {
for (int i = 0; i < mesh->getGeo(VERTEX); i++)
if (mapDelDofs.count(elInfo->getElement()->getDof(i)) == 1) {
- // MSG("SET DOF %d TO DOF %d in EL %d\n",
- // elInfo->getElement()->getDof(i, 0),
- // *(mapDelDofs[elInfo->getElement()->getDof(i)]),
- // elInfo->getElement()->getIndex());
+// MSG("SET DOF %d TO DOF %d in EL %d\n",
+// elInfo->getElement()->getDof(i, 0),
+// *(mapDelDofs[elInfo->getElement()->getDof(i)]),
+// elInfo->getElement()->getIndex());
elInfo->getElement()->setDof(i, const_cast<DegreeOfFreedom*>(mapDelDofs[elInfo->getElement()->getDof(i)]));
}
@@ -143,8 +147,6 @@ namespace AMDiS {
mesh->freeDof(const_cast<DegreeOfFreedom*>(it->first), VERTEX);
}
-
- MSG("FINISHED DELLING!\n");
}
}
diff --git a/AMDiS/src/parallel/ParallelDebug.cc b/AMDiS/src/parallel/ParallelDebug.cc
index f5af359b..c5096fe0 100644
--- a/AMDiS/src/parallel/ParallelDebug.cc
+++ b/AMDiS/src/parallel/ParallelDebug.cc
@@ -150,7 +150,7 @@ namespace AMDiS {
DOFVector<WorldVector<double> > coords(pdb.feSpace, "dofCorrds");
pdb.mesh->getDofIndexCoords(pdb.feSpace, coords);
-
+
for (RankToDofContainer::iterator it = pdb.sendDofs.begin();
it != pdb.sendDofs.end(); ++it)
for (DofContainer::iterator dofIt = it->second.begin();
--
GitLab