From e82f87e3edb051bdd4dff0925905158fb4a999fb Mon Sep 17 00:00:00 2001
From: Thomas Witkowski <thomas.witkowski@gmx.de>
Date: Sun, 18 Jul 2010 08:13:49 +0000
Subject: [PATCH] Fixed Sebastians bug no. 1 in parallelization code with
periodic boundary conditions.
---
AMDiS/libtool | 40 +++++++++++++--------------
AMDiS/src/parallel/MeshDistributor.cc | 21 +++++++++-----
AMDiS/src/parallel/MpiHelper.h | 2 ++
3 files changed, 36 insertions(+), 27 deletions(-)
diff --git a/AMDiS/libtool b/AMDiS/libtool
index 1a37a1c4..976a07ed 100755
--- a/AMDiS/libtool
+++ b/AMDiS/libtool
@@ -44,7 +44,7 @@ available_tags=" CXX F77"
# ### BEGIN LIBTOOL CONFIG
-# Libtool was configured on host deimos102:
+# Libtool was configured on host linux-dxyz.site:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
@@ -82,13 +82,13 @@ AR="ar"
AR_FLAGS="cru"
# A C compiler.
-LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
+LTCC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
# A language-specific compiler.
-CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
+CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
# Is the compiler the GNU C compiler?
with_gcc=yes
@@ -325,10 +325,10 @@ variables_saved_for_relink="PATH LD_LIBRARY_PATH LD_RUN_PATH GCC_EXEC_PREFIX COM
link_all_deplibs=unknown
# Compile-time system search path for libraries
-sys_lib_search_path_spec=" /fastfs/wir/local/lib/x86_64-suse-linux/4.1.2/ /fastfs/wir/local/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/ /usr/lib/gcc/x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/ /lib/x86_64-suse-linux/4.1.2/ /lib/../lib64/ /usr/lib/x86_64-suse-linux/4.1.2/ /usr/lib/../lib64/ /fastfs/wir/local/lib/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../ /lib/ /usr/lib/"
+sys_lib_search_path_spec=" /usr/lib64/gcc/x86_64-suse-linux/4.5/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib/x86_64-suse-linux/4.5/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../x86_64-suse-linux/4.5/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/ /lib/x86_64-suse-linux/4.5/ /lib/../lib64/ /usr/lib/x86_64-suse-linux/4.5/ /usr/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../ /lib/ /usr/lib/"
# Run-time system search path for libraries
-sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
+sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/lib64/Xaw3d /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/lib/Xaw3d /usr/x86_64-suse-linux/lib /usr/local/lib /opt/kde3/lib /lib64 /lib /usr/lib64 /usr/lib /usr/local/lib64 /opt/kde3/lib64 /usr/lib64/graphviz /usr/lib64/graphviz/sharp /usr/lib64/graphviz/java /usr/lib64/graphviz/perl /usr/lib64/graphviz/php /usr/lib64/graphviz/ocaml /usr/lib64/graphviz/python /usr/lib64/graphviz/lua /usr/lib64/graphviz/tcl /usr/lib64/graphviz/guile /usr/lib64/graphviz/ruby "
# Fix the shell variable $srcfile for the compiler.
fix_srcfile_path=""
@@ -6760,7 +6760,7 @@ build_old_libs=`case $build_libtool_libs in yes) $echo no;; *) $echo yes;; esac`
# End:
# ### BEGIN LIBTOOL TAG CONFIG: CXX
-# Libtool was configured on host deimos102:
+# Libtool was configured on host linux-dxyz.site:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
@@ -6798,13 +6798,13 @@ AR="ar"
AR_FLAGS="cru"
# A C compiler.
-LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
+LTCC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
# A language-specific compiler.
-CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicxx"
+CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicxx"
# Is the compiler the GNU C compiler?
with_gcc=yes
@@ -6942,11 +6942,11 @@ striplib="strip --strip-unneeded"
# Dependencies to place before the objects being linked to create a
# shared library.
-predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtbeginS.o"
+predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.5/crtbeginS.o"
# Dependencies to place after the objects being linked to create a
# shared library.
-postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crtn.o"
+postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crtn.o"
# Dependencies to place before the objects being linked to create a
# shared library.
@@ -6954,11 +6954,11 @@ predeps=""
# Dependencies to place after the objects being linked to create a
# shared library.
-postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
+postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking
# a shared library.
-compiler_lib_search_path="-L/usr/lib64 -L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/fastfs/wir/local/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
+compiler_lib_search_path="-L/usr/lib64/mpi/gcc/openmpi/lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
# Method to check whether dependent libraries are shared objects.
deplibs_check_method="pass_all"
@@ -7038,10 +7038,10 @@ variables_saved_for_relink="PATH LD_LIBRARY_PATH LD_RUN_PATH GCC_EXEC_PREFIX COM
link_all_deplibs=unknown
# Compile-time system search path for libraries
-sys_lib_search_path_spec=" /fastfs/wir/local/lib/x86_64-suse-linux/4.1.2/ /fastfs/wir/local/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/ /usr/lib/gcc/x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/ /lib/x86_64-suse-linux/4.1.2/ /lib/../lib64/ /usr/lib/x86_64-suse-linux/4.1.2/ /usr/lib/../lib64/ /fastfs/wir/local/lib/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../ /lib/ /usr/lib/"
+sys_lib_search_path_spec=" /usr/lib64/gcc/x86_64-suse-linux/4.5/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib/x86_64-suse-linux/4.5/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../x86_64-suse-linux/4.5/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/ /lib/x86_64-suse-linux/4.5/ /lib/../lib64/ /usr/lib/x86_64-suse-linux/4.5/ /usr/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../ /lib/ /usr/lib/"
# Run-time system search path for libraries
-sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
+sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/lib64/Xaw3d /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/lib/Xaw3d /usr/x86_64-suse-linux/lib /usr/local/lib /opt/kde3/lib /lib64 /lib /usr/lib64 /usr/lib /usr/local/lib64 /opt/kde3/lib64 /usr/lib64/graphviz /usr/lib64/graphviz/sharp /usr/lib64/graphviz/java /usr/lib64/graphviz/perl /usr/lib64/graphviz/php /usr/lib64/graphviz/ocaml /usr/lib64/graphviz/python /usr/lib64/graphviz/lua /usr/lib64/graphviz/tcl /usr/lib64/graphviz/guile /usr/lib64/graphviz/ruby "
# Fix the shell variable $srcfile for the compiler.
fix_srcfile_path=""
@@ -7065,7 +7065,7 @@ include_expsyms=""
# ### BEGIN LIBTOOL TAG CONFIG: F77
-# Libtool was configured on host deimos102:
+# Libtool was configured on host linux-dxyz.site:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
@@ -7103,16 +7103,16 @@ AR="ar"
AR_FLAGS="cru"
# A C compiler.
-LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
+LTCC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
# A language-specific compiler.
-CC="g77"
+CC="gfortran"
# Is the compiler the GNU C compiler?
-with_gcc=
+with_gcc=yes
# An ERE matcher.
EGREP="grep -E"
@@ -7346,10 +7346,10 @@ variables_saved_for_relink="PATH LD_LIBRARY_PATH LD_RUN_PATH GCC_EXEC_PREFIX COM
link_all_deplibs=unknown
# Compile-time system search path for libraries
-sys_lib_search_path_spec=" /fastfs/wir/local/lib/x86_64-suse-linux/3.3.5/ /fastfs/wir/local/lib/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/ /usr/lib/gcc/x86_64-suse-linux/3.3.5/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/../../../../x86_64-suse-linux/lib/x86_64-suse-linux/3.3.5/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/../../../../x86_64-suse-linux/lib/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/../../../x86_64-suse-linux/3.3.5/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/../../../ /lib/x86_64-suse-linux/3.3.5/ /lib/ /usr/lib/x86_64-suse-linux/3.3.5/ /usr/lib/"
+sys_lib_search_path_spec=" /usr/lib64/gcc/x86_64-suse-linux/4.5/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib/x86_64-suse-linux/4.5/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../x86_64-suse-linux/4.5/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/ /lib/x86_64-suse-linux/4.5/ /lib/../lib64/ /usr/lib/x86_64-suse-linux/4.5/ /usr/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib/ /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../ /lib/ /usr/lib/"
# Run-time system search path for libraries
-sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
+sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/lib64/Xaw3d /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/lib/Xaw3d /usr/x86_64-suse-linux/lib /usr/local/lib /opt/kde3/lib /lib64 /lib /usr/lib64 /usr/lib /usr/local/lib64 /opt/kde3/lib64 /usr/lib64/graphviz /usr/lib64/graphviz/sharp /usr/lib64/graphviz/java /usr/lib64/graphviz/perl /usr/lib64/graphviz/php /usr/lib64/graphviz/ocaml /usr/lib64/graphviz/python /usr/lib64/graphviz/lua /usr/lib64/graphviz/tcl /usr/lib64/graphviz/guile /usr/lib64/graphviz/ruby "
# Fix the shell variable $srcfile for the compiler.
fix_srcfile_path=""
diff --git a/AMDiS/src/parallel/MeshDistributor.cc b/AMDiS/src/parallel/MeshDistributor.cc
index 1afe17ff..cb410076 100644
--- a/AMDiS/src/parallel/MeshDistributor.cc
+++ b/AMDiS/src/parallel/MeshDistributor.cc
@@ -1992,6 +1992,7 @@ namespace AMDiS {
DofContainer& dofs = rankPeriodicDofs[it->first];
for (std::vector<AtomicBoundary>::iterator boundIt = it->second.begin();
boundIt != it->second.end(); ++boundIt) {
+ int nDofs = dofs.size();
Element *el = boundIt->rankObj.el;
if (boundIt->rankObj.reverseMode) {
@@ -2010,7 +2011,7 @@ namespace AMDiS {
el->getVertexDofs(feSpace, boundIt->rankObj, dofs);
el->getNonVertexDofs(feSpace, boundIt->rankObj, dofs);
- for (unsigned int i = 0; i < dofs.size(); i++)
+ for (unsigned int i = 0; i < (dofs.size() - nDofs); i++)
rankToDofType[it->first].push_back(boundIt->type);
}
@@ -2048,16 +2049,21 @@ namespace AMDiS {
int mapGlobalDofIndex = stdMpi.getRecvData(it->first)[i];
BoundaryType type = types[i];
- periodicDof[type][globalDofIndex] = mapGlobalDofIndex;
- periodicDofAssociations[globalDofIndex].insert(type);
- dofFromRank[globalDofIndex].insert(it->first);
- }
+ // Check if this global dof with the corresponding boundary type was
+ // not added before by another periodic boundary from other rank.
+ if (periodicDofAssociations[globalDofIndex].count(type) == 0) {
+ periodicDof[type][globalDofIndex] = mapGlobalDofIndex;
+ periodicDofAssociations[globalDofIndex].insert(type);
+ dofFromRank[globalDofIndex].insert(it->first);
+ }
+ }
}
- if (dofFromRank.size() > 0)
+ if (dofFromRank.size() > 0) {
TEST_EXIT_DBG(mesh->getDim() == 2)
("Periodic boundary corner problem must be generalized to 3d!\n");
+ }
MPI::Request request[min(static_cast<int>(periodicBoundary.boundary.size() * 2), 4)];
int requestCounter = 0;
@@ -2067,7 +2073,8 @@ namespace AMDiS {
it != dofFromRank.end(); ++it) {
if (it->second.size() == 2) {
TEST_EXIT_DBG(periodicDofAssociations[it->first].size() == 2)
- ("Missing periodic dof!\n");
+ ("DOF %d has only %d periodic associations!\n",
+ it->first, periodicDofAssociations[it->first].size());
int type0 = *(periodicDofAssociations[it->first].begin());
int type1 = *(++(periodicDofAssociations[it->first].begin()));
diff --git a/AMDiS/src/parallel/MpiHelper.h b/AMDiS/src/parallel/MpiHelper.h
index 1dde56e1..e77e6c8d 100644
--- a/AMDiS/src/parallel/MpiHelper.h
+++ b/AMDiS/src/parallel/MpiHelper.h
@@ -24,6 +24,8 @@
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+#include <time.h>
+#include <stdlib.h>
#include <mpi.h>
namespace AMDiS {
--
GitLab