From d341b941b4c6bf761d42c8719b1ef020852aa40e Mon Sep 17 00:00:00 2001
From: Thomas Witkowski <thomas.witkowski@gmx.de>
Date: Fri, 19 Jun 2009 15:28:15 +0000
Subject: [PATCH] Huh, pdd works somehow ...
---
AMDiS/libtool | 64 ++++++++++-----------
AMDiS/src/ParallelDomainProblem.cc | 89 +++++++++++++++++++++---------
2 files changed, 96 insertions(+), 57 deletions(-)
diff --git a/AMDiS/libtool b/AMDiS/libtool
index 41ace2e6..5f608e9a 100755
--- a/AMDiS/libtool
+++ b/AMDiS/libtool
@@ -30,10 +30,10 @@
# the same distribution terms that you use for the rest of that program.
# A sed program that does not truncate output.
-SED="/bin/sed"
+SED="/usr/bin/sed"
# Sed that helps us avoid accidentally triggering echo(1) options like -n.
-Xsed="/bin/sed -e 1s/^X//"
+Xsed="/usr/bin/sed -e 1s/^X//"
# The HP-UX ksh and POSIX shell print the target directory to stdout
# if CDPATH is set.
@@ -44,7 +44,7 @@ available_tags=" CXX F77"
# ### BEGIN LIBTOOL CONFIG
-# Libtool was configured on host NWRW15:
+# Libtool was configured on host p2q001:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
@@ -66,12 +66,12 @@ fast_install=yes
# The host system.
host_alias=
-host=i686-pc-linux-gnu
+host=x86_64-unknown-linux-gnu
host_os=linux-gnu
# The build system.
build_alias=
-build=i686-pc-linux-gnu
+build=x86_64-unknown-linux-gnu
build_os=linux-gnu
# An echo program that does not interpret backslashes.
@@ -82,13 +82,13 @@ AR="ar"
AR_FLAGS="cru"
# A C compiler.
-LTCC="gcc"
+LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
# A language-specific compiler.
-CC="gcc"
+CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# Is the compiler the GNU C compiler?
with_gcc=yes
@@ -97,7 +97,7 @@ with_gcc=yes
EGREP="grep -E"
# The linker used to build libraries.
-LD="/usr/bin/ld"
+LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
# Whether we need hard or soft links.
LN_S="ln -s"
@@ -171,7 +171,7 @@ dlopen_self=unknown
dlopen_self_static=unknown
# Compiler flag to prevent dynamic linking.
-link_static_flag="-static"
+link_static_flag=""
# Compiler flag to turn off builtin functions.
no_builtin_flag=" -fno-builtin"
@@ -325,10 +325,10 @@ variables_saved_for_relink="PATH LD_LIBRARY_PATH LD_RUN_PATH GCC_EXEC_PREFIX COM
link_all_deplibs=unknown
# Compile-time system search path for libraries
-sys_lib_search_path_spec=" /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../ /lib/i386-redhat-linux/4.1.2/ /lib/ /usr/lib/i386-redhat-linux/4.1.2/ /usr/lib/"
+sys_lib_search_path_spec=" /usr/lib64/gcc/x86_64-suse-linux/4.1.2/ /usr/lib/gcc/x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/ /lib/x86_64-suse-linux/4.1.2/ /lib/../lib64/ /usr/lib/x86_64-suse-linux/4.1.2/ /usr/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../ /lib/ /usr/lib/"
# Run-time system search path for libraries
-sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/lib/octave-2.9.9 /usr/lib/qt-3.3/lib "
+sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
# Fix the shell variable $srcfile for the compiler.
fix_srcfile_path=""
@@ -6760,7 +6760,7 @@ build_old_libs=`case $build_libtool_libs in yes) $echo no;; *) $echo yes;; esac`
# End:
# ### BEGIN LIBTOOL TAG CONFIG: CXX
-# Libtool was configured on host NWRW15:
+# Libtool was configured on host p2q001:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
@@ -6782,12 +6782,12 @@ fast_install=yes
# The host system.
host_alias=
-host=i686-pc-linux-gnu
+host=x86_64-unknown-linux-gnu
host_os=linux-gnu
# The build system.
build_alias=
-build=i686-pc-linux-gnu
+build=x86_64-unknown-linux-gnu
build_os=linux-gnu
# An echo program that does not interpret backslashes.
@@ -6798,13 +6798,13 @@ AR="ar"
AR_FLAGS="cru"
# A C compiler.
-LTCC="gcc"
+LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
# A language-specific compiler.
-CC="g++"
+CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpiCC"
# Is the compiler the GNU C compiler?
with_gcc=yes
@@ -6813,7 +6813,7 @@ with_gcc=yes
EGREP="grep -E"
# The linker used to build libraries.
-LD="/usr/bin/ld"
+LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
# Whether we need hard or soft links.
LN_S="ln -s"
@@ -6887,7 +6887,7 @@ dlopen_self=unknown
dlopen_self_static=unknown
# Compiler flag to prevent dynamic linking.
-link_static_flag="-static"
+link_static_flag=""
# Compiler flag to turn off builtin functions.
no_builtin_flag=" -fno-builtin"
@@ -6942,11 +6942,11 @@ striplib="strip --strip-unneeded"
# Dependencies to place before the objects being linked to create a
# shared library.
-predep_objects="/usr/lib/gcc/i386-redhat-linux/4.1.2/../../../crti.o /usr/lib/gcc/i386-redhat-linux/4.1.2/crtbeginS.o"
+predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtbeginS.o"
# Dependencies to place after the objects being linked to create a
# shared library.
-postdep_objects="/usr/lib/gcc/i386-redhat-linux/4.1.2/crtendS.o /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../crtn.o"
+postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crtn.o"
# Dependencies to place before the objects being linked to create a
# shared library.
@@ -6954,11 +6954,11 @@ predeps=""
# Dependencies to place after the objects being linked to create a
# shared library.
-postdeps="-lstdc++ -lm -lgcc_s -lc -lgcc_s"
+postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking
# a shared library.
-compiler_lib_search_path="-L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2/../../.."
+compiler_lib_search_path="-L/usr/lib64 -L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# Method to check whether dependent libraries are shared objects.
deplibs_check_method="pass_all"
@@ -7038,10 +7038,10 @@ variables_saved_for_relink="PATH LD_LIBRARY_PATH LD_RUN_PATH GCC_EXEC_PREFIX COM
link_all_deplibs=unknown
# Compile-time system search path for libraries
-sys_lib_search_path_spec=" /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../ /lib/i386-redhat-linux/4.1.2/ /lib/ /usr/lib/i386-redhat-linux/4.1.2/ /usr/lib/"
+sys_lib_search_path_spec=" /usr/lib64/gcc/x86_64-suse-linux/4.1.2/ /usr/lib/gcc/x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/ /lib/x86_64-suse-linux/4.1.2/ /lib/../lib64/ /usr/lib/x86_64-suse-linux/4.1.2/ /usr/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../ /lib/ /usr/lib/"
# Run-time system search path for libraries
-sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/lib/octave-2.9.9 /usr/lib/qt-3.3/lib "
+sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
# Fix the shell variable $srcfile for the compiler.
fix_srcfile_path=""
@@ -7065,7 +7065,7 @@ include_expsyms=""
# ### BEGIN LIBTOOL TAG CONFIG: F77
-# Libtool was configured on host NWRW15:
+# Libtool was configured on host p2q001:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
@@ -7087,12 +7087,12 @@ fast_install=yes
# The host system.
host_alias=
-host=i686-pc-linux-gnu
+host=x86_64-unknown-linux-gnu
host_os=linux-gnu
# The build system.
build_alias=
-build=i686-pc-linux-gnu
+build=x86_64-unknown-linux-gnu
build_os=linux-gnu
# An echo program that does not interpret backslashes.
@@ -7103,7 +7103,7 @@ AR="ar"
AR_FLAGS="cru"
# A C compiler.
-LTCC="gcc"
+LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
@@ -7112,13 +7112,13 @@ LTCFLAGS="-g -O2"
CC="g77"
# Is the compiler the GNU C compiler?
-with_gcc=yes
+with_gcc=
# An ERE matcher.
EGREP="grep -E"
# The linker used to build libraries.
-LD="/usr/bin/ld"
+LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
# Whether we need hard or soft links.
LN_S="ln -s"
@@ -7346,10 +7346,10 @@ variables_saved_for_relink="PATH LD_LIBRARY_PATH LD_RUN_PATH GCC_EXEC_PREFIX COM
link_all_deplibs=unknown
# Compile-time system search path for libraries
-sys_lib_search_path_spec=" /usr/lib/gcc/i386-redhat-linux/3.4.6/ /usr/lib/gcc/i386-redhat-linux/3.4.6/ /usr/lib/gcc/i386-redhat-linux/3.4.6/../../../../i386-redhat-linux/lib/i386-redhat-linux/3.4.6/ /usr/lib/gcc/i386-redhat-linux/3.4.6/../../../../i386-redhat-linux/lib/ /usr/lib/gcc/i386-redhat-linux/3.4.6/../../../i386-redhat-linux/3.4.6/ /usr/lib/gcc/i386-redhat-linux/3.4.6/../../../ /lib/i386-redhat-linux/3.4.6/ /lib/ /usr/lib/i386-redhat-linux/3.4.6/ /usr/lib/"
+sys_lib_search_path_spec=" /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/ /usr/lib/gcc/x86_64-suse-linux/3.3.5/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/../../../../x86_64-suse-linux/lib/x86_64-suse-linux/3.3.5/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/../../../../x86_64-suse-linux/lib/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/../../../x86_64-suse-linux/3.3.5/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/../../../ /lib/x86_64-suse-linux/3.3.5/ /lib/ /usr/lib/x86_64-suse-linux/3.3.5/ /usr/lib/"
# Run-time system search path for libraries
-sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/lib/octave-2.9.9 /usr/lib/qt-3.3/lib "
+sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
# Fix the shell variable $srcfile for the compiler.
fix_srcfile_path=""
diff --git a/AMDiS/src/ParallelDomainProblem.cc b/AMDiS/src/ParallelDomainProblem.cc
index 0489a891..33f639f3 100644
--- a/AMDiS/src/ParallelDomainProblem.cc
+++ b/AMDiS/src/ParallelDomainProblem.cc
@@ -88,7 +88,11 @@ namespace AMDiS {
// === Global refinements. ===
- refinementManager->globalRefine(mesh, 12);
+ int globalRefinement = 0;
+ GET_PARAMETER(0, "testgr", "%d", &globalRefinement);
+ std::cout << "GR = " << globalRefinement << std::endl;
+
+ refinementManager->globalRefine(mesh, globalRefinement);
updateLocalGlobalNumbering(nRankDOFs, nOverallDOFs);
@@ -154,7 +158,7 @@ namespace AMDiS {
void ParallelDomainProblemBase::solvePetscMatrix(DOFVector<double> *vec)
{
clock_t t = clock();
-
+
KSP ksp;
PC pc;
@@ -179,6 +183,9 @@ namespace AMDiS {
std::vector<double*> sendBuffers(sendDofs.size());
std::vector<double*> recvBuffers(recvDofs.size());
+
+ MPI::Request request[sendDofs.size() + recvDofs.size()];
+ int requestCounter = 0;
int i = 0;
for (std::map<int, std::vector<const DegreeOfFreedom*> >::iterator
@@ -191,7 +198,8 @@ namespace AMDiS {
for (int j = 0; j < nSendDOFs; j++)
sendBuffers[i][j] = (*vec)[(sendIt->second)[j][0]];
- mpiComm.Isend(sendBuffers[i], nSendDOFs, MPI_DOUBLE, sendIt->first, 0);
+ request[requestCounter++] =
+ mpiComm.Isend(sendBuffers[i], nSendDOFs, MPI_DOUBLE, sendIt->first, 0);
}
i = 0;
@@ -202,11 +210,13 @@ namespace AMDiS {
int nRecvDOFs = recvIt->second.size();
recvBuffers[i] = new double[nRecvDOFs];
- mpiComm.Irecv(recvBuffers[i], nRecvDOFs, MPI_DOUBLE, recvIt->first, 0);
+ request[requestCounter++] =
+ mpiComm.Irecv(recvBuffers[i], nRecvDOFs, MPI_DOUBLE, recvIt->first, 0);
}
-
- mpiComm.Barrier();
+
+ MPI::Request::Waitall(requestCounter, request);
+
i = 0;
for (std::map<int, std::vector<const DegreeOfFreedom*> >::iterator
@@ -355,30 +365,43 @@ namespace AMDiS {
std::vector<int*> sendBuffers;
std::vector<int*> recvBuffers;
- for (std::map<int, std::vector<AtomicBoundary> >::iterator rankIt = myIntBoundary.boundary.begin();
+ MPI::Request request[myIntBoundary.boundary.size() +
+ otherIntBoundary.boundary.size()];
+ int requestCounter = 0;
+
+ for (std::map<int, std::vector<AtomicBoundary> >::iterator rankIt =
+ myIntBoundary.boundary.begin();
rankIt != myIntBoundary.boundary.end();
++rankIt) {
- int* buffer = new int[rankIt->second.size()];
- for (int i = 0; i < static_cast<int>(rankIt->second.size()); i++)
+ int nSendInt = rankIt->second.size();
+ int* buffer = new int[nSendInt];
+ for (int i = 0; i < nSendInt; i++)
buffer[i] = (rankIt->second)[i].rankObject.el->getIndex();
sendBuffers.push_back(buffer);
- mpiComm.Isend(buffer, rankIt->second.size(), MPI_INT, rankIt->first, 0);
+ request[requestCounter++] =
+ mpiComm.Isend(buffer, nSendInt, MPI_INT, rankIt->first, 0);
}
- for (std::map<int, std::vector<AtomicBoundary> >::iterator rankIt = otherIntBoundary.boundary.begin();
+ for (std::map<int, std::vector<AtomicBoundary> >::iterator rankIt =
+ otherIntBoundary.boundary.begin();
rankIt != otherIntBoundary.boundary.end();
++rankIt) {
- int *buffer = new int[rankIt->second.size()];
+ int nRecvInt = rankIt->second.size();
+ int *buffer = new int[nRecvInt];
recvBuffers.push_back(buffer);
- mpiComm.Irecv(buffer, rankIt->second.size(), MPI_INT, rankIt->first, 0);
+ request[requestCounter++] =
+ mpiComm.Irecv(buffer, nRecvInt, MPI_INT, rankIt->first, 0);
}
- mpiComm.Barrier();
+
+ MPI::Request::Waitall(requestCounter, request);
+
int i = 0;
- for (std::map<int, std::vector<AtomicBoundary> >::iterator rankIt = otherIntBoundary.boundary.begin();
+ for (std::map<int, std::vector<AtomicBoundary> >::iterator rankIt =
+ otherIntBoundary.boundary.begin();
rankIt != otherIntBoundary.boundary.end();
++rankIt) {
@@ -452,7 +475,7 @@ namespace AMDiS {
// === Get starting position for global rank dof ordering. ====
int rstart = 0;
- MPI_Scan(&nRankDOFs, &rstart, 1, MPI_INT, MPI_SUM, PETSC_COMM_WORLD);
+ mpiComm.Scan(&nRankDOFs, &rstart, 1, MPI_INT, MPI_SUM);
rstart -= nRankDOFs;
@@ -510,6 +533,9 @@ namespace AMDiS {
sendDofs.clear();
recvDofs.clear();
+ MPI::Request request[sendNewDofs.size() + recvNewDofs.size()];
+ int requestCounter = 0;
+
int i = 0;
for (std::map<int, std::map<const DegreeOfFreedom*, DegreeOfFreedom> >::iterator
sendIt = sendNewDofs.begin();
@@ -528,7 +554,8 @@ namespace AMDiS {
sendDofs[sendIt->first].push_back(dofIt->first);
}
- mpiComm.Isend(sendBuffers[i], nSendDofs, MPI_INT, sendIt->first, 0);
+ request[requestCounter++] =
+ mpiComm.Isend(sendBuffers[i], nSendDofs, MPI_INT, sendIt->first, 0);
}
i = 0;
@@ -538,11 +565,12 @@ namespace AMDiS {
int nRecvDOFs = recvIt->second * 2;
recvBuffers[i] = new int[nRecvDOFs];
- mpiComm.Irecv(recvBuffers[i], nRecvDOFs, MPI_INT, recvIt->first, 0);
+ request[requestCounter++] =
+ mpiComm.Irecv(recvBuffers[i], nRecvDOFs, MPI_INT, recvIt->first, 0);
}
- mpiComm.Barrier();
+ MPI::Request::Waitall(requestCounter, request);
// === Delete send buffers. ===
@@ -735,13 +763,13 @@ namespace AMDiS {
// === Get starting position for global rank dof ordering. ====
int rstart = 0;
- MPI_Scan(&nRankDOFs, &rstart, 1, MPI_INT, MPI_SUM, PETSC_COMM_WORLD);
+ mpiComm.Scan(&nRankDOFs, &rstart, 1, MPI_INT, MPI_SUM);
rstart -= nRankDOFs;
// === Calculate number of overall DOFs of all partitions. ===
- MPI_Allreduce(&nRankDOFs, &nOverallDOFs, 1, MPI_INT, MPI_SUM, PETSC_COMM_WORLD);
+ mpiComm.Allreduce(&nRankDOFs, &nOverallDOFs, 1, MPI_INT, MPI_SUM);
// === Create new local DOF index numbering. ===
@@ -766,6 +794,9 @@ namespace AMDiS {
std::vector<int*> sendBuffers(sendNewDofs.size());
std::vector<int*> recvBuffers(recvNewDofs.size());
+ MPI::Request request[sendNewDofs.size() + recvNewDofs.size()];
+ int requestCounter = 0;
+
i = 0;
for (std::map<int, std::vector<const DegreeOfFreedom*> >::iterator
sendIt = sendNewDofs.begin();
@@ -779,7 +810,8 @@ namespace AMDiS {
++dofIt)
sendBuffers[i][c++] = (*dofIt)[0] + rstart;
- mpiComm.Isend(sendBuffers[i], nSendDofs, MPI_INT, sendIt->first, 0);
+ request[requestCounter++] =
+ mpiComm.Isend(sendBuffers[i], nSendDofs, MPI_INT, sendIt->first, 0);
}
i = 0;
@@ -789,10 +821,13 @@ namespace AMDiS {
int nRecvDofs = recvIt->second.size();
recvBuffers[i] = new int[nRecvDofs];
- mpiComm.Irecv(recvBuffers[i], nRecvDofs, MPI_INT, recvIt->first, 0);
+ request[requestCounter++] =
+ mpiComm.Irecv(recvBuffers[i], nRecvDofs, MPI_INT, recvIt->first, 0);
}
- mpiComm.Barrier();
+
+ MPI::Request::Waitall(requestCounter, request);
+
i = 0;
for (std::map<int, std::vector<const DegreeOfFreedom*> >::iterator
@@ -945,7 +980,11 @@ namespace AMDiS {
{
ParallelDomainProblemBase::initParallelization(adaptInfo);
- probScal->getSystemMatrix()->setIsRankDOF(isRankDOF);
+ DOFMatrix* m = probScal->getSystemMatrix();
+
+ TEST_EXIT(m)("No DOF Matrix!\n");
+
+ m->setIsRankDOF(isRankDOF);
}
Flag ParallelDomainProblemScal::oneIteration(AdaptInfo *adaptInfo, Flag toDo)
--
GitLab