Merge branch 'feature/openmp' of gitlab.math.tu-dresden.de:iwr/amdis into feature/openmp

AMDiS/cmake3/CMakeLists.txt

@@ -72,6 +72,7 @@ add_library(amdis
     ${SOURCE_DIR}/DOFAdmin.cc
     ${SOURCE_DIR}/DOFIndexed.cc
     ${SOURCE_DIR}/DOFMatrix.cc
+    ${SOURCE_DIR}/DOFSerializer.cc
     ${SOURCE_DIR}/DOFVector.cc
     ${SOURCE_DIR}/DirichletBC.cc
     ${SOURCE_DIR}/DualTraverse.cc

AMDiS/cmake3/target_enable_boost.cmake

@@ -13,7 +13,7 @@ macro(target_enable_boost _TARGET_ _SCOPE_)
     endif (BOOST_LIBRARYDIR)
 
     set(BOOST_VERSION "1.48")
-    set(BOOST_LIBS_REQUIRED system iostreams filesystem program_options date_time thread)
+    set(BOOST_LIBS_REQUIRED system iostreams filesystem program_options date_time)
     if (WIN32)
       list(APPEND BOOST_LIBS_REQUIRED zlib)
       if (ENABLE_COMPRESSION OR AMDIS_NEED_COMPRESSION)

AMDiS/src/DOFAdmin.h

@@ -182,6 +182,12 @@ namespace AMDiS {
       return mesh;
     }
 
+    /// Returns \ref mesh
+    inline Mesh* getMesh()
+    {
+      return mesh;
+    }
+
     /// Returns \ref dofFree, the array denoting DOFs to be either free or used.
     inline const std::vector<bool>& getDofFree() const
     {

AMDiS/src/DOFSerializer.cc

+#include "DOFSerializer.h"
+
+#include <algorithm>
+
+#include "DOFVector.h"
+#include "Global.h"
+#include "Traverse.h"
+
+namespace AMDiS
+{    
+    
+// satre values of DOFVector on all macro elements to internal container
+void DOFSerializer::gather(DOFVector<double> const* vec, std::vector<double>& values)
+{
+  std::fill(visited_.begin(), visited_.end(), false);
+  values.clear();
+    
+  for (auto* macroEl : mesh_->getMacroElements())
+    gather(macroEl->getIndex(), vec, values, false);
+}
+
+
+// satre values of DOFVector on macro element `macroIndex` to internal container
+void DOFSerializer::gather(int macroIndex, DOFVector<double> const* vec, std::vector<double>& values, bool reset)
+{
+  if (reset) {
+    std::fill(visited_.begin(), visited_.end(), false);
+    values.clear();
+  }
+  
+  TEST_EXIT(mesh_ == vec->getFeSpace()->getMesh())("Incompatible meshes!\n");
+  
+  TraverseStack stack;
+  ElInfo *elInfo = stack.traverseFirstOneMacro(mesh_, macroIndex, -1, Mesh::CALL_EVERY_EL_PREORDER);
+  while (elInfo) {
+    Element *el = elInfo->getElement();
+    
+    if (el->isLeaf()) {
+      gather(VERTEX, elInfo, vec, values);
+      if (mesh_->getDim() > 1)
+        gather(EDGE, elInfo, vec, values);
+      if (mesh_->getDim() == 3)
+        gather(FACE, elInfo, vec, values);
+      gather(CENTER, elInfo, vec, values);
+    }
+    
+    elInfo = stack.traverseNext(elInfo);
+  }
+}
+
+
+void DOFSerializer::gather(GeoIndex geo, ElInfo* elInfo, DOFVector<double> const* vec, std::vector<double>& values) 
+{
+  int nd;
+  if ((nd = admin_->getNumberOfDofs(geo)))  {
+    int entities = mesh_->getGeo(geo);
+    int nd0 = admin_->getNumberOfPreDofs(geo);
+    int n0 = mesh_->getNode(geo);
+    for (int n = 0; n < entities; n++) {
+      for(int d = 0; d < nd; d++) {
+        DegreeOfFreedom globalDof = elInfo->getElement()->getDof(n0 + n, nd0 + d);
+        TEST_EXIT(globalDof < visited_.size())("visited container not large enough!\n");
+        if (!visited_[globalDof]) {
+          visited_[globalDof] = true;
+          values.push_back((*vec)[globalDof]);
+        }
+      }
+    }
+  }
+}
+
+
+// assign stored values to DOFVector, for all macro elements
+void DOFSerializer::scatter(std::vector<double> const& values, DOFVector<double>* vec) const
+{
+  std::fill(visited_.begin(), visited_.end(), false);
+  counter_ = 0u;
+    
+  for (auto* macroEl : mesh_->getMacroElements())
+    scatter(macroEl->getIndex(), values, vec, false);
+}
+
+
+// assign stored values to DOFVector, on macroElement with index `macroIndex`
+void DOFSerializer::scatter(int macroIndex, std::vector<double> const& values, DOFVector<double>* vec, bool reset) const
+{    
+  if (reset) {
+    std::fill(visited_.begin(), visited_.end(), false);
+    counter_ = 0u;
+  }
+  
+  TEST_EXIT(mesh_ == vec->getFeSpace()->getMesh())("Incompatible meshes!\n");
+  
+  TraverseStack stack;
+  ElInfo *elInfo = stack.traverseFirstOneMacro(mesh_, macroIndex, -1, Mesh::CALL_EVERY_EL_PREORDER);
+  while (elInfo) {
+    Element *el = elInfo->getElement();
+    
+    if (el->isLeaf()) {
+      scatter(VERTEX, elInfo, values, vec);
+      if (mesh_->getDim() > 1)
+        scatter(EDGE, elInfo, values, vec);
+      if (mesh_->getDim() == 3)
+        scatter(FACE, elInfo, values, vec);
+      scatter(CENTER, elInfo, values, vec);
+    }
+    
+    elInfo = stack.traverseNext(elInfo);
+  }
+}
+
+
+void DOFSerializer::scatter(GeoIndex geo, ElInfo* elInfo, std::vector<double> const& values, DOFVector<double>* vec) const
+{
+  int nd;
+  if ((nd = admin_->getNumberOfDofs(geo)))  {
+    int entities = mesh_->getGeo(geo);
+    int nd0 = admin_->getNumberOfPreDofs(geo);
+    int n0 = mesh_->getNode(geo);
+    for (int n = 0; n < entities; n++) {
+      for(int d = 0; d < nd; d++) {
+        DegreeOfFreedom globalDof = elInfo->getElement()->getDof(n0 + n, nd0 + d);
+        TEST_EXIT(globalDof < visited_.size())("visited container not large enough!\n");
+        if (!visited_[globalDof]) {
+          visited_[globalDof] = true;
+          TEST_EXIT(counter_ < values.size())("Not enough values in value-container!\n");
+          (*vec)[globalDof] = values[counter_++];
+        }
+      }
+    }
+  }
+}
+
+
+} // end namespace AMDiS

AMDiS/src/DOFSerializer.h

+/******************************************************************************
+ *
+ * AMDiS - Adaptive multidimensional simulations
+ *
+ * Copyright (C) 2013 Dresden University of Technology. All Rights Reserved.
+ * Web: https://fusionforge.zih.tu-dresden.de/projects/amdis
+ *
+ * Authors:
+ * Simon Vey, Thomas Witkowski, Andreas Naumann, Simon Praetorius, et al.
+ *
+ * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
+ * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ *
+ * This file is part of AMDiS
+ *
+ * See also license.opensource.txt in the distribution.
+ *
+ ******************************************************************************/
+
+
+
+/** \file DOFSerializer.h */
+
+#ifndef AMDIS_DOFSERIALIZER_H
+#define AMDIS_DOFSERIALIZER_H
+
+#include <vector>
+#include <algorithm>
+
+#include "DOFAdmin.h"
+#include "ElInfo.h"
+#include "Global.h"
+#include "Mesh.h"
+#include "Traverse.h"
+
+namespace AMDiS
+{
+  // Collect all DOFValues into a std::vector, by traversing the mesh macro-elenment wise
+  // Can be used to transfer data on macro elements or to store values in a file, e.g. in ARHWriter
+  class DOFSerializer
+  {
+  public:
+    
+    /// Constructor, stores a pointer to the DOFAdmin. If the number of values to store can be estimated,
+    /// pass a second argument `numValues`
+    DOFSerializer(DOFAdmin* admin, std::size_t numValues = 0)
+      : admin_(admin)
+      , mesh_(admin->getMesh())
+      , visited_(admin->getUsedSize(), false)
+    {}
+    
+    
+    // satre values of DOFVector on all macro elements to internal container
+    void gather(DOFVector<double> const* vec, std::vector<double>& values);
+    
+    // satre values of DOFVector on macro element `macroIndex` to internal container
+    void gather(int macroIndex, DOFVector<double> const* vec, std::vector<double>& values, bool reset = true);
+    
+    
+    // assign stored values to DOFVector, for all macro elements
+    void scatter(std::vector<double> const& values, DOFVector<double>* vec) const;
+    
+    // assign stored values to DOFVector, on macroElement with index `macroIndex`
+    void scatter(int macroIndex, std::vector<double> const& values, DOFVector<double>* vec, bool reset = true) const;
+
+      
+  protected:
+    
+    // collect values of one geometry-type
+    void gather(GeoIndex geo, ElInfo* elInfo, DOFVector<double> const* vec, std::vector<double>& values);
+    
+    // assign values of one geometry type
+    void scatter(GeoIndex geo, ElInfo* elInfo, std::vector<double> const& values, DOFVector<double>* vec) const;
+    
+    
+  protected:
+    
+    DOFAdmin* admin_;
+    Mesh* mesh_;
+    
+    // stored which DOFs were already visited during traversal
+    mutable std::vector<bool> visited_;
+    
+    // a counter used during assignment of values. To guarantee the same order as during gather
+    mutable std::size_t counter_ = 0u;
+  };
+
+} // end namespace AMDiS
+
+#endif // AMDIS_DOFSERIALIZER_H

AMDiS/src/DualTraverse.cc

@@ -111,6 +111,93 @@ namespace AMDiS {
   }
 
 
+  bool DualTraverse::traverseFirstOneMacro(Mesh *mesh1,
+				   Mesh *mesh2,
+                                   int macroIndex,
+				   int level1,
+				   int level2,
+				   Flag flag1,
+				   Flag flag2,
+				   ElInfo **elInfo1,
+				   ElInfo **elInfo2,
+				   ElInfo **elInfoSmall,
+				   ElInfo **elInfoLarge)
+  {
+    FUNCNAME("DualTraverse::traverseFirst()");
+    // replace CALL_EL_LEVEL by CALL_MG_LEVEL (covers whole domain)
+    if (flag1.isSet(Mesh::CALL_EL_LEVEL)) {
+      flag1 &= ~Mesh::CALL_EL_LEVEL;
+      flag1 |= Mesh::CALL_MG_LEVEL;
+      level1_ = level1;
+    } else {
+      level1_ = -1;
+    }
+
+    if (flag2.isSet(Mesh::CALL_EL_LEVEL)) {
+      flag2 &= ~Mesh::CALL_EL_LEVEL;
+      flag2 |= Mesh::CALL_MG_LEVEL;
+      level2_ = level2;
+    } else {
+      level2_ = -1;
+    }
+
+    // replace CALL_LEAF_EL_LEVEL by CALL_MG_LEVEL (covers whole domain)
+    if (flag1.isSet(Mesh::CALL_LEAF_EL_LEVEL)) {
+      flag1 &= ~Mesh::CALL_LEAF_EL_LEVEL;
+      flag1 |= Mesh::CALL_MG_LEVEL;
+      level1_ = level1;
+      callLeafElLevel1_ = true;
+    } else {
+      level1_ = -1;
+      callLeafElLevel1_ = false;
+    }
+
+    if (flag2.isSet(Mesh::CALL_LEAF_EL_LEVEL)) {
+      flag2 &= ~Mesh::CALL_LEAF_EL_LEVEL;
+      flag2 |= Mesh::CALL_MG_LEVEL;
+      level2_ = level2;
+      callLeafElLevel2_ = true;
+    } else {
+      level2_ = -1;
+      callLeafElLevel2_ = false;
+    }
+
+    // call standard traverse
+    *elInfo1 = stack1.traverseFirstOneMacro(mesh1, macroIndex, level1, flag1);
+    while (*elInfo1 != NULL && skipEl1(*elInfo1)) {
+      *elInfo1 = stack1.traverseNext(*elInfo1);
+    }
+
+    *elInfo2 = stack2.traverseFirstOneMacro(mesh2, macroIndex, level2, flag2);
+    while (*elInfo2 != NULL && skipEl2(*elInfo2)) {
+      *elInfo2 = stack2.traverseNext(*elInfo2);
+    }
+
+    // finished ?
+    if (*elInfo1 == NULL || *elInfo2 == NULL) {
+      TEST_EXIT(*elInfo1 == *elInfo2)("invalid dual traverse\n");
+      return false;
+    }
+
+    rest = 1.0;
+
+    bool accepted = check(elInfo1, elInfo2, elInfoSmall, elInfoLarge);
+
+    // check for non domain covering level traverse
+    if (!accepted ||
+	(level1_ != -1 && (*elInfo1)->getLevel() != level1_) ||
+	(callLeafElLevel1_ && !(*elInfo1)->getElement()->isLeaf()) ||
+	(level2_ != -1 && (*elInfo2)->getLevel() != level2_) ||
+	(callLeafElLevel2_ && !(*elInfo2)->getElement()->isLeaf())) {
+      return traverseNext(elInfo1, elInfo2, elInfoSmall, elInfoLarge);
+    }
+
+    fillSubElInfo(*elInfo1, *elInfo2, *elInfoSmall, *elInfoLarge);
+
+    return true;
+  }
+
+
   bool DualTraverse::traverseNext(ElInfo **elInfo1,
 				  ElInfo **elInfo2,
 				  ElInfo **elInfoSmall,

AMDiS/src/DualTraverse.h

@@ -78,6 +78,18 @@ namespace AMDiS {
 			   &(dualElInfo.smallElInfo),
 			   &(dualElInfo.largeElInfo));
     }
+    
+    bool traverseFirstOneMacro(Mesh *mesh1,
+                              Mesh *mesh2,
+                              int macroIndex,
+                              int level1,
+                              int level2,
+                              Flag flag1,
+                              Flag flag2,
+                              ElInfo **elInfo1,
+                              ElInfo **elInfo2,
+                              ElInfo **elInfoSmall,
+                              ElInfo **elInfoLarge);
 
     /// Get next ElInfo combination
     bool traverseNext(ElInfo **elInfoNext1,

AMDiS/src/MeshStructure.cc

@@ -504,7 +504,7 @@ namespace AMDiS {
       elInfo = stack.traverseNext(elInfo);
     }
   }
-
+  
 
   void MeshStructure::setMeshStructureValues(int macroElIndex,
 					     DOFVector<double>* vec,
@@ -519,8 +519,8 @@ namespace AMDiS {
     Mesh *mesh = feSpace->getMesh();
     bool feSpaceHasNonVertexDofs = (feSpace->getBasisFcts()->getDegree() > 1);
     int nVertexPreDofs = feSpace->getAdmin()->getNumberOfPreDofs(VERTEX);
-    unsigned int counter = 0;
-
+    
+    std::size_t counter = 0;
     if (withElIndex) {
       TEST_EXIT(static_cast<int>(values[0]) == macroElIndex)
 	("Value structure code was created for macro element index %d, but should be set to macro element index %d\n",
@@ -568,5 +568,4 @@ namespace AMDiS {
     TEST_EXIT_DBG(values.size() == counter)
       ("Should not happen! values size %d, counter %d\n", values.size(), counter);
   }
-
 }

AMDiS/src/Timer.cc

@@ -42,7 +42,7 @@ namespace AMDiS {
 #endif
   }
 
-  double Timer::elapsed()
+  double Timer::elapsed() const
   {
 #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
     return MPI::Wtime() - first_mpi;

AMDiS/src/Timer.h

@@ -42,7 +42,7 @@ namespace AMDiS {
     void reset();
 
     /// returns the elapsed time (from construction or last reset) to now in seconds
-    double elapsed();
+    double elapsed() const;
   };
 }
 

configure.sh

@@ -196,7 +196,7 @@ if [ "${DOWNLOAD_BOOST}" -eq "1" ]; then
   BOOST_PREFIX=${SRC_DIR}/packages/boost/${BOOST_VERSION}
 
   cd /tmp/src/boost
-  ./bootstrap.sh --prefix=${BOOST_PREFIX} --with-libraries=system,iostreams,filesystem,program_options,date_time,test,thread
+  ./bootstrap.sh --prefix=${BOOST_PREFIX} --with-libraries=system,iostreams,filesystem,program_options,date_time,test,thread,mpi,serialization
   if [ "${ENABLE_COMPRESSION}" = "ON" ]; then
     ./b2 -s cxxflags="-std=c++14" --build-type=minimal variant=release install
   else

tools/install_boost.sh

@@ -5,22 +5,23 @@ set -x
 
 ROOT=${PWD}
 
-BOOST_VERSION="1.62.0"
+BOOST_VERSION="1.64.0"
 BOOST_FILENAME="boost_${BOOST_VERSION//[.]/_}.tar.gz"
 
-INSTALL_PREFIX=${ROOT}/install
-mkdir -p ${INSTALL_PREFIX}
+BOOST_PREFIX=${ROOT}/../packages/boost/${BOOST_VERSION}
+mkdir -p ${BOOST_PREFIX}
 
 BUILD_DIR=/tmp/$USER/boost_build
 mkdir -p ${BUILD_DIR}
 
+# download boost
+curl -SL "http://netcologne.dl.sourceforge.net/project/boost/boost/${BOOST_VERSION}/${BOOST_FILENAME}" \
+  | tar --strip-components=1 -xzC ${BUILD_DIR}
+cd ${BUILD_DIR}
+  
 # install boost
-if [ ! -d ${INSTALL_PREFIX}/boost ]; then
-  curl -SL "http://netcologne.dl.sourceforge.net/project/boost/boost/${BOOST_VERSION}/${BOOST_FILENAME}" \
-    | tar --strip-components=1 -xzC ${BUILD_DIR}
-  cd ${BUILD_DIR}
-  ./bootstrap.sh --prefix=${INSTALL_PREFIX}/boost \
-                 --with-libraries=system,iostreams,filesystem,program_options,date_time,test
-  ./b2 -s NO_BZIP2=1 cxxflags="-std=c++11" --build-type=minimal variant=release -j 4 install
-  rm -rf ${BUILD_DIR}
-fi
+./bootstrap.sh --prefix=${BOOST_PREFIX} \
+                --with-libraries=system,iostreams,filesystem,program_options,date_time,test,thread,mpi,serialization
+echo "using mpi ;" >> project-config.jam
+./b2 cxxflags="-std=c++14" --build-type=minimal variant=release install
+rm -rf ${BUILD_DIR}