MeshDistributor.h 23.9 KB
 Thomas Witkowski committed Apr 03, 2009 1 2 3 4 // ============================================================================ // == == // == AMDiS - Adaptive multidimensional simulations == // == ==  Thomas Witkowski committed Dec 16, 2010 5 // == http://www.amdis-fem.org ==  Thomas Witkowski committed Apr 03, 2009 6 7 // == == // ============================================================================  Thomas Witkowski committed Dec 16, 2010 8 9 10 11 12 13 14 15 16 17 18 19 // // Software License for AMDiS // // Copyright (c) 2010 Dresden University of Technology // All rights reserved. // Authors: Simon Vey, Thomas Witkowski et al. // // This file is part of AMDiS // // See also license.opensource.txt in the distribution.  Thomas Witkowski committed Apr 03, 2009 20   21 /** \file MeshDistributor.h */  Thomas Witkowski committed Jul 09, 2009 22   23 24 #ifndef AMDIS_MESHDISTRIBUTOR_H #define AMDIS_MESHDISTRIBUTOR_H  Thomas Witkowski committed Apr 03, 2009 25 26   Thomas Witkowski committed Feb 08, 2010 27 #include  Thomas Witkowski committed Jan 17, 2012 28 #include "parallel/DofComm.h"  Thomas Witkowski committed Feb 13, 2012 29 #include "parallel/ElementObjectDatabase.h"  Thomas Witkowski committed Apr 21, 2011 30 #include "parallel/ParallelTypes.h"  Thomas Witkowski committed Apr 11, 2012 31 #include "parallel/MeshLevelData.h"  Thomas Witkowski committed Jan 24, 2011 32 #include "parallel/MeshPartitioner.h"  Thomas Witkowski committed Nov 29, 2010 33 #include "parallel/InteriorBoundary.h"  Thomas Witkowski committed Jan 26, 2012 34 #include "parallel/PeriodicMap.h"  Thomas Witkowski committed Nov 08, 2011 35 #include "parallel/StdMpi.h"  Thomas Witkowski committed Jan 27, 2011 36 #include "AMDiS_fwd.h"  Thomas Witkowski committed Feb 09, 2012 37 #include "Containers.h"  Thomas Witkowski committed Feb 08, 2010 38 #include "Global.h"  Thomas Witkowski committed Apr 03, 2009 39 40 #include "ProblemTimeInterface.h" #include "ProblemIterationInterface.h"  Thomas Witkowski committed Apr 15, 2009 41 #include "FiniteElemSpace.h"  Thomas Witkowski committed Sep 29, 2009 42 #include "Serializer.h"  Thomas Witkowski committed Feb 08, 2010 43 #include "BoundaryManager.h"  Thomas Witkowski committed Jan 27, 2011 44 #include "SystemVector.h"  Thomas Witkowski committed Jun 12, 2009 45   Thomas Witkowski committed Apr 03, 2009 46 namespace AMDiS {  Thomas Witkowski committed Jan 07, 2011 47 48  using namespace std;  Thomas Witkowski committed Apr 08, 2011 49 50 51 52  struct BoundaryDofInfo {  Thomas Witkowski committed Apr 08, 2011 53  map geoDofs;  Thomas Witkowski committed Apr 08, 2011 54 55  };  56 57 58 59 60 61 62 63 64 65 66 67  struct DofData { /// Number of DOFs in the rank mesh. int nRankDofs; /// Is the index of the first global DOF index, which is owned by the rank. int rStartDofs; /// Number of DOFs in the whole domain. int nOverallDofs;  68 69 70  /// Maps all DOFs in ranks partition to a bool value. If it is true, the DOF /// is owned by the rank. Otherwise, its an interior boundary DOF that is /// owned by another rank.  71 72 73  DofIndexToBool isRankDof; /// Maps local to global dof indices.  Thomas Witkowski committed Mar 15, 2012 74  DofMap mapDofToGlobal;  75 76  /// Maps local dof indices to real dof indices.  Thomas Witkowski committed Mar 15, 2012 77  DofMap mapLocalToDof;  78  };  Thomas Witkowski committed Jan 17, 2012 79 80   Thomas Witkowski committed Jun 08, 2010 81  class MeshDistributor  Thomas Witkowski committed Apr 03, 2009 82  {  Thomas Witkowski committed May 19, 2011 83  private:  Thomas Witkowski committed Mar 18, 2011 84  MeshDistributor();  Thomas Witkowski committed May 19, 2011 85   Thomas Witkowski committed Jun 08, 2010 86  virtual ~MeshDistributor() {}  Thomas Witkowski committed Apr 03, 2009 87   Thomas Witkowski committed May 19, 2011 88  public:  89  void initParallelization();  Thomas Witkowski committed Apr 03, 2009 90   91  void exitParallelization();  Thomas Witkowski committed Jun 08, 2010 92   Thomas Witkowski committed Nov 14, 2011 93 94 95  /// Adds a DOFVector to the set of \ref interchangeVecs. Thus, this vector /// will be automatically interchanged between ranks when mesh is /// repartitioned.  Thomas Witkowski committed Jan 19, 2011 96 97 98 99 100  void addInterchangeVector(DOFVector *vec) { interchangeVectors.push_back(vec); }  Thomas Witkowski committed Jan 27, 2011 101 102 103 104 105 106 107  /// Adds all DOFVectors of a SystemVector to \ref interchangeVecs. void addInterchangeVector(SystemVector *vec) { for (int i = 0; i < vec->getSize(); i++) interchangeVectors.push_back(vec->getDOFVector(i)); }  Thomas Witkowski committed Jan 29, 2010 108  /** \brief  Thomas Witkowski committed Nov 14, 2011 109 110 111 112 113  * This function checks if the mesh has changed on at least on rank. In * this case, the interior boundaries are adapted on all ranks such that * they fit together on all ranks. Furthermore the function * \ref updateLocalGlobalNumbering() is called to update the DOF numberings * and mappings on all rank due to the new mesh structure.  114  *  Thomas Witkowski committed Nov 14, 2011 115 116 117 118 119  * \param[in] tryRepartition If this parameter is true, repartitioning * may be done. This depends on several other * parameters. If the parameter is false, the * mesh is only checked and adapted but never * repartitioned.  Thomas Witkowski committed Jan 29, 2010 120  */  121  void checkMeshChange(bool tryRepartition = true);  Thomas Witkowski committed Jan 29, 2010 122   Thomas Witkowski committed Jan 02, 2012 123 124 125 126 127 128 129 130 131 132  /** \brief * Checks if is required to repartition the mesh. If this is the case, a new * partition will be created and the mesh will be redistributed between the * ranks. */ void repartitionMesh(); /// Calculates the imbalancing factor and prints it to screen. void printImbalanceFactor();  Thomas Witkowski committed Jun 26, 2009 133  /** \brief  Thomas Witkowski committed Nov 14, 2011 134 135 136 137  * Test, if the mesh consists of macro elements only. The mesh partitioning * of the parallelization works for macro meshes only and would fail, if the * mesh is already refined in some way. Therefore, this function will exit * the program if it finds a non macro element in the mesh.  Thomas Witkowski committed Jun 26, 2009 138 139  */ void testForMacroMesh();  Thomas Witkowski committed Apr 03, 2009 140   Thomas Witkowski committed Nov 14, 2011 141 142  /// Set for each element on the partitioning level the number of /// leaf elements.  143  void setInitialElementWeights();  Thomas Witkowski committed Apr 03, 2009 144   Thomas Witkowski committed Jan 07, 2011 145  inline virtual string getName()  Thomas Witkowski committed Jun 08, 2010 146 147 148  { return name; }  149   Thomas Witkowski committed Jan 14, 2011 150 151 152 153 154  inline Mesh* getMesh() { return mesh; }  155 156  /// Returns an FE space from \ref feSpaces. inline const FiniteElemSpace* getFeSpace(unsigned int i = 0)  157  {  158 159 160 161 162  FUNCNAME("MeshDistributor::getFeSpace()"); TEST_EXIT_DBG(i < feSpaces.size())("Should not happen!\n"); return feSpaces[i];  163  }  164 165 166 167 168 169 170  /// Returns all FE spaces, thus \ref feSpaces. inline vector& getFeSpaces() { return feSpaces; }  Thomas Witkowski committed Apr 18, 2012 171 172  /// Returns the number of DOFs in rank's domain for a given FE space. inline int getNumberRankDofs(const FiniteElemSpace *feSpace)  Thomas Witkowski committed Jul 10, 2009 173  {  Thomas Witkowski committed Jan 17, 2012 174 175 176 177  FUNCNAME("MeshDistributor::getNumberRankDofs()"); TEST_EXIT_DBG(dofFeData.count(feSpace))("Should not happen!\n");  178  return dofFeData[feSpace].nRankDofs;  Thomas Witkowski committed Jul 10, 2009 179  }  Thomas Witkowski committed Apr 03, 2009 180   Thomas Witkowski committed Jan 18, 2012 181 182 183 184 185 186 187 188 189 190 191 192 193  /// Returns the number of DOFs in rank's domain for a set of FE spaces. inline int getNumberRankDofs(vector& feSpaces) { FUNCNAME("MeshDistributor::getNumberRankDofs()"); int result = 0; for (unsigned int i = 0; i < feSpaces.size(); i++) { TEST_EXIT_DBG(dofFeData.count(feSpaces[i]))("Should not happen!\n"); result += dofFeData[feSpaces[i]].nRankDofs; } return result; }  Thomas Witkowski committed Feb 06, 2012 194   Thomas Witkowski committed Jan 18, 2012 195  /// Returns the first global DOF index of an FE space, owned by rank.  196  inline int getStartDofs(const FiniteElemSpace *feSpace)  Thomas Witkowski committed Jan 03, 2012 197  {  Thomas Witkowski committed Jan 17, 2012 198 199 200 201  FUNCNAME("MeshDistributor::getStartDofs()"); TEST_EXIT_DBG(dofFeData.count(feSpace))("Should not happen!\n");  202  return dofFeData[feSpace].rStartDofs;  Thomas Witkowski committed Jan 03, 2012 203 204  }  Thomas Witkowski committed Jan 18, 2012 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219  /// Returns the first global DOF index for a set of FE spaces, owned by rank. inline int getStartDofs(vector& feSpaces) { FUNCNAME("MeshDistributor::getStartDofs()"); int result = 0; for (unsigned int i = 0; i < feSpaces.size(); i++) { TEST_EXIT_DBG(dofFeData.count(feSpaces[i]))("Should not happen!\n"); result += dofFeData[feSpaces[i]].rStartDofs; } return result; }  Thomas Witkowski committed Apr 18, 2012 220 221  /// Returns the global number of DOFs for a given FE space. inline int getNumberOverallDofs(const FiniteElemSpace *feSpace)  Thomas Witkowski committed Jul 10, 2009 222  {  Thomas Witkowski committed Jan 17, 2012 223 224 225 226  FUNCNAME("MeshDistributor::getNumberOverallDofs()"); TEST_EXIT_DBG(dofFeData.count(feSpace))("Should not happen!\n");  227  return dofFeData[feSpace].nOverallDofs;  Thomas Witkowski committed Jul 10, 2009 228  }  Thomas Witkowski committed Apr 03, 2009 229   Thomas Witkowski committed Jan 18, 2012 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244  /// Returns the global number of DOFs for a set of FE spaces. inline int getNumberOverallDofs(vector& feSpaces) { FUNCNAME("MeshDistributor::getNumberOverallDofs()"); int result = 0; for (unsigned int i = 0; i < feSpaces.size(); i++) { TEST_EXIT_DBG(dofFeData.count(feSpaces[i]))("Should not happen!\n"); result += dofFeData[feSpaces[i]].nOverallDofs; } return result; }  Thomas Witkowski committed Mar 15, 2012 245  inline DofMap& getMapDofToGlobal(const FiniteElemSpace *feSpace)  Thomas Witkowski committed Jan 14, 2011 246  {  247  FUNCNAME("MeshDistributor::getMapDofToGlobal()");  Thomas Witkowski committed Jan 17, 2012 248 249 250  TEST_EXIT_DBG(dofFeData.count(feSpace))("Should not happen!\n");  251  return dofFeData[feSpace].mapDofToGlobal;  Thomas Witkowski committed Jan 14, 2011 252 253  }  Thomas Witkowski committed Mar 18, 2011 254  /// Maps a local DOF to its global index.  255 256  inline DegreeOfFreedom mapDofToGlobal(const FiniteElemSpace *feSpace, DegreeOfFreedom dof)  Thomas Witkowski committed Jul 10, 2009 257  {  258  FUNCNAME("MeshDistributor::mapDofToGlobal()");  Thomas Witkowski committed Jan 17, 2012 259 260 261 262  TEST_EXIT_DBG(dofFeData.count(feSpace)) ("No DOF data for FE space at addr %p!\n", feSpace);  263  return dofFeData[feSpace].mapDofToGlobal[dof];  Thomas Witkowski committed Jul 10, 2009 264  }  Thomas Witkowski committed Apr 03, 2009 265   266 267 268 269  /// Returns for a global index the DOF index in rank's subdomain. As there /// is no direct data structure that stores this information, we have to /// search for it in \ref dofFeData.mapDofToGlobal. This is not very /// efficient and this function should thus be used for debugging only.  270 271  DegreeOfFreedom mapGlobalToLocal(const FiniteElemSpace *feSpace, DegreeOfFreedom dof);  Thomas Witkowski committed Jan 17, 2011 272   Thomas Witkowski committed Mar 18, 2011 273  /// Maps a local DOF to its local index.  274 275  inline DegreeOfFreedom mapLocalToDof(const FiniteElemSpace *feSpace, DegreeOfFreedom dof)  Thomas Witkowski committed Jun 09, 2010 276  {  277  FUNCNAME("MeshDistributor::mapLocalToDof()");  Thomas Witkowski committed Jan 17, 2012 278 279 280 281  TEST_EXIT_DBG(dofFeData.count(feSpace)) ("No DOF data for FE space at addr %p!\n", feSpace);  282  return dofFeData[feSpace].mapLocalToDof[dof];  Thomas Witkowski committed Jun 09, 2010 283 284  }  Thomas Witkowski committed Jan 26, 2012 285 286  /// Returns the periodic mapping handler, \ref periodicMap. inline PeriodicMap& getPeriodicMap()  Thomas Witkowski committed Jan 14, 2011 287  {  Thomas Witkowski committed Jan 26, 2012 288  return periodicMap;  Thomas Witkowski committed Apr 07, 2009 289 290  }  Thomas Witkowski committed Jan 17, 2012 291  DofComm& getSendDofs()  292 293 294 295  { return sendDofs; }  Thomas Witkowski committed Jan 17, 2012 296  DofComm& getRecvDofs()  297 298 299 300  { return recvDofs; }  301 302 303 304 305  DofComm& getPeriodicDofs() { return periodicDofs; }  Thomas Witkowski committed Nov 14, 2011 306 307  /// Return true, if the given DOF is owned by the rank. If false, the DOF /// is in rank's partition, but is owned by some other rank.  308  inline bool getIsRankDof(const FiniteElemSpace *feSpace, DegreeOfFreedom dof)  Thomas Witkowski committed Jun 08, 2010 309  {  310 311  if (dofFeData[feSpace].isRankDof.count(dof)) return dofFeData[feSpace].isRankDof[dof];  Thomas Witkowski committed Jan 18, 2011 312 313  return false;  Thomas Witkowski committed Jun 08, 2010 314  }  Thomas Witkowski committed Dec 08, 2009 315   316  inline DofIndexToBool& getIsRankDof(const FiniteElemSpace *feSpace)  Thomas Witkowski committed Mar 17, 2011 317  {  318  return dofFeData[feSpace].isRankDof;  Thomas Witkowski committed Mar 17, 2011 319 320  }  Thomas Witkowski committed Jun 08, 2010 321  inline long getLastMeshChangeIndex()  Thomas Witkowski committed Jun 10, 2009 322  {  Thomas Witkowski committed Jun 08, 2010 323  return lastMeshChangeIndex;  Thomas Witkowski committed Apr 07, 2009 324  }  Thomas Witkowski committed Apr 03, 2009 325   Thomas Witkowski committed Jun 08, 2010 326  inline int getMpiRank()  Thomas Witkowski committed Jun 11, 2009 327  {  Thomas Witkowski committed Jun 08, 2010 328  return mpiRank;  Thomas Witkowski committed Jun 11, 2009 329  }  Thomas Witkowski committed Apr 03, 2009 330   Thomas Witkowski committed Jan 14, 2011 331 332 333 334 335  inline int getMpiSize() { return mpiSize; }  Thomas Witkowski committed Jun 08, 2010 336 337 338  inline MPI::Intracomm& getMpiComm() { return mpiComm;  Thomas Witkowski committed Apr 03, 2009 339 340  }  341 342 343 344 345  inline bool isInitialized() { return initialized; }  Thomas Witkowski committed Sep 28, 2009 346  // Writes all data of this object to an output stream.  Thomas Witkowski committed Jan 07, 2011 347  void serialize(ostream &out);  Thomas Witkowski committed Jun 11, 2009 348   Thomas Witkowski committed Sep 28, 2009 349  // Reads the object data from an input stream.  Thomas Witkowski committed Jan 07, 2011 350  void deserialize(istream &in);  Thomas Witkowski committed Jun 08, 2010 351 352  /** \brief  Thomas Witkowski committed Nov 14, 2011 353 354 355 356  * This function must be used if the values of a DOFVector must be * synchronised over all ranks. That means, that each rank sends the * values of the DOFs, which are owned by the rank and lie on an interior * bounday, to all other ranks also having these DOFs.  Thomas Witkowski committed Jun 08, 2010 357  *  Thomas Witkowski committed Nov 14, 2011 358 359 360  * This function must be used, for example, after the lineary system is * solved, or after the DOFVector is set by some user defined functions, * e.g., initial solution functions.  Thomas Witkowski committed Jun 08, 2010 361  */  Thomas Witkowski committed Nov 08, 2011 362 363 364 365 366  template void synchVector(DOFVector &vec) { StdMpi > stdMpi(mpiComm);  367 368  const FiniteElemSpace *fe = vec.getFeSpace();  Thomas Witkowski committed Jan 17, 2012 369  for (DofComm::Iterator it(sendDofs, fe); !it.end(); it.nextRank()) {  Thomas Witkowski committed Nov 08, 2011 370  vector dofs;  Thomas Witkowski committed Jan 17, 2012 371  dofs.reserve(it.getDofs().size());  Thomas Witkowski committed Nov 08, 2011 372   Thomas Witkowski committed Jan 17, 2012 373 374  for (; !it.endDofIter(); it.nextDof()) dofs.push_back(vec[it.getDofIndex()]);  Thomas Witkowski committed Nov 08, 2011 375   Thomas Witkowski committed Jan 17, 2012 376  stdMpi.send(it.getRank(), dofs);  Thomas Witkowski committed Nov 08, 2011 377  }  Thomas Witkowski committed Jan 17, 2012 378 379 380 381  for (DofComm::Iterator it(recvDofs); !it.end(); it.nextRank()) stdMpi.recv(it.getRank());  Thomas Witkowski committed Nov 08, 2011 382  stdMpi.startCommunication();  Thomas Witkowski committed Jan 17, 2012 383 384 385 386 387  for (DofComm::Iterator it(recvDofs, fe); !it.end(); it.nextRank()) for (; !it.endDofIter(); it.nextDof()) vec[it.getDofIndex()] = stdMpi.getRecvData(it.getRank())[it.getDofCounter()];  Thomas Witkowski committed Nov 08, 2011 388 389  }  Thomas Witkowski committed Jun 08, 2010 390  /** \brief  Thomas Witkowski committed Nov 14, 2011 391 392 393  * Works in the same way as the function above defined for DOFVectors. Due * to performance, this function does not call \ref synchVector for each * DOFVector, but instead sends all values of all DOFVectors all at once.  Thomas Witkowski committed Jun 08, 2010 394 395 396  */ void synchVector(SystemVector &vec);  Thomas Witkowski committed Jan 20, 2011 397 398  void check3dValidMesh();  Thomas Witkowski committed Apr 08, 2011 399 400 401 402 403  void setBoundaryDofRequirement(Flag flag) { createBoundaryDofFlag = flag; }  404 405  BoundaryDofInfo& getBoundaryDofInfo(const FiniteElemSpace *feSpace, int level = 0)  Thomas Witkowski committed Apr 08, 2011 406  {  Thomas Witkowski committed Apr 16, 2012 407 408 409 410 411 412 413  FUNCNAME("MeshDistributor::getBoundaryDofInfo()"); TEST_EXIT_DBG(level < static_cast(boundaryDofInfo.size())) ("Wrong level number: %d, whereas array size is %d!\n", level, boundaryDofInfo.size()); return boundaryDofInfo[level][feSpace];  Thomas Witkowski committed Apr 08, 2011 414 415  }  416 417  void getAllBoundaryDofs(const FiniteElemSpace *feSpace, int level,  418  DofContainer& dofs);  Thomas Witkowski committed Apr 21, 2011 419   420 421 422 423  const ElementObjectDatabase& getElementObjectDb() { return elObjDb; }  Thomas Witkowski committed May 19, 2011 424 425 426  /// Adds a stationary problem to the global mesh distributor objects. static void addProblemStatGlobal(ProblemStatSeq *probStat);  Thomas Witkowski committed Apr 13, 2012 427 428 429 430 431 432  MeshLevelData& getMeshLevelData() { return levelData; }  Thomas Witkowski committed Jun 11, 2009 433  protected:  Thomas Witkowski committed May 19, 2011 434 435  void addProblemStat(ProblemStatSeq *probStat);  Thomas Witkowski committed Nov 14, 2011 436 437  /// Determines the interior boundaries, i.e. boundaries between ranks, and /// stores all information about them in \ref interiorBoundary.  Thomas Witkowski committed Oct 19, 2009 438  void createInteriorBoundaryInfo();  Thomas Witkowski committed Jun 11, 2009 439   Thomas Witkowski committed Nov 29, 2010 440 441 442 443 444  void updateInteriorBoundaryInfo(); void createMeshElementData(); void createBoundaryData();  Thomas Witkowski committed Jan 26, 2010 445   Thomas Witkowski committed Apr 08, 2011 446 447  void createBoundaryDofs();  Thomas Witkowski committed Apr 16, 2012 448  void createBoundaryDofs(const FiniteElemSpace *feSpace, int level);  449   Thomas Witkowski committed Nov 14, 2011 450 451  /// Removes all macro elements from the mesh that are not part of ranks /// partition.  Thomas Witkowski committed Jun 11, 2009 452 453  void removeMacroElements();  454 455  void updateLocalGlobalNumbering();  Thomas Witkowski committed Nov 14, 2011 456 457  /// Updates the local and global DOF numbering after the mesh has been /// changed.  458  void updateLocalGlobalNumbering(const FiniteElemSpace *feSpace);  Thomas Witkowski committed Jun 11, 2009 459   460 461 462 463  /// Calls \ref createPeriodicMap(feSpace) for all FE spaces that are /// handled by the mesh distributor. void createPeriodicMap();  464  /** \brief  465 466 467 468  * Creates, for a specific FE space, to all DOFs in rank's partition that * are on a periodic boundary the mapping from dof index to the other * periodic dof indices. This information is stored in \ref periodicDofMap. */  469  void createPeriodicMap(const FiniteElemSpace *feSpace);  Thomas Witkowski committed Oct 14, 2009 470   Thomas Witkowski committed Nov 14, 2011 471 472 473 474 475 476 477 478  /** \brief * This function is called only once during the initialization when the * whole macro mesh is available on all cores. It copies the pointers of all * macro elements to \ref allMacroElements and stores all neighbour * information based on macro element indices (and not pointer based) in * \ref macroElementNeighbours. These information are then used to * reconstruct macro elements during mesh redistribution. */  Thomas Witkowski committed Nov 25, 2010 479 480  void createMacroElementInfo();  481 482  void updateMacroElementInfo();  Thomas Witkowski committed Jan 19, 2010 483  /** \brief  Thomas Witkowski committed Nov 14, 2011 484 485 486 487 488 489  * Checks for all given interior boundaries if the elements fit together on * both sides of the boundaries. If this is not the case, the mesh is * adapted. Because refinement of a certain element may forces the * refinement of other elements, it is not guaranteed that all rank's meshes * fit together after this function terminates. Hence, it must be called * until a stable mesh refinement is reached.  Thomas Witkowski committed Jan 19, 2010 490  *  Thomas Witkowski committed Nov 14, 2011 491 492  * \param[in] allBound Defines a map from rank to interior boundaries * which should be checked.  493  *  Thomas Witkowski committed Nov 14, 2011 494 495 496  * \return If the mesh has been changed by this function, it returns * true. Otherwise, it returns false, i.e., the given interior * boundaries fit together on both sides.  Thomas Witkowski committed Jan 19, 2010 497 498  */ bool checkAndAdaptBoundary(RankToBoundMap &allBound);  Thomas Witkowski committed Nov 26, 2010 499   Thomas Witkowski committed Mar 28, 2011 500 501  /// Sets \ref isRankDof to all matrices and rhs vectors in a given /// stationary problem.  Thomas Witkowski committed Apr 21, 2011 502  void setRankDofs(ProblemStatSeq *probStat);  Thomas Witkowski committed Mar 28, 2011 503   Thomas Witkowski committed Nov 14, 2011 504 505  /// Sets \ref isRankDof to all matrices and rhs vectors in all /// stationary problems.  Thomas Witkowski committed Jul 18, 2010 506 507  void setRankDofs();  Thomas Witkowski committed Jun 29, 2010 508 509 510 511  /// Removes all periodic boundary condition information from all matrices and /// vectors of all stationary problems and from the mesh itself. void removePeriodicBoundaryConditions();  Thomas Witkowski committed Feb 28, 2012 512 513 514 515  /// Removes all periodic boundary condition information from all matrices and /// vector of a given stationary problem. void removePeriodicBoundaryConditions(ProblemStatSeq *probStat);  Thomas Witkowski committed Feb 08, 2010 516  // Removes all periodic boundaries from a given boundary map.  Thomas Witkowski committed Jun 29, 2010 517  void removePeriodicBoundaryConditions(BoundaryIndexMap& boundaryMap);  Thomas Witkowski committed Feb 08, 2010 518   Thomas Witkowski committed Apr 11, 2012 519 520  void createMeshLevelStructure();  Thomas Witkowski committed Sep 28, 2009 521  /// Writes a vector of dof pointers to an output stream.  Thomas Witkowski committed Jan 07, 2011 522  void serialize(ostream &out, DofContainer &data);  Thomas Witkowski committed Sep 28, 2009 523   524 525 526 527  /// Writes a \ref RankToDofContainer to an output stream. void serialize(ostream &out, map > &data);  Thomas Witkowski committed Sep 28, 2009 528  /// Reads a vector of dof pointers from an input stream.  Thomas Witkowski committed Jan 07, 2011 529 530  void deserialize(istream &in, DofContainer &data, map &dofMap);  Thomas Witkowski committed Sep 28, 2009 531 532  /// Reads a \ref RankToDofContainer from an input stream.  533 534 535  void deserialize(istream &in, map > &data, map > &dofMap);  Thomas Witkowski committed Sep 29, 2009 536 537 538  /// Writes a mapping from dof pointers to some values to an output stream. template  Thomas Witkowski committed Jan 07, 2011 539  void serialize(ostream &out, map &data)  Thomas Witkowski committed Sep 29, 2009 540  {  541 542  FUNCNAME("ParallelDomainBase::serialize()");  Thomas Witkowski committed Sep 29, 2009 543  int mapSize = data.size();  544  SerUtil::serialize(out, mapSize);  Thomas Witkowski committed Jan 07, 2011 545  for (typename map::iterator it = data.begin();  Thomas Witkowski committed Sep 29, 2009 546 547 548  it != data.end(); ++it) { int v1 = (*(it->first)); T v2 = it->second;  549 550  SerUtil::serialize(out, v1); SerUtil::serialize(out, v2);  Thomas Witkowski committed Sep 29, 2009 551 552 553 554 555  } } /// Reads a mapping from dof pointer to some values from an input stream. template  Thomas Witkowski committed Jan 07, 2011 556 557  void deserialize(istream &in, map &data, map &dofMap)  Thomas Witkowski committed Sep 29, 2009 558  {  Thomas Witkowski committed Oct 02, 2009 559 560  FUNCNAME("ParallelDomainBase::deserialize()");  Thomas Witkowski committed Sep 29, 2009 561  int mapSize = 0;  562  SerUtil::deserialize(in, mapSize);  Thomas Witkowski committed Sep 29, 2009 563 564 565  for (int i = 0; i < mapSize; i++) { int v1 = 0; T v2;  566 567  SerUtil::deserialize(in, v1); SerUtil::deserialize(in, v2);  Thomas Witkowski committed Oct 02, 2009 568 569 570  TEST_EXIT_DBG(dofMap.count(v1) != 0)("Cannot find DOF %d in map!\n", v1);  Thomas Witkowski committed Sep 29, 2009 571 572 573  data[dofMap[v1]] = v2; } }  Thomas Witkowski committed Nov 30, 2010 574   Thomas Witkowski committed Apr 03, 2009 575  protected:  Thomas Witkowski committed Nov 14, 2011 576 577  /// List of all stationary problems that are managed by this mesh /// distributor.  Thomas Witkowski committed Apr 21, 2011 578  vector problemStat;  Thomas Witkowski committed Feb 08, 2010 579   Thomas Witkowski committed Mar 28, 2011 580 581 582  /// If true, the mesh distributor is already initialized; bool initialized;  Thomas Witkowski committed Apr 03, 2009 583 584 585 586 587 588  /// The rank of the current process. int mpiRank; /// Overall number of processes. int mpiSize;  Thomas Witkowski committed Nov 14, 2011 589 590  /// MPI communicator collected all processes, which should be used for /// calculation. The Debug procces is not included in this communicator.  Thomas Witkowski committed Apr 03, 2009 591 592 593  MPI::Intracomm mpiComm; /// Name of the problem (as used in the init files)  Thomas Witkowski committed Jan 07, 2011 594  string name;  Thomas Witkowski committed Apr 03, 2009 595   596 597  /// Finite element spaces of the problem. vector feSpaces;  Thomas Witkowski committed Apr 15, 2009 598   Thomas Witkowski committed Apr 03, 2009 599 600 601  /// Mesh of the problem. Mesh *mesh;  Thomas Witkowski committed Feb 13, 2012 602 603 604  /// A refinement manager that should be used on the mesh. It is used to /// refine elements at interior boundaries in order to fit together with /// elements on the other side of the interior boundary.  605 606  RefinementManager *refineManager;  Thomas Witkowski committed Jun 26, 2009 607 608 609  /// Info level. int info;  Thomas Witkowski committed Nov 14, 2011 610 611  /// Pointer to a mesh partitioner that is used to partition the mesh to /// the ranks.  Thomas Witkowski committed Jan 24, 2011 612  MeshPartitioner *partitioner;  Thomas Witkowski committed Apr 03, 2009 613   Thomas Witkowski committed Nov 14, 2011 614 615  /// Weights for the elements, i.e., the number of leaf elements within /// this element.  Thomas Witkowski committed Jan 07, 2011 616  map elemWeights;  Thomas Witkowski committed Apr 03, 2009 617   Thomas Witkowski committed Feb 13, 2012 618 619  /// Stores to every macro element index the number of the rank that owns this /// macro element.  Thomas Witkowski committed Jan 24, 2011 620  map partitionMap;  Thomas Witkowski committed Apr 03, 2009 621   622  map dofFeData;  Thomas Witkowski committed Jun 08, 2010 623   Thomas Witkowski committed Feb 13, 2012 624  /// Database to store and query all sub-objects of all elements of the  Thomas Witkowski committed Nov 14, 2011 625  /// macro mesh.  Thomas Witkowski committed Feb 13, 2012 626  ElementObjectDatabase elObjDb;  Thomas Witkowski committed Nov 29, 2010 627   Thomas Witkowski committed Nov 14, 2011 628  /// Maps to each macro element index a pointer to the corresponding element.  Thomas Witkowski committed Jan 07, 2011 629  map macroElIndexMap;  Thomas Witkowski committed Nov 29, 2010 630   Thomas Witkowski committed Nov 14, 2011 631  /// Maps to each macro element index the type of this element.  Thomas Witkowski committed Jan 07, 2011 632  map macroElIndexTypeMap;  Thomas Witkowski committed Nov 29, 2010 633   Thomas Witkowski committed Feb 13, 2012 634 635 636 637  /// Defines the interior boundaries of the domain that result from /// partitioning the whole mesh. Contains only the boundaries, which are /// owned by the rank, i.e., the object gives for every neighbour rank i /// the boundaries this rank owns and shares with rank i.  Thomas Witkowski committed Feb 01, 2012 638  InteriorBoundary rankIntBoundary;  Thomas Witkowski committed Jun 15, 2009 639   Thomas Witkowski committed Feb 13, 2012 640 641 642 643  /// Defines the interior boundaries of the domain that result from /// partitioning the whole mesh. Contains only the boundaries, which are /// not owned by the rank, i.e., the object gives for every neighbour rank /// i the boundaries that are owned by rank i and are shared with this rank.  Thomas Witkowski committed Jun 15, 2009 644  InteriorBoundary otherIntBoundary;  Thomas Witkowski committed Apr 27, 2009 645   Thomas Witkowski committed Feb 13, 2012 646 647  /// Defines the periodic boundaries with other ranks. Periodic boundaries /// have no owner, as it is the case of interior boundaries.  Thomas Witkowski committed Oct 14, 2009 648 649  InteriorBoundary periodicBoundary;  Thomas Witkowski committed Feb 13, 2012 650 651  /// This map contains for each rank the list of DOFs the current rank must /// end to exchange solution DOFs at the interior boundaries.  Thomas Witkowski committed Jan 17, 2012 652  DofComm sendDofs;  Thomas Witkowski committed Jun 10, 2009 653   Thomas Witkowski committed Feb 13, 2012 654 655 656  /// This map contains on each rank the list of DOFs from which the current /// rank will receive DOF values (i.e., this are all DOFs at an interior /// boundary). The DOF indices are given in rank's local numbering.  Thomas Witkowski committed Jan 17, 2012 657  DofComm recvDofs;  Thomas Witkowski committed Aug 13, 2009 658   Thomas Witkowski committed Feb 13, 2012 659 660 661 662  /// This map contains on each rank a list of DOFs along the interior bound- /// aries to communicate with other ranks. The DOF indices are given in rank's /// local numbering. Periodic boundaries within one subdomain are not /// considered here.  663 664  DofComm periodicDofs;  Thomas Witkowski committed Jan 26, 2012 665  PeriodicMap periodicMap;  Thomas Witkowski committed Jul 08, 2010 666   Thomas Witkowski committed Jan 19, 2011 667 668 669 670  /// This set of values must be interchanged between ranks when the mesh is /// repartitioned. vector*> interchangeVectors;  Thomas Witkowski committed Feb 13, 2012 671 672 673 674  /// If the problem definition has been read from a serialization file, this /// variable is true, otherwise it is false. This variable is used to stop the /// initialization function, if the problem definition has already been read /// from a serialization file.  675  bool deserialized;  Thomas Witkowski committed Dec 08, 2009 676   677 678 679  /// Denotes whether there exists a filewriter for this object. bool writeSerializationFile;  Thomas Witkowski committed Nov 26, 2010 680 681 682  /// If true, it is possible to repartition the mesh during computations. bool repartitioningAllowed;  Thomas Witkowski committed Nov 14, 2011 683 684  /// Stores the number of mesh changes that must lie in between to /// repartitionings.  Thomas Witkowski committed Dec 20, 2010 685 686  int repartitionIthChange;  Thomas Witkowski committed Nov 14, 2011 687 688  /// Counts the number of mesh changes after the last mesh repartitioning /// was done.  689  int nMeshChangesAfterLastRepartitioning;  Thomas Witkowski committed Dec 03, 2010 690   691 692 693  /// Countes the number of mesh repartitions that were done. Till now, this /// variable is used only for debug outputs. int repartitioningCounter;  Thomas Witkowski committed Dec 03, 2010 694   Thomas Witkowski committed Dec 16, 2010 695  /// Directory name where all debug output files should be written to.  Thomas Witkowski committed Jan 07, 2011 696  string debugOutputDir;  Thomas Witkowski committed Dec 16, 2010 697   Thomas Witkowski committed Feb 13, 2012 698 699  /// Stores the mesh change index. This is used to recognize changes in the /// mesh structure (e.g. through refinement or coarsening managers).  Thomas Witkowski committed Dec 08, 2009 700  long lastMeshChangeIndex;  701   Thomas Witkowski committed Nov 14, 2011 702 703 704 705  /// Stores for all macro elements of the original macro mesh the /// neighbourhood information based on element indices. Thus, each macro /// element index is mapped to a vector containing all indices of /// neighbouring macro elements.  Thomas Witkowski committed Jan 07, 2011 706  map > macroElementNeighbours;  Thomas Witkowski committed Nov 25, 2010 707   Thomas Witkowski committed Nov 14, 2011 708 709  /// Store all macro elements of the overall mesh, i.e., before the /// mesh is redistributed for the first time.  Thomas Witkowski committed Jan 07, 2011 710  vector allMacroElements;  711   Thomas Witkowski committed Apr 08, 2011 712 713  Flag createBoundaryDofFlag;  Thomas Witkowski committed Apr 16, 2012 714 715 716  /// Stores on each mesh level for all FE spaces the information about /// all boundary DOFs. vector > boundaryDofInfo;  Thomas Witkowski committed May 19, 2011 717   Thomas Witkowski committed Apr 11, 2012 718 719  MeshLevelData levelData;  Thomas Witkowski committed Apr 08, 2011 720  public:  Thomas Witkowski committed Jan 31, 2012 721 722  bool sebastianMode;  Thomas Witkowski committed Apr 21, 2011 723 724 725  /// The boundary DOFs are sorted by subobject entities, i.e., first all /// face DOFs, edge DOFs and to the last vertex DOFs will be set to /// communication structure vectors, \ref sendDofs and \ref recvDofs.  Thomas Witkowski committed Apr 08, 2011 726 727  static const Flag BOUNDARY_SUBOBJ_SORTED;  Thomas Witkowski committed Apr 21, 2011 728 729 730 731 732 733 734 735 736  /// When boundary DOFs are created, \ref boundaryDofInfo is filled for /// all DOFs that this rank will send to other ranks (thus, rank /// owned DOFs. static const Flag BOUNDARY_FILL_INFO_SEND_DOFS; /// When boundary DOFs are created, \ref boundaryDofInfo is filled for /// all DOFs that this rank will receive from other ranks (thus, DOFs /// that are owned by another rank). static const Flag BOUNDARY_FILL_INFO_RECV_DOFS;  Thomas Witkowski committed Apr 08, 2011 737   Thomas Witkowski committed May 19, 2011 738 739  static MeshDistributor *globalMeshDistributor;  740  friend class ParallelDebug;  Thomas Witkowski committed Apr 03, 2009 741 742 743  }; }  744 #endif // AMDIS_MESHDISTRIBUTOR_H