From 96d0b4bc501693fdd21bcf6904c8667c7eb253ff Mon Sep 17 00:00:00 2001
From: Siqi Ling <lsq0473@gmail.com>
Date: Tue, 20 Oct 2015 11:39:09 +0000
Subject: [PATCH] change in arh, parh file. update with meshconv

---
 AMDiS/src/io/Arh3Writer.h         |  36 ++--
 AMDiS/src/io/FileWriter.cc        |  20 ++-
 AMDiS/src/io/FileWriter.h         |   9 +
 AMDiS/src/io/FileWriter.hh        |   3 +
 AMDiS/src/io/detail/Arh3Reader.cc | 124 +++++++++-----
 AMDiS/src/io/detail/Arh3Reader.h  |   5 +-
 AMDiS/src/io/detail/Arh3Writer.cc | 274 +++++++++++++++++++-----------
 AMDiS/src/io/detail/Arh3Writer.h  |  20 ++-
 8 files changed, 330 insertions(+), 161 deletions(-)

diff --git a/AMDiS/src/io/Arh3Writer.h b/AMDiS/src/io/Arh3Writer.h
index fffe9b77..4312209b 100644
--- a/AMDiS/src/io/Arh3Writer.h
+++ b/AMDiS/src/io/Arh3Writer.h
@@ -5,6 +5,12 @@
 #include "SystemVector.h"
 #include "detail/Arh3Writer.h"
 
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+#define WRITE_MACRO false
+#else
+#define WRITE_MACRO true
+#endif
+
 namespace AMDiS { namespace io {
   
   /** \ingroup Output
@@ -45,21 +51,23 @@ namespace AMDiS { namespace io {
 			  std::string filename,
 			  bool writeParallel = true,
 			  Cpsformat cps = NONE,
-			  std::string dataformat = "SF64")
+			  std::string dataformat = "SF64",
+			  bool writeMacro = WRITE_MACRO)
     {
       std::vector<DOFVector<double>*> vecs;
       for (int i = 0; i < sysVec->getSize(); i++)
 	vecs.push_back(sysVec->getDOFVector(i));
-      detail::write(filename, NULL, vecs, writeParallel, cps, dataformat);
+      detail::write(filename, NULL, vecs, writeParallel, cps, dataformat, writeMacro);
     }
     
     inline void writeFile(SystemVector &sysVec,
 			  std::string filename,
 			  bool writeParallel = true,
 			  Cpsformat cps = NONE,
-			  std::string dataformat = "SF64")
+			  std::string dataformat = "SF64",
+			  bool writeMacro = WRITE_MACRO)
     {
-      writeFile(&sysVec, filename, writeParallel, cps, dataformat);
+      writeFile(&sysVec, filename, writeParallel, cps, dataformat, writeMacro);
     }
     
     /// write the meshstructure and the dof values of DOFVectors in vec0
@@ -68,11 +76,12 @@ namespace AMDiS { namespace io {
 			  std::string filename,
 			  bool writeParallel = true,
 			  Cpsformat cps = NONE,
-			  std::string dataformat = "SF64")
+			  std::string dataformat = "SF64",
+			  bool writeMacro = WRITE_MACRO)
     {
         std::vector<DOFVector<double>*> vecs;
         vecs.push_back(vec0);
-	detail::write(filename, NULL, vecs, writeParallel, cps, dataformat);
+	detail::write(filename, NULL, vecs, writeParallel, cps, dataformat, writeMacro);
     }
     
     /// write the meshstructure and the dof values of DOFVectors in vec0
@@ -81,9 +90,10 @@ namespace AMDiS { namespace io {
 			  std::string filename,
 			  bool writeParallel = true,
 			  Cpsformat cps = NONE,
-			  std::string dataformat = "SF64")
+			  std::string dataformat = "SF64",
+			  bool writeMacro = WRITE_MACRO)
     {
-        writeFile(&vec0, filename, writeParallel, cps, dataformat);
+        writeFile(&vec0, filename, writeParallel, cps, dataformat, writeMacro);
     }
 
     /// write the meshstructure and the dof values of DOFVectors in vecs
@@ -92,9 +102,10 @@ namespace AMDiS { namespace io {
 			  std::string filename,
 			  bool writeParallel = true,
 			  Cpsformat cps = NONE,
-			  std::string dataformat = "SF64")
+			  std::string dataformat = "SF64",
+			  bool writeMacro = WRITE_MACRO)
     {
-      detail::write(filename, NULL, vecs, writeParallel, cps, dataformat);
+      detail::write(filename, NULL, vecs, writeParallel, cps, dataformat, writeMacro);
     }
     
     /// write the meshstructure of the mesh to arh file.
@@ -102,10 +113,11 @@ namespace AMDiS { namespace io {
 			  std::string filename,
 			  bool writeParallel = true,
 			  Cpsformat cps = NONE,
-			  std::string dataformat = "SF64")
+			  std::string dataformat = "SF64",
+			  bool writeMacro = WRITE_MACRO)
     {
       std::vector<DOFVector<double>*> vecs;
-      detail::write(filename, mesh, vecs, writeParallel, cps, dataformat);
+      detail::write(filename, mesh, vecs, writeParallel, cps, dataformat, writeMacro);
     }
     
 #ifdef HAVE_PARALLEL_DOMAIN_AMDIS 
diff --git a/AMDiS/src/io/FileWriter.cc b/AMDiS/src/io/FileWriter.cc
index 593fe04b..c2b39ee4 100644
--- a/AMDiS/src/io/FileWriter.cc
+++ b/AMDiS/src/io/FileWriter.cc
@@ -73,6 +73,13 @@ namespace AMDiS
       
       for (size_t i = 0; i < solutionVecs.size(); i++)
 	solutionNames.push_back(solutionVecs[i]->getName());
+      
+      std::set<Mesh*> meshset;
+      for (size_t i = 0; i < solutionVecs.size(); i++)
+	if (solutionVecs[i])
+	  meshset.insert(solutionVecs[i]->getFeSpace()->getMesh());
+
+      multiMesh = meshset.size() > 1;
     }
 
 
@@ -201,8 +208,17 @@ namespace AMDiS
 	Arh2Writer::writeFile(solutionVecs, fn_ + ".arh");
       else if (writeArh1)
 	ArhWriter::write(fn_ + ".arh", feSpace->getMesh(), solutionVecs);
-      else if (writeArh3 || writeArhFormat)
-	Arh3Writer::writeFile(solutionVecs, fn_ + ".arh");
+      else if (writeArh3 || writeArhFormat) {
+#if HAVE_PARALLEL_DOMAIN_AMDIS
+	if (MPI::COMM_WORLD.Get_rank() == 0) {
+	  TEST_EXIT(!multiMesh) ("Multi mesh arh writer is not supported in parallel.\n");
+	  Arh3Writer::detail::writeParallelFile(paraFilename + arhParallelFileExt, 
+						feSpace->getMesh(), 
+						createSubDir > 0);
+	}
+#endif
+	Arh3Writer::writeFile(solutionVecs, fn + ".arh");
+      }
     
     
   #ifdef HAVE_PNG
diff --git a/AMDiS/src/io/FileWriter.h b/AMDiS/src/io/FileWriter.h
index 81e6f653..2341b46f 100644
--- a/AMDiS/src/io/FileWriter.h
+++ b/AMDiS/src/io/FileWriter.h
@@ -102,6 +102,9 @@ namespace AMDiS {
 
       /// Reads all file writer dependend parameters from the init file.
       virtual void readParameters(std::string name) override;
+      
+      /// Multi mesh flag
+      bool multiMesh;
 
       /// Name of the writer.
       std::string name;
@@ -117,6 +120,12 @@ namespace AMDiS {
 
       /// Parallel VTK file extension.
       std::string paraviewParallelFileExt;
+      
+      /// Parallel ARH file extension.
+      std::string arhParallelFileExt;
+      
+      /// Time ARH file extension.
+      std::string arhTimeFileExt;
 
       /// Periodic file extension.
       std::string periodicFileExt;
diff --git a/AMDiS/src/io/FileWriter.hh b/AMDiS/src/io/FileWriter.hh
index 7e74f233..b060e9ca 100644
--- a/AMDiS/src/io/FileWriter.hh
+++ b/AMDiS/src/io/FileWriter.hh
@@ -119,10 +119,13 @@ namespace AMDiS
     template<typename T>
     void FileWriter<T>::initialize()
     {
+      multiMesh = false;
       amdisMeshExt = ".mesh";
       amdisDataExt = ".dat";
       paraviewFileExt = ".vtu";
       paraviewParallelFileExt = ".pvtu";
+      arhParallelFileExt = ".parh";
+      arhTimeFileExt = ".tarh";
       periodicFileExt = ".per";
       writeAMDiSFormat = 0;
       writeParaViewFormat = 0;
diff --git a/AMDiS/src/io/detail/Arh3Reader.cc b/AMDiS/src/io/detail/Arh3Reader.cc
index a45b067a..3cbe5b6c 100644
--- a/AMDiS/src/io/detail/Arh3Reader.cc
+++ b/AMDiS/src/io/detail/Arh3Reader.cc
@@ -24,6 +24,20 @@ namespace AMDiS { namespace io {
   {
     namespace detail
     {
+      void firstRead(std::string id_, uint8_t major_, uint8_t minor_, //version in file
+		     std::string id, uint8_t major, uint8_t minor)   //class version
+      {
+	TEST_EXIT(id == id_)
+	  ("Cannot read Arh format: this file is not \"%s\" format.\n", id.c_str());
+	  
+	TEST_EXIT(major == major_)
+	  ("Cannot read Arh format: Arh2Reader major version is %i, the file major version is %i. \n",
+	  major, major_);
+	  
+	TEST_EXIT(minor >= minor_)
+	  ("Cannot read Arh format: ArhReader minor version is %i is smaller than the file minor version %i.\n", 
+	  minor, minor_);
+      }
       
       void firstRead(ifstream& file, string id, uint8_t major, uint8_t minor)
       {
@@ -34,18 +48,10 @@ namespace AMDiS { namespace io {
 	string typeId(4, ' ');
     
 	file.read(const_cast<char*>(typeId.data()), 4);
-	TEST_EXIT(typeId == id)
-	  ("Cannot read Arh format: this file is not \"%s\" format.\n", id.c_str());
-
 	file.read(reinterpret_cast<char*>(&major_), 1);
-	TEST_EXIT(major == major_)
-	  ("Cannot read Arh format: Arh2Reader major version is %i, the file major version is %i. \n",
-	  major, major_);
-
 	file.read(reinterpret_cast<char*>(&minor_), 1);
-	TEST_EXIT(minor <= minor_)
-	  ("Cannot read Arh format: ArhReader minor version is %i is smaller than the file minor version %i.\n", 
-	  minor, minor_);
+	
+	firstRead(typeId, major_, minor_, id, major, minor);
       }
       
       void setDofValues(int macroElIndex, Mesh *mesh,
@@ -315,7 +321,19 @@ namespace AMDiS { namespace io {
 	  tmpString.resize(4, ' ');
 	  file.read(const_cast<char*>(tmpString.data()), 4);
 	  dataformat.push_back(tmpString);
-	} 
+	}
+	
+	// Read macroFile_nl
+	uint32_t macroFile_nl = 0;
+	file.read(reinterpret_cast<char*>(&macroFile_nl), 4);
+	if (macroFile_nl > 0) {
+	  string tmpString("");
+	  tmpString.resize(macroFile_nl, ' ');
+	  file.read(const_cast<char*>(tmpString.data()), macroFile_nl);
+	}
+	
+	//================header is over==================//
+	
 	// Adjust and check vecs
 	if(byName)
 	{
@@ -546,38 +564,49 @@ namespace AMDiS { namespace io {
 	}
 	if (writeParallel) {
 	  using boost::lexical_cast;
+	  using namespace boost::filesystem;
+	  
+	  path file_name = filename;
+	  path file_onlyname = file_name.filename();
 	  int sPos = filename.find(".arh");
-	  TEST_EXIT(sPos >= 0)("Failed to find file postfix!\n");
-	  string name = filename.substr(0, sPos);
+	  string filenameWithoutExt = filename.substr(0, sPos);
+	  sPos = file_onlyname.string().find(".arh");
+	  string onlynameWithoutExt = file_onlyname.string().substr(0, sPos);
+	  string parh = filenameWithoutExt + ".parh";
+	  string basedir = "";
+	  path file_path = file_name.remove_filename();
+	  
+	  int nProcs_ = 0, nMacros_ = 0, nMacros = 0;
+	  vector<int> partition;
+	  bool parhExists = exists(parh);
+	  
+	  if (parhExists)
+	    basedir = readParallelFile(parh, partition, nProcs_, nMacros_);
 
+	  if (basedir != "")
+	    filenameWithoutExt = file_path.string() + '/' + basedir + onlynameWithoutExt;
+	    
 	  if (nProcs == -1) {
 #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
-	    string procFilename = name + "-p" + lexical_cast<string>(MPI::COMM_WORLD.Get_rank()) + "-.arh";
+	    string procFilename = filenameWithoutExt + "-p" + lexical_cast<string>(MPI::COMM_WORLD.Get_rank()) + "-.arh";
 	    read(procFilename, mesh, vecs, byName);
 	    MSG("ARH file read from: %s\n", procFilename.c_str());
 #else
 	    ERROR_EXIT("Reading parallel ARH files in sequential computations requires to specify the number of nodes on which the ARH file was created!\n");
 #endif
 	  } else {
-	    string parhfn = name + ".parh", filenameType = "";
-	    int nProcs_ = 0, nMacros_ = 0, nMacros = 0;
-	    vector<int> partition;
-	    
-	    bool parh = boost::filesystem::exists(parhfn);
-	    if (parh)
-	      readParallelFile(parhfn, filenameType, partition, nProcs_, nMacros_);
-	    else {
+	    if (!parhExists) {
 	      for (; nProcs_ < nProcs + 1; nProcs_++) {
-		string fn = name + "-p" + boost::lexical_cast<string>(nProcs_) + "-.arh";
+		string fn = filenameWithoutExt + "-p" + boost::lexical_cast<string>(nProcs_) + "-.arh";
 		if(!boost::filesystem::exists(fn)) break;
 	      }
 	    }
 	    TEST_EXIT(nProcs_ == nProcs)
 	      ("Number of arh files doesn't match number of processors: %d vs %d\n", nProcs_, nProcs);
 	      
-	    if (!parh) {
+	    if (!parhExists) {
 #ifdef HAVE_PARALLEL_DOMAIN_AMDIS	      
-	      if(MPI::COMM_WORLD.Get_rank() == 0)
+	      if (MPI::COMM_WORLD.Get_rank() == 0)
 #endif
 	        for(int i = 0; i < nProcs; i++)
 		  nMacros_ += readNumOfMacrosFromSgArh(filename, i);
@@ -594,9 +623,9 @@ namespace AMDiS { namespace io {
 		  nMacros, nMacros_); 
 	    }
 
-	    if (!parh) {
+	    if (!parhExists) {
 	      for (int i = 0; i < nProcs; i++) {
-		string procFilename = name + "-p" + lexical_cast<string>(i) + "-.arh";
+		string procFilename = filenameWithoutExt + "-p" + lexical_cast<string>(i) + "-.arh";
 		read(procFilename, mesh, vecs, byName);
 	      }
 	    } else {
@@ -607,7 +636,7 @@ namespace AMDiS { namespace io {
 	      
 	      std::set<int>::iterator it2 = needFiles.begin();
 	      for (;it2 != needFiles.end(); it2++) {
-		string procFilename = name + "-p" + lexical_cast<string>(*it2) + "-.arh";
+		string procFilename = filenameWithoutExt + "-p" + lexical_cast<string>(*it2) + "-.arh";
 		read(procFilename, mesh, vecs, byName);
 	      }
 	    }
@@ -693,7 +722,15 @@ namespace AMDiS { namespace io {
 	  tmpString.resize(4, ' ');
 	  file.read(const_cast<char*>(tmpString.data()), 4);
 	  dataformat.push_back(tmpString);
-	} 
+	}
+	
+	// Read macroFile_nl
+	uint32_t macroFile_nl = 0;
+	file.read(reinterpret_cast<char*>(&macroFile_nl), 4);
+	string tmpString("");
+	file.read(const_cast<char*>(tmpString.data()), macroFile_nl);
+	
+	//================header is over==================//
     
 	for (uint32_t i = 0; i < nMacroElements; i++) {
 	  stringstream dataStream(ios::out | ios::in | ios::binary);
@@ -799,32 +836,41 @@ namespace AMDiS { namespace io {
 	return nMacroElements;
       }
         
-      void readParallelFile(string filename, string& filenameType, vector<int>& partition, int& nFiles, int& nMacros)
+      std::string readParallelFile(string filename, vector<int>& partition, int& nFiles, int& nMacros)
       {
 	ifstream file;
 	file.open(filename.c_str(), ios::in | ios::binary);
 	TEST_EXIT(file.is_open())
 	  ("Cannot open file %s\n", filename.c_str());
-	  
-	firstRead(file, "parh", 1, 0);
 	
-	uint32_t macroFile_nl = 0;
-	string macroFilename;
+	string fd(16, ' ');
+	file.read(const_cast<char*>(fd.data()), 16);
+	string Id = fd.substr(0, 4);
+	int major = boost::lexical_cast<int>(fd.substr(5, 1));
+	int minor = boost::lexical_cast<int>(fd.substr(7, 1));
+	firstRead(Id, major, minor, "parh", 1, 0);
 	
-	filenameType.resize(4, ' ');
+	uint32_t baseDirLen = 0, macroFile_nl = 0;
+	string baseDir, macroFilename;
 	
+	file.read(reinterpret_cast<char*>(&baseDirLen), 4);
+	baseDir.resize(baseDirLen, ' ');
+	file.read(const_cast<char*>(baseDir.data()), baseDirLen);
 	file.read(reinterpret_cast<char*>(&nFiles), 4);
-	file.read(const_cast<char*>(filenameType.data()), 4);
-	file.read(reinterpret_cast<char*>(&nMacros), 4);
 	file.read(reinterpret_cast<char*>(&macroFile_nl), 4);
-	macroFilename.resize(macroFile_nl, ' ');
-	file.read(const_cast<char*>(macroFilename.data()), macroFile_nl);
+	if (macroFile_nl > 0) {
+	  macroFilename.resize(macroFile_nl, ' ');
+	  file.read(const_cast<char*>(macroFilename.data()), macroFile_nl);
+	}
+	file.read(reinterpret_cast<char*>(&nMacros), 4);
 	
 	uint32_t rank = 0;
 	for (int i = 0; i < nMacros; i++) {
 	  file.read(reinterpret_cast<char*>(&rank), 4);
 	  partition.push_back(rank);
 	}
+	
+	return baseDir;
       }
 
     } // end namespace detail
diff --git a/AMDiS/src/io/detail/Arh3Reader.h b/AMDiS/src/io/detail/Arh3Reader.h
index 6821b1a2..857266c4 100644
--- a/AMDiS/src/io/detail/Arh3Reader.h
+++ b/AMDiS/src/io/detail/Arh3Reader.h
@@ -23,6 +23,8 @@ namespace AMDiS { namespace io {
      * 3. the minor version of Arh2Reader is bigger than the one in the file.
      * return value: minor version
      */
+      void firstRead(std::string, uint8_t, uint8_t, std::string, uint8_t, uint8_t);
+      
       void firstRead(std::ifstream& file, std::string, uint8_t, uint8_t);
       
       void setDofValues(int macroElIndex, Mesh *mesh,
@@ -80,8 +82,7 @@ namespace AMDiS { namespace io {
       int readNumOfMacrosFromSgArh(std::string filename, int nProc = -1);
       
      
-      void readParallelFile(std::string filename, 
-			    std::string& filenameType, 
+      std::string readParallelFile(std::string filename,
 			    std::vector<int>& partition, 
 			    int& nFiles,
 			    int& nMacros);
diff --git a/AMDiS/src/io/detail/Arh3Writer.cc b/AMDiS/src/io/detail/Arh3Writer.cc
index 5003c41c..6d010420 100644
--- a/AMDiS/src/io/detail/Arh3Writer.cc
+++ b/AMDiS/src/io/detail/Arh3Writer.cc
@@ -1,6 +1,7 @@
 #include <fstream>
 #include <stdint.h>
 #include <iostream>
+#include <streambuf>
 
 #include "Arh3Writer.h"
 #include "Mesh.h"
@@ -31,7 +32,7 @@ namespace AMDiS { namespace io {
 			bool writeParallel,
 			Cpsformat cps,
 			string dataformat,
-			string filenameType)
+			bool writeMacro)
       {
         vector<DOFVector<double>*> vecs(0);
         if (vec0 != NULL)
@@ -41,35 +42,53 @@ namespace AMDiS { namespace io {
         if (vec2 != NULL)
           vecs.push_back(vec2);
 
-        write(filename, NULL, vecs, writeParallel, cps, dataformat, filenameType);
+        write(filename, NULL, vecs, writeParallel, cps, dataformat, writeMacro);
       }
   
 #ifdef HAVE_PARALLEL_DOMAIN_AMDIS  
-      void writeParallelFile(string filename, Mesh* mesh, string filenameType)
+      void writeParallelFile(string filename, Mesh* mesh, bool createSubDir, bool writeMacro)
       {
-	TEST_EXIT(filenameType == "cont")("Unsupported filename type.\n");
-	
 	ofstream file;
         file.open(filename.c_str(), ios::out | ios::binary | ios::trunc);
 	
-	string typeId = "parh", macroFilename = "";
+	string typeId = "parh", macroFilename = "", perFilename = "";
+	string baseDir = createSubDir ?  "./data/" : "";
+	string macroData = "", periodicData = "";
+	uint32_t baseDirLen = baseDir.length();
 	Parameters::get(mesh->getName() + "->macro file name", macroFilename);
-	uint8_t major = 1;
-	uint8_t minor = 0;
+	Parameters::get(mesh->getName() + "->periodic file", perFilename);
+	
+	int major = 1, minor = 0;
 	uint32_t nFiles = MPI::COMM_WORLD.Get_size();
-	uint32_t macroFile_nl = macroFilename.length();
+	uint32_t macroFile_nl = 0;
 	map<int, int> partitionMap = 
 	  Parallel::MeshDistributor::globalMeshDistributor->getPartitionMap();
-	uint32_t nMacros =partitionMap.size();
-
-	file.write(typeId.c_str(), 4);
-	file.write(reinterpret_cast<char*>(&major), 1);
-        file.write(reinterpret_cast<char*>(&minor), 1);
+	uint32_t nMacros = partitionMap.size();
+	
+	typeId += '_' + boost::lexical_cast<string>(major) + '.' + boost::lexical_cast<string>(minor);
+	if (typeId.length() <= 16) {	// 16 is the Id size
+	  string rest(16 - typeId.length(), ' ');
+	  typeId.append(rest);
+	}
+	else
+	  ERROR_EXIT("Should not happen.\n");
+	
+	file.write(typeId.c_str(), 16);
+	file.write(reinterpret_cast<char*>(&baseDirLen), 4);
+	file.write(baseDir.c_str(), baseDirLen);
 	file.write(reinterpret_cast<char*>(&nFiles), 4);
-	file.write(filenameType.c_str(), 4);
-	file.write(reinterpret_cast<char*>(&nMacros), 4);
+	
+	if (writeMacro && macroFilename.length()) {
+	  macroFile_nl = 13;
+	  readFileToString(macroFilename, macroData);
+	  
+	  if (perFilename.length()) {
+	    macroFile_nl = 27;
+	    readFileToString(perFilename, periodicData);
+	  }
+	}
 	file.write(reinterpret_cast<char*>(&macroFile_nl), 4);
-	file.write(macroFilename.c_str(), macroFile_nl);
+	file.write(reinterpret_cast<char*>(&nMacros), 4);
 	
 	map<int, int>::const_iterator it = partitionMap.begin();
 	uint32_t rank = 0;
@@ -77,6 +96,29 @@ namespace AMDiS { namespace io {
 	  rank = it->second;
 	  file.write(reinterpret_cast<char*>(&rank), 4);
 	}
+	
+	// write macro and periodic file
+	if (writeMacro && macroFilename.length()) {
+	  file.seekp(0, ios_base::end);
+	  long macroPos = file.tellp(), perPos = 0;
+	  file.write(macroData.c_str(), macroData.length());
+	  
+	  if (perFilename.length()) {
+	    perPos = file.tellp();
+	    file.write(periodicData.c_str(), periodicData.length());
+	  }
+	  
+	  // update macroFile_nl
+	  int offset = 16 +		//typeId
+		       4 +		//baseDirLen
+		       baseDirLen +	//baseDir
+		       4 +		//nFiles
+		       4;		//macroFile_nl
+	  file.seekp(offset);
+	  file << "this:" << macroPos;
+	  if (perFilename.length())
+	    file << ";this:" << perPos;
+	}
       }
 #endif
 
@@ -86,100 +128,74 @@ namespace AMDiS { namespace io {
 		 bool writeParallel,
 		 Cpsformat cps,
 		 string dataformat,
-		 string filenameType)
+		 bool writeMacro)
       {
 	FUNCNAME("Arh3Writer::detail::write()");
 	
-	if (!mesh && vecs.empty()) {
-	  WARNING("There is nothing to be writen.\n");
-	  return;
-	}
-	
 	map<string,Valformat>::const_iterator it = dataformatMap.find(dataformat);
 	TEST_EXIT(it != dataformatMap.end())("Wrong data format.\n");
 	
-	std::set<string> nameSet;
-	pair<std::set<string>::iterator,bool> ret;
-	
-	for(size_t i = 0; i < vecs.size(); i++)
-	{
+	std::set<Mesh*> meshset;
+	std::set<string> nameset;
+	if (mesh)
+	  meshset.insert(mesh);
+	for (size_t i = 0; i < vecs.size(); i++) {
 	  TEST_EXIT(vecs[i] != NULL)("Vecs[%i] is NULL. Please check.\n", i);
-	  ret = nameSet.insert(vecs[i]->getName());
-	  TEST_EXIT(ret.second)("DOFVectors in vecs cannot have idential name. Please check.\n");
-	} 
-	
-#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
-	if (writeParallel) {
-	  using boost::lexical_cast;
-	  
-	  int sPos = filename.find(".arh");
-	  TEST_EXIT(sPos >= 0)("Failed to find file postfix!\n");
-	  string name = filename.substr(0, sPos);
-	  std::vector<int> macroIdx;
-	  
-	  Mesh* mesh_ = mesh ? mesh : vecs[0]->getFeSpace()->getMesh();
-	  
-	  if (MPI::COMM_WORLD.Get_rank() == 0) {
-	    writeParallelFile(name + ".parh", mesh_, filenameType);
-	  }
+	  meshset.insert(vecs[i]->getFeSpace()->getMesh());
+	  nameset.insert(vecs[i]->getName());
+	}
 	  
-	  TEST_EXIT(filenameType == "cont")("Only filename type \"cont\".\n");
-	  filename = name + "-p" + lexical_cast<string>(MPI::COMM_WORLD.Get_rank()) + "-.arh";
+	if (meshset.size() == 0) {
+	  WARNING("There is nothing to be writen.\n");
+	  return;
 	}
-#endif
+	
+	TEST_EXIT(nameset.size() == vecs.size())
+	  ("DOFVectors in vecs cannot have idential name. Please check.\n");
+	
+	bool multiMesh = meshset.size() > 1;
+	
 	//if mesh exists, the meshes in vecs should be the same.
-	if(mesh) 
-	{
-	  for(size_t i = 0; i < vecs.size(); i++)
-	  {
+	if (mesh) {
+	  for (size_t i = 0; i < vecs.size(); i++)
 	    TEST_EXIT(mesh == vecs[i]->getFeSpace()->getMesh())
-	    ("The mesh of DOFVector %i in vecs is not equal to the second parameter.\n", i);
-	  }
-	  writeAux(filename, mesh, vecs, writeParallel, cps, dataformat); 
-	}
-	//multiple meshes are allowed here.
-	else
-	{
-	  vector<bool> visited(vecs.size(), false);
-	  vector<DOFVector<double>*> splitedVecs(0);
-	  bool moreMesh = false;
-	  Mesh* tmpMesh = vecs[0]->getFeSpace()->getMesh();
-	  for(size_t i = 1; i < vecs.size(); i++)
-	  {
-	    if(vecs[i]->getFeSpace()->getMesh() != tmpMesh)
-	    {
-	      moreMesh = true;
-	      break;
-	    }
-	  }
-	  for(size_t i = 0; i < vecs.size(); i++)
-	  {
-	    if(!visited[i])
-	    {
-	      splitedVecs.clear();
-	      splitedVecs.push_back(vecs[i]);
-	      visited[i] = true;
-	      tmpMesh = vecs[i]->getFeSpace()->getMesh();
-	      for(size_t j = i + 1; j < vecs.size(); j++)
-	      {
-		if(vecs[j]->getFeSpace()->getMesh() == tmpMesh)
-		{
-		  splitedVecs.push_back(vecs[j]);
-		  visited[j] = true;
-		}
+	      ("The mesh of DOFVector %i in vecs is not equal to the second parameter.\n", i);
+	  writeAux(filename, mesh, vecs, writeParallel, cps, dataformat, writeMacro); 
+	} else {
+	  if (!multiMesh)
+	    writeAux(filename, vecs[0]->getFeSpace()->getMesh(), vecs, writeParallel, cps, dataformat, writeMacro); 
+	  else {
+	    vector<bool> visited(vecs.size(), false);
+	    vector<DOFVector<double>*> splitedVecs(0);
+	    Mesh* tmpMesh = NULL;
+	  
+	    for(size_t i = 0; i < vecs.size(); i++) {
+	      if(!visited[i]) {
+		splitedVecs.clear();
+		splitedVecs.push_back(vecs[i]);
+		visited[i] = true;
+		tmpMesh = vecs[i]->getFeSpace()->getMesh();
+		for(size_t j = i + 1; j < vecs.size(); j++)
+		  if(vecs[j]->getFeSpace()->getMesh() == tmpMesh)
+		  {
+		    splitedVecs.push_back(vecs[j]);
+		    visited[j] = true;
+		  }
 	      }
 	      string newfilename = filename;
-	      if(moreMesh)
-	      {
-		if(filename.length() > 4 && filename.substr(filename.length()- 4, filename.length()) == ".arh")
-		  newfilename = filename.substr(0, filename.length() - 4) +
-		  "." + tmpMesh->getName() + 
-		  filename.substr(filename.length()-4 , filename.length());
-		else
-		  newfilename = filename + "." + tmpMesh->getName() + ".arh";
-    
-	      }
-	      writeAux(newfilename, splitedVecs[0]->getFeSpace()->getMesh(), splitedVecs, writeParallel, cps, dataformat); 
+	      if(filename.length() > 4 && filename.substr(filename.length()- 4, filename.length()) == ".arh")
+		newfilename = filename.substr(0, filename.length() - 4) +
+		  "." + tmpMesh->getName() + filename.substr(filename.length()-4 , filename.length());
+	      else
+		newfilename = filename + "." + tmpMesh->getName() + ".arh";
+		
+	      writeAux(newfilename, 
+		       splitedVecs[0]->getFeSpace()->getMesh(), 
+		       splitedVecs, 
+		       writeParallel, 
+		       cps, 
+		       dataformat,
+		       writeMacro); 
 	    }
 	  }
         }        
@@ -225,7 +241,8 @@ namespace AMDiS { namespace io {
 			     fileNamesLen +		  //feSpaces table
 			     nFeSpaces * 20 + 	          //feSpaces table
 			     valueNamesLen + 	          //value vector table
-			     nValueVectors * 12;           //also value vector table
+			     nValueVectors * 12 +         //also value vector table
+			     4;				  //macroFile_nl
         string typeId = "sarh";
 #ifndef HAVE_COMPRESSION
 	cps = NONE;
@@ -300,6 +317,9 @@ namespace AMDiS { namespace io {
 	  file.write(reinterpret_cast<char*>(&feSpaceNumOfVecs[i]), 4);
 	  file.write(dataformat.c_str(), 4);
         }
+        
+        uint32_t macroFile_nl = 0;
+        file.write(reinterpret_cast<char*>(&macroFile_nl), 4);
         return headerLen;
       }
  
@@ -307,10 +327,13 @@ namespace AMDiS { namespace io {
                         vector<DOFVector<double>*> vecs,
                         bool writeParallel,
 			Cpsformat cps,
-			string dataformat)
+			string dataformat,
+			bool writeMacro)
       {
         FUNCNAME("Arh3Writer::detail::writeAux()");
 	
+	TEST_EXIT(mesh)("empty mesh.\n");
+	
         //initialization
         ofstream file;
         file.open(filename.c_str(), ios::out | ios::binary | ios::trunc);
@@ -453,6 +476,8 @@ namespace AMDiS { namespace io {
 	TEST_EXIT(macroSize.size() == (unsigned)mesh->getNumberOfMacros())("Should not happen.\n");
 	//reset the macro positions in file
 	setMacrosPos(file, headerLen, macroSize);
+	if (writeMacro)
+	  setMacroFile(file, headerLen, mesh);
         file.close();
         MSG("ARH file written to: %s\n", filename.c_str());
       }
@@ -475,6 +500,55 @@ namespace AMDiS { namespace io {
 	file.seekp(pos + 4);
 	file.write(reinterpret_cast<char*>(&startPos), 4);
       }
+      
+      void setMacroFile(std::ofstream& file, int headerLen, Mesh* mesh)
+      {
+	string macroFilename = "", perFilename = "";
+	string macroData = "", periodicData = "";
+	
+	Parameters::get(mesh->getName() + "->macro file name", macroFilename);
+	Parameters::get(mesh->getName() + "->periodic file", perFilename);
+	
+	if (!macroFilename.length()) {
+	  WARNING("macro file not found.\n");
+	  return;
+	}
+	
+	// write macro file to the end
+	readFileToString(macroFilename, macroData);
+	file.seekp(0, ios_base::end);
+	long macroPos = file.tellp(), perPos = 0;
+	file.write(macroData.c_str(), macroData.length());
+	
+	// write periodic file to the end
+	if (perFilename.length()) {
+	  readFileToString(perFilename, periodicData);
+	  perPos = file.tellp();
+	  file.write(periodicData.c_str(), periodicData.length());
+	}
+	
+	// update macroFile_nl
+	uint32_t macroFile_nl = perFilename.length() ? 27 : 13;
+	file.seekp(headerLen - 4);
+	file.write(reinterpret_cast<char*>(&macroFile_nl), 4);
+	file << "this:" << macroPos;
+	if (perFilename.length())
+	  file << ";this:" << perPos;
+	
+	// update headerLen
+	headerLen += macroFile_nl;
+	file.seekp(6);
+	file.write(reinterpret_cast<char*>(&headerLen), 4);
+      }
+      
+      void readFileToString(std::string filename, std::string& data)
+      {
+	ifstream file(filename.c_str());
+	file.seekg(0, std::ios::end);
+	data.reserve(file.tellg());
+	file.seekg(0, std::ios::beg);
+	data.assign((istreambuf_iterator<char>(file)), istreambuf_iterator<char>());
+      }
   
       pair<int, int> writeMacroElement(ofstream &file, 
 				    MeshStructure &code,
diff --git a/AMDiS/src/io/detail/Arh3Writer.h b/AMDiS/src/io/detail/Arh3Writer.h
index b3e3a378..6e46649d 100644
--- a/AMDiS/src/io/detail/Arh3Writer.h
+++ b/AMDiS/src/io/detail/Arh3Writer.h
@@ -43,7 +43,7 @@ namespace AMDiS { namespace io {
 		      bool writeParallel = true,
 		      Cpsformat cps = NONE,
 		      std::string dataformat = "SF64",
-		      std::string filenameType = "cont");
+		      bool writeMacro = false);
     
       /**
        * \ingroup Output
@@ -71,12 +71,13 @@ namespace AMDiS { namespace io {
 		  bool writeParallel = true,
 		  Cpsformat cps = NONE,
 		  std::string dataformat = "SF64",
-		  std::string filenameType = "cont");
+		  bool writeMacro = false);
 
        void writeAux(std::string filename, Mesh *mesh,
 		       std::vector<DOFVector<double>*> vecs,
 		       bool writeParallel, Cpsformat cps,
-		       std::string dataformat);
+		       std::string dataformat,
+		       bool writeMacro = false);
 
        ///\return the size of the macro block in file
        std::pair<int, int> writeMacroElement(std::ofstream &file,
@@ -98,16 +99,23 @@ namespace AMDiS { namespace io {
                             std::vector<DOFVector<double>*> vecs,
                             std::map<const FiniteElemSpace*, 
 			    std::vector<int> >& feSpaces, 
-			    Cpsformat cps, std::string dataformat);    
+			    Cpsformat cps, 
+			    std::string dataformat);    
        
        ///internal method, don't call
        void setMacrosPos(std::ofstream& file, int headerLen,
-	                    std::vector<std::pair<int, int> >& macroSize);      
+	                    std::vector<std::pair<int, int> >& macroSize);  
+       
+       ///internal method, don't call
+       void setMacroFile(std::ofstream& file, int headerLen, Mesh* mesh);
+       
+       void readFileToString(std::string filename, std::string& data);
        
 #ifdef HAVE_PARALLEL_DOMAIN_AMDIS         
        void writeParallelFile(std::string filename,
 			      Mesh* mesh,
-			      std::string filenameType);
+			      bool createSubDir,
+			      bool writeMacro = true);
 #endif
     }//end namespace detail
   } // end namespace Arh3Writer
-- 
GitLab