parallel_dual_graph.h

Go to the documentation of this file.

/*
 * Copyright (c) 2012-2015:  G-CSC, Goethe University Frankfurt
 * Author: Sebastian Reiter
 * 
 * This file is part of UG4.
 * 
 * UG4 is free software: you can redistribute it and/or modify it under the
 * terms of the GNU Lesser General Public License version 3 (as published by the
 * Free Software Foundation) with the following additional attribution
 * requirements (according to LGPL/GPL v3 §7):
 * 
 * (1) The following notice must be displayed in the Appropriate Legal Notices
 * of covered and combined works: "Based on UG4 (www.ug4.org/license)".
 * 
 * (2) The following notice must be displayed at a prominent place in the
 * terminal output of covered works: "Based on UG4 (www.ug4.org/license)".
 * 
 * (3) The following bibliography is recommended for citation and must be
 * preserved in all covered files:
 * "Reiter, S., Vogel, A., Heppner, I., Rupp, M., and Wittum, G. A massively
 *   parallel geometric multigrid solver on hierarchically distributed grids.
 *   Computing and visualization in science 16, 4 (2013), 151-164"
 * "Vogel, A., Reiter, S., Rupp, M., Nägel, A., and Wittum, G. UG4 -- a novel
 *   flexible software system for simulating pde based models on high performance
 *   computers. Computing and visualization in science 16, 4 (2013), 165-179"
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 */
 
#ifndef __H__LIB_GRID__PARALLEL_DUAL_GRAPH__
#define __H__LIB_GRID__PARALLEL_DUAL_GRAPH__
 
#include <vector>
#include "lib_grid/lg_base.h"
#include "pcl/pcl_process_communicator.h"
#include "../distributed_grid.h"
#include "../parallelization_util.h"
 
namespace ug
{
 
 
template <class TGeomBaseObj, class TIndexType,
      class TConnectingObj = typename TGeomBaseObj::side>
class ParallelDualGraph{
  public:
    typedef typename std::vector<TGeomBaseObj*>::iterator element_iterator_t;
 
    ParallelDualGraph(MultiGrid* pmg = NULL);
    ~ParallelDualGraph();
 
    void set_grid(MultiGrid* pmg);
 
 
    TIndexType num_graph_vertices();
    TIndexType num_graph_edges();
    TIndexType* adjacency_map_structure();
    TIndexType* adjacency_map();
    TIndexType* parallel_offset_map();
 
    bool was_considered(TGeomBaseObj* o);
 
    TGeomBaseObj* get_element(size_t gvrtIndex)   {return m_elems[gvrtIndex];}
 
 
    TConnectingObj* get_connection(size_t connIndex)    {return m_connections.at(connIndex);}
 
    element_iterator_t elements_begin()       {return m_elems.begin();}
 
    element_iterator_t elements_end()       {return m_elems.end();}
 
    TIndexType get_index(TGeomBaseObj* elem)    {return m_aaElemIndex[elem];}
 
 
    void generate_graph(int level, pcl::ProcessCommunicator procCom =
                      pcl::ProcessCommunicator(pcl::PCD_WORLD));
 
 
    pcl::ProcessCommunicator process_communicator() const {return m_procCom;}
 
  private:
    void attach_data();
    void detach_data();
 
  //  Members
    typedef Attachment<int>         AElemIndex;
    typedef Attachment<std::vector<int> > AElemIndices;
 
    pcl::ProcessCommunicator  m_procCom;
    MultiGrid*        m_pMG;
    std::vector<TGeomBaseObj*>  m_elems;
    std::vector<TConnectingObj*>  m_connections;
    std::vector<TIndexType> m_adjacencyMapStructure;
    std::vector<TIndexType> m_adjacencyMap;
    std::vector<TIndexType> m_nodeOffsetMap;
    AElemIndex        m_aElemIndex;
    AElemIndices      m_aElemIndices;
    Grid::AttachmentAccessor<TGeomBaseObj, AElemIndex>    m_aaElemIndex;
    Grid::AttachmentAccessor<TConnectingObj, AElemIndices>  m_aaElemIndices;
};
 
 
template <class TGeomBaseObj, class TIndexType>
void ConstructParallelDualGraphMGLevel(
    std::vector<TIndexType>& adjacencyMapStructureOut,
    std::vector<TIndexType>& adjacencyMapOut,
    std::vector<TIndexType>& nodeOffsetMapOut,
    MultiGrid& mg, size_t level,
    pcl::ProcessCommunicator procCom,
    Attachment<TIndexType>* paIndex = NULL,
    TGeomBaseObj** pGeomObjsOut = NULL,
    NeighborhoodType nbhType = NHT_DEFAULT)
{
  GDIST_PROFILE_FUNC();
  using namespace std;
  typedef TGeomBaseObj Elem;
  typedef typename geometry_traits<Elem>::iterator ElemIterator;
 
//  set up index attachment and attachment accessor
  typedef Attachment<TIndexType> AIndex;
  AIndex aIndex;
  if(paIndex)
    aIndex = *paIndex;
 
  if(!mg.has_attachment<Elem>(aIndex))
    mg.attach_to<Elem>(aIndex);
 
  Grid::AttachmentAccessor<Elem, AIndex> aaInd(mg, aIndex);
 
//  init the indices and count ghosts and normal elements on the fly
  size_t numElems = 0;
  {
    DistributedGridManager& distGridMgr = *mg.distributed_grid_manager();
    for(ElemIterator iter = mg.begin<Elem>(level);
      iter != mg.end<Elem>(level); ++iter)
    {
      if(distGridMgr.is_ghost(*iter))
        aaInd[*iter] = -1;
      else
        aaInd[*iter] = numElems++;
    }
  }
 
//  generate the nodeOffsetMap
  nodeOffsetMapOut.clear();
  int localNodeOffset = 0;
 
  int numElemsTmp = (int)numElems;
  vector<int> elemCounts(procCom.size());
  procCom.allgather(&numElemsTmp, 1, PCL_DT_INT,
            &elemCounts.front(), 1, PCL_DT_INT);
 
  nodeOffsetMapOut.resize(procCom.size() + 1);
  int numElemsTotal = 0;
  for(size_t i = 0; i < elemCounts.size(); ++i){
    nodeOffsetMapOut[i] = numElemsTotal;
    numElemsTotal += elemCounts[i];
  }
  nodeOffsetMapOut[elemCounts.size()] = numElemsTotal;
  localNodeOffset = nodeOffsetMapOut[procCom.get_local_proc_id()];
 
//  init the adjacencyMapStructure
  adjacencyMapStructureOut.resize(numElems + 1);
  adjacencyMapOut.clear();
 
//  construct the graph
  vector<Elem*> vNeighbours;
  int ind = 0;
 
//  generate adjacency structure first
  for(ElemIterator iter = mg.begin<Elem>(level);
    iter != mg.end<Elem>(level); ++iter)
  {
    Elem* elem = *iter;
 
    if(aaInd[elem] == -1)
      continue;
 
  //  get all neighbours
    CollectNeighbors(vNeighbours, elem, mg, nbhType);
 
  //  store first entry at which the connections will be written to the map
    adjacencyMapStructureOut[ind] = adjacencyMapOut.size();
 
  //  iterate over the neighbours and push adjacent indices to the adjacencyMap
    for(size_t i = 0; i < vNeighbours.size(); ++i){
      if(aaInd[vNeighbours[i]] != -1)
        adjacencyMapOut.push_back(localNodeOffset + aaInd[vNeighbours[i]]);
    }
 
    ++ind;
  }
 
//  add the final element
  adjacencyMapStructureOut[adjacencyMapStructureOut.size() - 1] = adjacencyMapOut.size();
 
//  fill pGeomObjsOut
  if(pGeomObjsOut)
  {
    int ind = 0;
    for(ElemIterator iter = mg.begin<Elem>(level);
      iter != mg.end<Elem>(level); ++iter)
    {
      if(aaInd[*iter] != -1){
        pGeomObjsOut[ind] = *iter;
        ++ind;
      }
    }
  }
 
//  clean up
  if(!paIndex)
    mg.detach_from<Elem>(aIndex);
}
 
}// end of namespace
 
 
//  include implementation
#include "parallel_dual_graph_impl.hpp"
 
#endif