pcl_interface_communicator.h

Go to the documentation of this file.

/*
 * Copyright (c) 2009-2015:  G-CSC, Goethe University Frankfurt
 * Author: Sebastian Reiter
 * 
 * This file is part of UG4.
 * 
 * UG4 is free software: you can redistribute it and/or modify it under the
 * terms of the GNU Lesser General Public License version 3 (as published by the
 * Free Software Foundation) with the following additional attribution
 * requirements (according to LGPL/GPL v3 §7):
 * 
 * (1) The following notice must be displayed in the Appropriate Legal Notices
 * of covered and combined works: "Based on UG4 (www.ug4.org/license)".
 * 
 * (2) The following notice must be displayed at a prominent place in the
 * terminal output of covered works: "Based on UG4 (www.ug4.org/license)".
 * 
 * (3) The following bibliography is recommended for citation and must be
 * preserved in all covered files:
 * "Reiter, S., Vogel, A., Heppner, I., Rupp, M., and Wittum, G. A massively
 *   parallel geometric multigrid solver on hierarchically distributed grids.
 *   Computing and visualization in science 16, 4 (2013), 151-164"
 * "Vogel, A., Reiter, S., Rupp, M., Nägel, A., and Wittum, G. UG4 -- a novel
 *   flexible software system for simulating pde based models on high performance
 *   computers. Computing and visualization in science 16, 4 (2013), 165-179"
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 */
 
#ifndef __H__PCL__PCL_INTERFACE_COMMUNICATOR__
#define __H__PCL__PCL_INTERFACE_COMMUNICATOR__
 
#include <map>
#include <set>
#include <cassert>
#include "common/util/binary_buffer.h"
#include "pcl_communication_structs.h"
#include "pcl_process_communicator.h"
 
namespace pcl
{
 
 
//  There are two types of communicators:
//  - InterfaceCommunicator: Data is exchanged between elements of
//        interfaces / layouts. CommunicationPolicies are used to
//        collect and to extract interface-data.
//        Benefits are low communication overhead and ease of use.
//
//  - ProcessCommunicator: Arbitrary data is exchanged between all processes
//        that are linked by layouts.
//        Benefits are high flexibility and independency of
//        interface element order.
 
 
//  InterfaceCommunicator
template <class TLayout>
class InterfaceCommunicator
{
  public:
  //  typedefs
    typedef TLayout           Layout;
    typedef typename Layout::Interface  Interface;
    typedef typename Layout::Type   Type;
 
  protected:
    typedef ICommunicationPolicy<Layout>  CommPol;
    
  public:
    InterfaceCommunicator();
    
  //  SEND
  
 
    void send_raw(int targetProc, const void* pBuff, int bufferSize,
            bool bSizeKnownAtTarget = false);
 
 
    void send_data(int targetProc,
             const Interface& interface,
             ICommunicationPolicy<TLayout>& commPol);
 
 
    void send_data(const Layout& layout,
             ICommunicationPolicy<TLayout>& commPol);
 
  //  RECEIVE
  
 
    void receive_raw(int srcProc, ug::BinaryBuffer& bufOut,
             int bufSize = -1);
 
 
    void receive_raw(int srcProc, void* bufOut, int bufSize);
    
 
    void receive_data(int srcProc,
              const Interface& interface,
              ICommunicationPolicy<TLayout>& commPol);
 
 
    void receive_data(const Layout& layout,
              ICommunicationPolicy<TLayout>& commPol);
 
  //  EXCHANGE
 
    template <class TLayoutMap>
    void exchange_data(const TLayoutMap& layoutMap,
               const typename TLayoutMap::Key& keyFrom,
               const typename TLayoutMap::Key& keyTo,
               ICommunicationPolicy<TLayout>& commPol);
  
 
    bool communicate(int tag = 749345);
    
 
 
    bool communicate_and_resume(int tag = 749345);
 
 
    void wait();
  
 
 
    void enable_communication_debugging(const ProcessCommunicator& involvedProcs =
                        ProcessCommunicator(PCD_WORLD));
   
    void disable_communication_debugging();
  
    bool communication_debugging_enabled();
   
  protected:
    typedef std::map<int, ug::BinaryBuffer> BufferMap;
 
  protected:
    void send_data(const Layout& layout,
          ICommunicationPolicy<TLayout>& commPol,
          const layout_tags::single_level_layout_tag&);
 
    void send_data(const Layout& layout,
               ICommunicationPolicy<TLayout>& commPol,
               const layout_tags::multi_level_layout_tag&);
  
    void prepare_receiver_buffer_map(BufferMap& bufMap,
                     std::set<int>& curProcs,
                     const TLayout& layout);
    void prepare_receiver_buffer_map(BufferMap& streamPack,
                     std::set<int>& curProcs,
                     const TLayout& layout,
                     const layout_tags::single_level_layout_tag&);
    void prepare_receiver_buffer_map(BufferMap& streamPack,
                     std::set<int>& curProcs,
                     const TLayout& layout,
                     const layout_tags::multi_level_layout_tag&);
 
 
    bool collect_layout_buffer_sizes(const TLayout& layout,
                     ICommunicationPolicy<TLayout>& commPol,
                     std::map<int, int>* pMapBuffSizesOut,
                     const layout_tags::single_level_layout_tag&);
 
 
    bool collect_layout_buffer_sizes(const TLayout& layout,
                     ICommunicationPolicy<TLayout>& commPol,
                     std::map<int, int>* pMapBuffSizesOut,
                     const layout_tags::multi_level_layout_tag&);
  
    void extract_data(const TLayout& layout, BufferMap& bufMap,
              CommPol& extractor);
    
    void extract_data(const TLayout& layout, BufferMap& bufMap,
              CommPol& extractor,
              const layout_tags::single_level_layout_tag&);
    
    void extract_data(const TLayout& layout, BufferMap& bufMap,
              CommPol& extractor,
              const layout_tags::multi_level_layout_tag&);
    
  protected:    
 
    struct ExtractorInfo
    {
      ExtractorInfo()     {}
      ExtractorInfo(int srcProc, CommPol* pExtractor,
            const Interface* pInterface, const Layout* pLayout,
            void* buffer, ug::BinaryBuffer* binBuffer, int rawSize) :
        m_srcProc(srcProc), m_extractor(pExtractor),
        m_interface(pInterface), m_layout(pLayout),
        m_buffer(buffer), m_binBuffer(binBuffer), m_rawSize(rawSize)
      {
        assert((srcProc == -1) || ((srcProc >= 0) && (srcProc < pcl::NumProcs())));
      }
 
      int         m_srcProc;
      CommPol*      m_extractor;      
      const Interface*  m_interface;
      const Layout*   m_layout;
      void*       m_buffer;
      ug::BinaryBuffer* m_binBuffer;
      int         m_rawSize;
    };
 
    typedef std::list<ExtractorInfo> ExtractorInfoList;
 
  protected:
    BufferMap   m_bufMapOut;
    std::set<int> m_curOutProcs;
 
    BufferMap   m_bufMapIn;
    std::set<int> m_curInProcs;
 
    ExtractorInfoList m_extractorInfos;
    
    std::vector<MPI_Request> m_vSendRequests;
    std::vector<MPI_Request> m_vReceiveRequests;
 
    ProcessCommunicator m_debugProcComm;
    
 
    int m_curComTag;
 
    bool m_bDebugCommunication;
    
 
    bool m_bSendBuffersFixed;
};
 
// end group pcl
 
}// end of namespace pcl
 
//  include implementation
#include "pcl_interface_communicator_impl.hpp"
 
#endif