pcl_process_communicator.h

Go to the documentation of this file.

/*
 * Copyright (c) 2010-2015:  G-CSC, Goethe University Frankfurt
 * Author: Sebastian Reiter
 * 
 * This file is part of UG4.
 * 
 * UG4 is free software: you can redistribute it and/or modify it under the
 * terms of the GNU Lesser General Public License version 3 (as published by the
 * Free Software Foundation) with the following additional attribution
 * requirements (according to LGPL/GPL v3 §7):
 * 
 * (1) The following notice must be displayed in the Appropriate Legal Notices
 * of covered and combined works: "Based on UG4 (www.ug4.org/license)".
 * 
 * (2) The following notice must be displayed at a prominent place in the
 * terminal output of covered works: "Based on UG4 (www.ug4.org/license)".
 * 
 * (3) The following bibliography is recommended for citation and must be
 * preserved in all covered files:
 * "Reiter, S., Vogel, A., Heppner, I., Rupp, M., and Wittum, G. A massively
 *   parallel geometric multigrid solver on hierarchically distributed grids.
 *   Computing and visualization in science 16, 4 (2013), 151-164"
 * "Vogel, A., Reiter, S., Rupp, M., Nägel, A., and Wittum, G. UG4 -- a novel
 *   flexible software system for simulating pde based models on high performance
 *   computers. Computing and visualization in science 16, 4 (2013), 165-179"
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 */
 
#ifndef __H__PCL__PCL_PROCESS_COMMUNICATOR__
#define __H__PCL__PCL_PROCESS_COMMUNICATOR__
 
#include <iostream>
#include <map>
#include <vector>
#include "pcl_methods.h"
#include "common/util/smart_pointer.h"
#include "common/util/binary_stream.h"
#include "common/util/binary_buffer.h"
#include "common/error.h"
 
namespace pcl
{
 
 
enum ProcessCommunicatorDefaults
{
  PCD_EMPTY = 0, //< an empty communicator
  PCD_WORLD = 1, //< a communicator to all available processes
  PCD_LOCAL = 2  //< a local communicator, simulating current proc is the only proc
};
 
 
class ProcessCommunicator
{
  public:
 
    ProcessCommunicator(ProcessCommunicatorDefaults pcd = PCD_WORLD);
    
    inline bool empty() const   {return !is_local() && m_comm->m_mpiComm == MPI_COMM_NULL;}
 
    inline bool is_world() const  { return !is_local() && m_comm->m_mpiComm == PCL_COMM_WORLD; }
    
    inline bool is_local() const {return m_comm.valid() == false;}
 
    size_t size() const;
    
    int get_proc_id(size_t index) const;
 
    bool is_proc_id(size_t index) const
    {
      return get_proc_id(index) == ProcRank();
    }
 
 
    int get_local_proc_id(int globalProcID = pcl::ProcRank()) const;
 
    MPI_Comm get_mpi_communicator() const {return m_comm->m_mpiComm;}
 
 
    ProcessCommunicator create_sub_communicator(bool participate) const;
    
    ProcessCommunicator create_sub_communicator(std::vector<int> &newProcs) const;
 
    static ProcessCommunicator create_communicator(std::vector<int> &newGlobalProcs);
    static ProcessCommunicator create_communicator(size_t first, size_t num);
 
    void gather(const void* sendBuf, int sendCount, DataType sendType,
          void* recBuf, int recCount, DataType recType, int root) const;
 
    void gather(ug::BinaryBuffer &buf, int root=0) const;
 
 
    void scatter(const void* sendBuf, int sendCount, DataType sendType,
           void* recBuf, int recCount, DataType recType, int root) const;
 
 
    void gatherv(const void* sendBuf, int sendCount, DataType sendType,
            void* recBuf, int* recCounts, int* displs,
            DataType recType, int root) const;
 
 
    template<class TValue>
    void gatherv(std::vector<TValue>& recBufOut,
           std::vector<TValue>& sendBuf, int root,
           std::vector<int>* pSizesOut = NULL,
           std::vector<int>* pOffsetsOut = NULL) const;
 
 
    void allgather(const void* sendBuf, int sendCount, DataType sendType,
             void* recBuf, int recCount, DataType recType) const;
 
 
    void allgatherv(const void* sendBuf, int sendCount, DataType sendType,
            void* recBuf, int* recCounts, int* displs,
            DataType recType) const;
 
    void allgather(ug::BinaryBuffer &buf) const;
 
 
    void alltoall(const void* sendBuf, int sendCount, DataType sendType,
              void* recBuf, int recCount, DataType recType);
 
 
    template<class TValue>
    void allgatherv(std::vector<TValue>& recBufOut,
            std::vector<TValue>& sendBuf,
            std::vector<int>* pSizesOut = NULL,
            std::vector<int>* pOffsetsOut = NULL) const;
 
    
 
    void reduce(const void* sendBuf, void* recBuf, int count,
          DataType type, ReduceOperation op, int rootProc) const;
 
    template<typename T>
    T reduce(const T &t, pcl::ReduceOperation op, int rootProc) const;
 
    template<typename T>
    void reduce(const T *pSendBuff, T *pReceiveBuff, size_t count,
          pcl::ReduceOperation op, int rootProc) const;
 
    template<typename T>
    void reduce(const std::vector<T> &send, std::vector<T> &receive,
             pcl::ReduceOperation op, int rootProc) const;
 
    size_t reduce(const size_t &t, pcl::ReduceOperation op, int rootProc) const;
 
 
    void allreduce(const void* sendBuf, void* recBuf, int count,
             DataType type, ReduceOperation op) const;
 
    template<typename T>
    T allreduce(const T &t, pcl::ReduceOperation op) const;
 
    size_t allreduce(const size_t &t, pcl::ReduceOperation op) const;
 
    template<typename T>
    void allreduce(const T *pSendBuff, T *pReceiveBuff, size_t count,
             pcl::ReduceOperation op) const;
 
    template<typename T>
    void allreduce(const std::vector<T> &send, std::vector<T> &receive,
             pcl::ReduceOperation op) const;
 
 
    void broadcast(void *v, size_t size, DataType type, int root=0) const;
 
    template<typename T>
    inline void broadcast(T *p, size_t size=1, int root=0) const;
 
    template<typename T>
    inline void broadcast(T &t, int root=0) const;
 
    template<typename T>
    inline void broadcast(T &t, int root, DataTypeDirectlySupported d) const;
    template<typename T>
    void broadcast(T &t, int root, DataTypeIndirectlySupported d) const;
 
    void broadcast(size_t &s, int root=0) const;
 
    void broadcast(ug::BinaryBuffer &buf, int root=0) const;
 
 
    void barrier() const;
 
 
 
    void send_data(void* pBuffer, int bufferSize, int destProc, int tag) const;
 
 
    void send_data(void* pBuffer, int* pBufferSegSizes,
             int* pRecProcMap, int numRecProcs, int tag) const;
 
 
    void receive_data(void* pBuffOut, int bufferSize, int srcProc, int tag) const;
 
 
    void distribute_data(void* pBufferOut, int* pBufferOutSegSizes,
               int* pSenderProcMap, int numSenderProcs,
               void* pBuffer, int* pBufferSegSizes,
               int* pRecvProcMap, int numRecvProcs,
               int tag = 1) const;
 
 
    void distribute_data(ug::BinaryBuffer& recvBufOut, int* segSizesOut,
              int* recvFromRanks, int numRecvFroms,
              void* sendBuf, int* sendSegSizes,
              int* sendToRanks, int numSendTos,
               int tag = 1) const;
 
 
    void distribute_data(ug::BinaryBuffer* recvBufs, int* recvFromRanks, int numRecvs,
               ug::BinaryBuffer* sendBufs, int* sendToRanks, int numSendTos,
               int tag = 1) const;
  private:
 
    struct CommWrapper{
      CommWrapper();
      CommWrapper(const MPI_Comm& comm,
            bool bReleaseComm);
      ~CommWrapper();
      
      MPI_Comm      m_mpiComm;
      bool        m_bReleaseCommunicator;
 
      std::vector<int>  m_procs;
    };
    
    typedef SmartPtr<CommWrapper> SPCommWrapper;
    
  private:
    SPCommWrapper m_comm;
 
};
 
std::ostream &operator << (std::ostream &out, const ProcessCommunicator &processCommunicator);
 
// end group pcl
 
}// end of namespace pcl
 
//  include implementation
#include "pcl_process_communicator_impl.hpp"
 
#endif