5#ifndef DUNE_COMMON_PARALLEL_VARIABLESIZECOMMUNICATOR_HH
6#define DUNE_COMMON_PARALLEL_VARIABLESIZECOMMUNICATOR_HH
44 template <
typename H>
auto require(
H &&
h) ->
decltype(
h.fixedSize());
52 std::enable_if_t<models<Concept::HasFixedSize, H>(),
int> = 0>
53constexpr bool callFixedSize(
H &&
handle) {
65template<
class T,
class Allocator=std::allocator<T> >
73 explicit MessageBuffer(
int size)
74 : buffer_(new T[size]), size_(size), position_(0)
80 explicit MessageBuffer(
const MessageBuffer& o)
81 : buffer_(new T[o.size_]), size_(o.size_), position_(o.position_)
93 void write(
const T& data)
95 buffer_[position_++]=data;
104 data=buffer_[position_++];
123 return position_==size_;
131 bool hasSpaceForItems(
int noItems)
133 return position_+noItems<=size_;
139 std::size_t size()
const
164 std::size_t position_;
170class InterfaceTracker
178 InterfaceTracker(
int rank, InterfaceInformation info, std::size_t fixedsize=0,
179 bool allocateSizes=
false)
180 :
fixedSize(fixedsize),rank_(rank), index_(), interface_(info), sizes_()
184 sizes_.resize(info.size());
191 void moveToNextIndex()
194 assert(index_<=interface_.size());
201 void increment(std::size_t i)
204 assert(index_<=interface_.size());
210 bool finished()
const
212 return index_==interface_.size();
215 void skipZeroIndices()
218 while(sizes_.size() && index_!=interface_.size() &&!size())
226 std::size_t index()
const
228 return interface_[index_];
233 std::size_t size()
const
235 assert(sizes_.size());
236 return sizes_[index_];
241 std::size_t* getSizesPointer()
251 return !interface_.size();
258 std::size_t indicesLeft()
const
260 return interface_.size()-index_;
276 std::size_t offset()
const
286 InterfaceInformation interface_;
287 std::vector<std::size_t> sizes_;
330template<
class Allocator=std::allocator<std::pair<InterfaceInformation,InterfaceInformation> > >
338 typedef std::map<int,std::pair<InterfaceInformation,InterfaceInformation>,
342#ifndef DUNE_PARALLEL_MAX_COMMUNICATION_BUFFER_SIZE
350 : maxBufferSize_(32768), interface_(&
inf)
359 : maxBufferSize_(32768), interface_(&
inf.interfaces())
381 : maxBufferSize_(DUNE_PARALLEL_MAX_COMMUNICATION_BUFFER_SIZE),
382 interface_(&inf.interfaces())
384 MPI_Comm_dup(inf.communicator(), &communicator_);
420 maxBufferSize_ = other.maxBufferSize_;
421 interface_ = other.interface_;
433 maxBufferSize_ = other.maxBufferSize_;
434 interface_ = other.interface_;
460 template<
class DataHandle>
485 template<
class DataHandle>
492 template<
bool FORWARD,
class DataHandle>
502 template<
bool forward,
class DataHandle>
513 template<
bool FORWARD,
class DataHandle>
524 template<
bool FORWARD,
class DataHandle>
533 template<
bool FORWARD,
class DataHandle>
541 std::size_t maxBufferSize_;
564template<
class DataHandle>
568 typedef std::size_t DataType;
570 SizeDataHandle(DataHandle& data,
571 std::vector<InterfaceTracker>& trackers)
572 : data_(data), trackers_(trackers), index_()
578 std::size_t size([[maybe_unused]] std::size_t i)
583 void gather(B& buf,
int i)
585 buf.write(data_.size(i));
587 void setReceivingIndex(std::size_t i)
591 std::size_t* getSizesPointer()
593 return trackers_[index_].getSizesPointer();
598 std::vector<InterfaceTracker>& trackers_;
603void setReceivingIndex(T&,
int)
607void setReceivingIndex(SizeDataHandle<T>& t,
int i)
609 t.setReceivingIndex(i);
618template<
bool FORWARD>
619struct InterfaceInformationChooser
624 static const InterfaceInformation&
625 getSend(
const std::pair<InterfaceInformation,InterfaceInformation>& info)
633 static const InterfaceInformation&
634 getReceive(
const std::pair<InterfaceInformation,InterfaceInformation>& info)
641struct InterfaceInformationChooser<false>
643 static const InterfaceInformation&
644 getSend(
const std::pair<InterfaceInformation,InterfaceInformation>& info)
649 static const InterfaceInformation&
650 getReceive(
const std::pair<InterfaceInformation,InterfaceInformation>& info)
661template<
class DataHandle>
665 int operator()(DataHandle& handle, InterfaceTracker& tracker,
666 MessageBuffer<typename DataHandle::DataType>& buffer,
667 [[maybe_unused]]
int i)
const
669 return operator()(handle,tracker,buffer);
679 int operator()(DataHandle& handle, InterfaceTracker& tracker,
680 MessageBuffer<typename DataHandle::DataType>& buffer)
const
682 if(tracker.fixedSize)
685 std::size_t noIndices=std::min(buffer.size()/tracker.fixedSize, tracker.indicesLeft());
686 for(std::size_t i=0; i< noIndices; ++i)
688 handle.gather(buffer, tracker.index());
689 tracker.moveToNextIndex();
691 return noIndices*tracker.fixedSize;
696 tracker.skipZeroIndices();
697 while(!tracker.finished())
698 if(buffer.hasSpaceForItems(handle.size(tracker.index())))
700 handle.gather(buffer, tracker.index());
701 packed+=handle.size(tracker.index());
702 tracker.moveToNextIndex();
716template<
class DataHandle>
726 bool operator()(DataHandle& handle, InterfaceTracker& tracker,
727 MessageBuffer<typename DataHandle::DataType>& buffer,
730 if(tracker.fixedSize)
732 std::size_t noIndices=std::min(buffer.size()/tracker.fixedSize, tracker.indicesLeft());
734 for(std::size_t i=0; i< noIndices; ++i)
736 handle.scatter(buffer, tracker.index(), tracker.fixedSize);
737 tracker.moveToNextIndex();
739 return tracker.finished();
744 for(
int unpacked=0;unpacked<count;)
746 assert(!tracker.finished());
747 assert(buffer.hasSpaceForItems(tracker.size()));
748 handle.scatter(buffer, tracker.index(), tracker.size());
749 unpacked+=tracker.size();
750 tracker.moveToNextIndex();
752 return tracker.finished();
761template<
class DataHandle>
762struct UnpackSizeEntries{
771 bool operator()(SizeDataHandle<DataHandle>& handle, InterfaceTracker& tracker,
772 MessageBuffer<
typename SizeDataHandle<DataHandle>::DataType>& buffer)
const
774 std::size_t noIndices=std::min(buffer.size(), tracker.indicesLeft());
775 std::copy(
static_cast<std::size_t*
>(buffer),
static_cast<std::size_t*
>(buffer)+noIndices,
776 handle.getSizesPointer()+tracker.offset());
777 tracker.increment(noIndices);
780 bool operator()(SizeDataHandle<DataHandle>& handle, InterfaceTracker& tracker,
781 MessageBuffer<
typename SizeDataHandle<DataHandle>::DataType>& buffer,
int)
const
783 return operator()(handle,tracker,buffer);
794void sendFixedSize(std::vector<InterfaceTracker>& send_trackers,
795 std::vector<MPI_Request>& send_requests,
796 std::vector<InterfaceTracker>& recv_trackers,
797 std::vector<MPI_Request>& recv_requests,
798 MPI_Comm communicator)
800 typedef std::vector<InterfaceTracker>::iterator TIter;
801 std::vector<MPI_Request>::iterator mIter=recv_requests.begin();
803 for(TIter iter=recv_trackers.begin(), end=recv_trackers.end(); iter!=end;
807 iter->rank(), 933881, communicator, &(*mIter));
811 std::vector<MPI_Request>::iterator mIter1=send_requests.begin();
812 for(TIter iter=send_trackers.begin(), end=send_trackers.end();
817 iter->rank(), 933881, communicator, &(*mIter1));
826template<
class DataHandle>
827struct SetupSendRequest{
828 void operator()(DataHandle& handle,
829 InterfaceTracker& tracker,
830 MessageBuffer<typename DataHandle::DataType>& buffer,
831 MPI_Request& request,
835 int size=PackEntries<DataHandle>()(handle, tracker, buffer);
837 while(!tracker.finished() && !handle.size(tracker.index()))
838 tracker.moveToNextIndex();
841 tracker.rank(), 933399, comm, &request);
850template<
class DataHandle>
851struct SetupRecvRequest{
852 void operator()(DataHandle& ,
853 InterfaceTracker& tracker,
854 MessageBuffer<typename DataHandle::DataType>& buffer,
855 MPI_Request& request,
859 if(tracker.indicesLeft())
861 tracker.rank(), 933399, comm, &request);
868template<
class DataHandle>
869struct NullPackUnpackFunctor
871 int operator()(DataHandle&, InterfaceTracker&,
872 MessageBuffer<typename DataHandle::DataType>&,
int)
876 int operator()(DataHandle&, InterfaceTracker&,
877 MessageBuffer<typename DataHandle::DataType>&)
897template<
class DataHandle,
class BufferFunctor,
class CommunicationFunctor>
898std::size_t checkAndContinue(DataHandle& handle,
899 std::vector<InterfaceTracker>& trackers,
900 std::vector<MPI_Request>& requests,
901 std::vector<MPI_Request>& requests2,
902 std::vector<MessageBuffer<typename DataHandle::DataType> >& buffers,
904 BufferFunctor buffer_func,
905 CommunicationFunctor comm_func,
909 std::size_t size=requests.size();
910 std::vector<MPI_Status> statuses(size);
912 std::vector<int> indices(size, -1);
914 MPI_Testsome(size, &(requests[0]), &no_completed, &(indices[0]), &(statuses[0]));
915 indices.resize(no_completed);
916 for(std::vector<int>::iterator index=indices.begin(), end=indices.end();
919 InterfaceTracker& tracker=trackers[*index];
920 setReceivingIndex(handle, *index);
925 MPI_Get_count(&(statuses[index-indices.begin()]),
929 buffer_func(handle, tracker, buffers[*index], count);
931 buffer_func(handle, tracker, buffers[*index]);
932 tracker.skipZeroIndices();
933 if(!tracker.finished()){
935 comm_func(handle, tracker, buffers[*index], requests2[*index], comm);
936 tracker.skipZeroIndices();
954template<
class DataHandle>
955std::size_t receiveSizeAndSetupReceive(DataHandle& handle,
956 std::vector<InterfaceTracker>& trackers,
957 std::vector<MPI_Request>& size_requests,
958 std::vector<MPI_Request>& data_requests,
959 std::vector<MessageBuffer<typename DataHandle::DataType> >& buffers,
962 return checkAndContinue(handle, trackers, size_requests, data_requests, buffers, comm,
963 NullPackUnpackFunctor<DataHandle>(), SetupRecvRequest<DataHandle>(),
false);
974template<
class DataHandle>
975std::size_t checkSendAndContinueSending(DataHandle& handle,
976 std::vector<InterfaceTracker>& trackers,
977 std::vector<MPI_Request>& requests,
978 std::vector<MessageBuffer<typename DataHandle::DataType> >& buffers,
981 return checkAndContinue(handle, trackers, requests, requests, buffers, comm,
982 NullPackUnpackFunctor<DataHandle>(), SetupSendRequest<DataHandle>());
993template<
class DataHandle>
994std::size_t checkReceiveAndContinueReceiving(DataHandle& handle,
995 std::vector<InterfaceTracker>& trackers,
996 std::vector<MPI_Request>& requests,
997 std::vector<MessageBuffer<typename DataHandle::DataType> >& buffers,
1000 return checkAndContinue(handle, trackers, requests, requests, buffers, comm,
1001 UnpackEntries<DataHandle>(), SetupRecvRequest<DataHandle>(),
1002 true, !Impl::callFixedSize(handle));
1006bool validRecvRequests(
const std::vector<MPI_Request> reqs)
1008 for(std::vector<MPI_Request>::const_iterator i=reqs.begin(), end=reqs.end();
1010 if(*i!=MPI_REQUEST_NULL)
1025template<
class DataHandle,
class Functor>
1026std::size_t setupRequests(DataHandle& handle,
1027 std::vector<InterfaceTracker>& trackers,
1028 std::vector<MessageBuffer<typename DataHandle::DataType> >& buffers,
1029 std::vector<MPI_Request>& requests,
1030 const Functor& setupFunctor,
1031 MPI_Comm communicator)
1033 typedef typename std::vector<InterfaceTracker>::iterator TIter;
1034 typename std::vector<MessageBuffer<typename DataHandle::DataType> >::iterator
1035 biter=buffers.begin();
1036 typename std::vector<MPI_Request>::iterator riter=requests.begin();
1037 std::size_t complete=0;
1038 for(TIter titer=trackers.begin(), end=trackers.end(); titer!=end; ++titer, ++biter, ++riter)
1040 setupFunctor(handle, *titer, *biter, *riter, communicator);
1041 complete+=titer->finished();
1047template<
class Allocator>
1048template<
bool FORWARD,
class DataHandle>
1049void VariableSizeCommunicator<Allocator>::setupInterfaceTrackers(DataHandle& handle,
1050 std::vector<InterfaceTracker>& send_trackers,
1051 std::vector<InterfaceTracker>& recv_trackers)
1053 if(interface_->size()==0)
1059 if(Impl::callFixedSize(
handle))
1063 typedef typename InterfaceMap::const_iterator
IIter;
1067 if(Impl::callFixedSize(
handle) && InterfaceInformationChooser<FORWARD>::getSend(
inf->second).size())
1068 fixedsize=
handle.size(InterfaceInformationChooser<FORWARD>::getSend(
inf->second)[0]);
1071 InterfaceInformationChooser<FORWARD>::getSend(
inf->second),
fixedsize));
1077template<
class Allocator>
1078template<
bool FORWARD,
class DataHandle>
1079void VariableSizeCommunicator<Allocator>::communicateFixedSize(DataHandle& handle)
1091 typedef typename DataHandle::DataType DataType;
1103 typedef typename std::vector<InterfaceTracker>::const_iterator
Iter;
1135template<
class Allocator>
1136template<
bool FORWARD,
class DataHandle>
1137void VariableSizeCommunicator<Allocator>::communicateSizes(DataHandle& handle,
1138 std::vector<InterfaceTracker>& data_recv_trackers)
1142 std::size_t
size = interface_->size();
1145 std::vector<MessageBuffer<std::size_t> >
1181template<
class Allocator>
1182template<
bool FORWARD,
class DataHandle>
1183void VariableSizeCommunicator<Allocator>::communicateVariableSize(DataHandle& handle)
1192 typedef typename DataHandle::DataType DataType;
1193 std::vector<MessageBuffer<DataType> >
1225template<
class Allocator>
1226template<
bool FORWARD,
class DataHandle>
1227void VariableSizeCommunicator<Allocator>::communicate(DataHandle& handle)
1229 if( interface_->size() == 0)
1234 if(Impl::callFixedSize(
handle))
Infrastructure for concepts.
std::size_t fixedSize
The number of data items per index if it is fixed, 0 otherwise.
Definition variablesizecommunicator.hh:265
Traits classes for mapping types onto MPI_Datatype.
size_type size() const
Get the number of elements in the list.
Definition arraylist.hh:472
ArrayList()
Constructs an Array list with one chunk.
Definition arraylist.hh:457
iterator end()
Get a random access iterator positioned after the last element.
Definition arraylist.hh:529
Dune namespace.
Definition alignedallocator.hh:13
A dynamically growing random access list.
Definition arraylist.hh:62
static MPI_Datatype getType()
Definition mpitraits.hh:48
Communication interface between remote and local indices.
Definition parallel/interface.hh:209
Definition variablesizecommunicator.hh:43
auto require(H &&h) -> decltype(h.fixedSize())
A buffered communicator where the amount of data sent does not have to be known a priori.
Definition variablesizecommunicator.hh:332
VariableSizeCommunicator(const Interface &inf, std::size_t max_buffer_size)
Creates a communicator with a specific maximum buffer size.
Definition variablesizecommunicator.hh:404
void backward(DataHandle &handle)
Communicate backwards.
Definition variablesizecommunicator.hh:486
~VariableSizeCommunicator()
Definition variablesizecommunicator.hh:410
VariableSizeCommunicator(MPI_Comm comm, const InterfaceMap &inf, std::size_t max_buffer_size)
Creates a communicator with a specific maximum buffer size.
Definition variablesizecommunicator.hh:393
VariableSizeCommunicator(const VariableSizeCommunicator &other)
Copy-constructs a communicator.
Definition variablesizecommunicator.hh:419
void forward(DataHandle &handle)
Communicate forward.
Definition variablesizecommunicator.hh:461
VariableSizeCommunicator & operator=(const VariableSizeCommunicator &other)
Copy-assignes a communicator.
Definition variablesizecommunicator.hh:429
std::map< int, std::pair< InterfaceInformation, InterfaceInformation >, std::less< int >, typename std::allocator_traits< Allocator >::template rebind_alloc< std::pair< const int, std::pair< InterfaceInformation, InterfaceInformation > > > > InterfaceMap
The type of the map from process number to InterfaceInformation for sending and receiving to and from...
Definition variablesizecommunicator.hh:340
VariableSizeCommunicator(MPI_Comm comm, const InterfaceMap &inf)
Creates a communicator with the default maximum buffer size.
Definition variablesizecommunicator.hh:349
VariableSizeCommunicator(const Interface &inf)
Creates a communicator with the default maximum buffer size.
Definition variablesizecommunicator.hh:358
Provides classes for building the communication interface between remote indices.