11 #include <dolfinx/common/MPI.h>
12 #include <dolfinx/common/Timer.h>
13 #include <dolfinx/common/array2d.h>
14 #include <dolfinx/graph/AdjacencyList.h>
34 using partition_fn = std::function<graph::AdjacencyList<std::int32_t>(
36 std::int32_t num_ghost_nodes,
bool ghosting)>;
52 std::int32_t num_ghost_nodes,
bool ghosting);
70 std::tuple<graph::AdjacencyList<std::int64_t>, std::vector<int>,
71 std::vector<std::int64_t>, std::vector<int>>
82 std::vector<std::int64_t>
84 const std::vector<std::int64_t>& global_indices,
85 const std::vector<int>& ghost_owners);
98 const std::vector<std::int64_t>& indices,
112 std::vector<std::int64_t>
124 std::vector<std::int32_t>
126 const std::vector<std::int64_t>& local1_to_global);
132 template <
typename T>
134 const std::vector<std::int64_t>& indices,
137 common::Timer timer(
"Fetch float data from remote processes");
139 const std::int64_t num_points_local = x.
shape[0];
142 std::vector<std::int64_t> global_sizes(size);
143 MPI_Allgather(&num_points_local, 1, MPI_INT64_T, global_sizes.data(), 1,
145 std::vector<std::int64_t> global_offsets(size + 1, 0);
146 std::partial_sum(global_sizes.begin(), global_sizes.end(),
147 global_offsets.begin() + 1);
150 std::vector<int> number_index_send(size, 0);
151 std::vector<int> index_owner(indices.size());
152 std::vector<int> index_order(indices.size());
153 std::iota(index_order.begin(), index_order.end(), 0);
154 std::sort(index_order.begin(), index_order.end(),
155 [&indices](
int a,
int b) { return (indices[a] < indices[b]); });
158 for (std::size_t i = 0; i < index_order.size(); ++i)
160 int j = index_order[i];
161 while (indices[j] >= global_offsets[p + 1])
164 number_index_send[p]++;
168 std::vector<int> disp_index_send(size + 1, 0);
169 std::partial_sum(number_index_send.begin(), number_index_send.end(),
170 disp_index_send.begin() + 1);
173 std::vector<std::int64_t> indices_send(disp_index_send.back());
174 std::vector<int> disp_tmp = disp_index_send;
175 for (std::size_t i = 0; i < indices.size(); ++i)
177 const int owner = index_owner[i];
178 indices_send[disp_tmp[owner]++] = indices[i];
182 std::vector<int> number_index_recv(size);
183 MPI_Alltoall(number_index_send.data(), 1, MPI_INT, number_index_recv.data(),
187 std::vector<int> disp_index_recv(size + 1, 0);
188 std::partial_sum(number_index_recv.begin(), number_index_recv.end(),
189 disp_index_recv.begin() + 1);
192 std::vector<std::int64_t> indices_recv(disp_index_recv.back());
193 MPI_Alltoallv(indices_send.data(), number_index_send.data(),
194 disp_index_send.data(), MPI_INT64_T, indices_recv.data(),
195 number_index_recv.data(), disp_index_recv.data(), MPI_INT64_T,
198 assert(x.
shape[1] != 0);
201 for (
int p = 0; p < size; ++p)
203 for (
int i = disp_index_recv[p]; i < disp_index_recv[p + 1]; ++i)
205 const std::int32_t index_local = indices_recv[i] - global_offsets[rank];
206 assert(index_local >= 0);
207 for (std::size_t j = 0; j < x.
shape[1]; ++j)
208 x_return(i, j) = x(index_local, j);
212 MPI_Datatype compound_type;
213 MPI_Type_contiguous(x.
shape[1], dolfinx::MPI::mpi_type<T>(), &compound_type);
214 MPI_Type_commit(&compound_type);
218 MPI_Alltoallv(x_return.
data(), number_index_recv.data(),
219 disp_index_recv.data(), compound_type, my_x.
data(),
220 number_index_send.data(), disp_index_send.data(), compound_type,
static int rank(MPI_Comm comm)
Return process rank for the communicator.
Definition: MPI.cpp:77
static int size(MPI_Comm comm)
Return size of the group (number of processes) associated with the communicator.
Definition: MPI.cpp:85
This class provides a dynamic 2-dimensional row-wise array data structure.
Definition: array2d.h:21
std::array< size_type, 2 > shape
The shape of the array.
Definition: array2d.h:157
constexpr value_type * data() noexcept
Get pointer to the first element of the underlying storage.
Definition: array2d.h:133
A timer can be used for timing tasks. The basic usage is.
Definition: Timer.h:31
This class provides a static adjacency list data structure. It is commonly used to store directed gra...
Definition: AdjacencyList.h:68
std::vector< std::int64_t > compute_ghost_indices(MPI_Comm comm, const std::vector< std::int64_t > &global_indices, const std::vector< int > &ghost_owners)
Compute ghost indices in a global IndexMap space, from a list of arbitrary global indices,...
Definition: partition.cpp:158
std::vector< std::int64_t > compute_local_to_global_links(const graph::AdjacencyList< std::int64_t > &global, const graph::AdjacencyList< std::int32_t > &local)
Given an adjacency list with global, possibly non-contiguous, link indices and a local adjacency list...
Definition: partition.cpp:274
std::tuple< graph::AdjacencyList< std::int64_t >, std::vector< int >, std::vector< std::int64_t >, std::vector< int > > distribute(MPI_Comm comm, const graph::AdjacencyList< std::int64_t > &list, const graph::AdjacencyList< std::int32_t > &destinations)
Distribute adjacency list nodes to destination ranks. The global index of each node is assumed to be ...
Definition: partition.cpp:31
array2d< T > distribute_data(MPI_Comm comm, const std::vector< std::int64_t > &indices, const array2d< T > &x)
Distribute data to process ranks where it it required.
Definition: partition.h:133
std::vector< std::int32_t > compute_local_to_local(const std::vector< std::int64_t > &local0_to_global, const std::vector< std::int64_t > &local1_to_global)
Compute a local0-to-local1 map from two local-to-global maps with common global indices.
Definition: partition.cpp:311
Graph data structures and algorithms.
Definition: AdjacencyList.h:18
AdjacencyList< std::int32_t > partition_graph(const MPI_Comm comm, int nparts, const AdjacencyList< std::int64_t > &local_graph, std::int32_t num_ghost_nodes, bool ghosting)
Partition graph across processes using the default graph partitioner.
Definition: partition.cpp:21
std::function< graph::AdjacencyList< std::int32_t >(MPI_Comm comm, int nparts, const AdjacencyList< std::int64_t > &local_graph, std::int32_t num_ghost_nodes, bool ghosting)> partition_fn
Signature of functions for computing the parallel partitioning of a distributed graph.
Definition: partition.h:36