XGC1
domain_decomposition.hpp
Go to the documentation of this file.
1 #ifndef DOMAIN_DECOMPOSITION_HPP
2 #define DOMAIN_DECOMPOSITION_HPP
3 #include "space_settings.hpp"
4 #include "NamelistReader.hpp"
5 #ifdef USE_MPI
6 # include "my_mpi.hpp"
7 #endif
9 #include "grid_weights.hpp"
10 #include "vertex_list.hpp"
11 
12 #include "vertex_list.hpp"
13 #include "host_array.hpp"
14 
16  View<int*,CLayout,HostType> cnts;
17  View<int*,CLayout,HostType> displs;
18  int my_rank;
19 
21 
22  DistributionPlan(int my_rank_in, int nranks)
23  : cnts(NoInit("cnts"), nranks),
24  displs(NoInit("displs"), nranks),
25  my_rank(my_rank_in)
26  {}
27 
28  int my_count() const{
29  return cnts(my_rank);
30  }
31 
32  int my_displ() const{
33  return displs(my_rank);
34  }
35 };
36 
37 // Decomposition class
38 template<class Device>
40  public:
41 
42  // Constructor
44 
45  DomainDecomposition(NLReader::NamelistReader& nlr, int nnodes_in);
46 
47  DomainDecomposition(int nranks_in);
48 
49  void setup_gvid0_pid(NLReader::NamelistReader& nlr, int nnodes_in);
50 
51  int nplanes;
52 
53 #ifdef USE_MPI
54  // MPI communicators etc
55  const MyMPI mpi;
56 #endif
57 
58  // Updater
59  void update(int new_f0_inode1);
60 
61  void update_pol_decomp();
62 
63  void update_flux_surf(const HostArray<VertexList>& surfaces);
64 
65  KOKKOS_INLINE_FUNCTION int node_to_plane_rank(int node) const;
66 
67  KOKKOS_INLINE_FUNCTION void node_to_rank(const SimdGridWeights<Order::Zero, PIT_GLOBAL>& grid_wts0, const Simd<int>& iphi, bool plane_sort_only, Simd<int>& pid) const;
68 
69  int node_to_plane_rank_h(int node, int initial_guess) const;
70 
72 
73  VertexList vertex_list() const;
74 
75  VertexList vertex_list(int ipid) const;
76 
77  // Constants
78  bool pol_decomp;
81  int n_pids;
82  int my_pid;
86 
89 
90  // Variables
92  int nnodes;
93 
94  // Views
95  Kokkos::View<int*,Kokkos::LayoutRight,HostType> gvid0_pid_h;
96  Kokkos::View<int*,Kokkos::LayoutRight,Device> gvid0_pid;
97 
98  View<int*,CLayout,HostType> surfs;
99  View<int*,CLayout,HostType> ranks_sharing_flux_surf;
100 
101 };
102 
103 #include "domain_decomposition.tpp"
104 #endif
Definition: domain_decomposition.hpp:39
int nnodes_on_plane
Number of nodes on local plane.
Definition: domain_decomposition.hpp:85
Kokkos::View< int *, Kokkos::LayoutRight, Device > gvid0_pid
Which processors get which vertices.
Definition: domain_decomposition.hpp:96
View< int *, CLayout, HostType > surfs
Which surfaces are at least partially on this rank.
Definition: domain_decomposition.hpp:98
int pe_per_plane
Processors per plane.
Definition: domain_decomposition.hpp:79
int n_pids
Processors per plane?
Definition: domain_decomposition.hpp:81
bool decompose_fields
Whether to decompose fields.
Definition: domain_decomposition.hpp:87
DistributionPlan mpi_distribution_plan(int nv) const
int nnodes
Number of nodes belonging to this MPI rank.
Definition: domain_decomposition.hpp:92
FieldDecomposition< Device > field_decomp
Definition: domain_decomposition.hpp:88
int node_to_plane_rank_h(int node, int initial_guess) const
Definition: domain_decomposition.cpp:249
int my_pid
ID of this processor.
Definition: domain_decomposition.hpp:82
void update(int new_f0_inode1)
Definition: domain_decomposition.cpp:206
int my_plane_pid
Poloidal ID of this processor.
Definition: domain_decomposition.hpp:83
bool pol_decomp
Use poloidal decomposition.
Definition: domain_decomposition.hpp:78
VertexList vertex_list() const
Definition: domain_decomposition.cpp:279
int nplanes
Definition: domain_decomposition.hpp:51
KOKKOS_INLINE_FUNCTION void node_to_rank(const SimdGridWeights< Order::Zero, PIT_GLOBAL > &grid_wts0, const Simd< int > &iphi, bool plane_sort_only, Simd< int > &pid) const
Definition: domain_decomposition.tpp:32
void setup_gvid0_pid(NLReader::NamelistReader &nlr, int nnodes_in)
Definition: domain_decomposition.cpp:68
KOKKOS_INLINE_FUNCTION int node_to_plane_rank(int node) const
Definition: domain_decomposition.tpp:14
void update_pol_decomp()
Definition: domain_decomposition.cpp:226
Kokkos::View< int *, Kokkos::LayoutRight, HostType > gvid0_pid_h
Which processors get which vertices (host)
Definition: domain_decomposition.hpp:95
void update_flux_surf(const HostArray< VertexList > &surfaces)
Definition: domain_decomposition.cpp:115
DomainDecomposition(NLReader::NamelistReader &nlr)
Definition: domain_decomposition.cpp:21
int plane_totalpe
Processors per plane?
Definition: domain_decomposition.hpp:80
View< int *, CLayout, HostType > ranks_sharing_flux_surf
Which processors share a flux surface with this one.
Definition: domain_decomposition.hpp:99
int plane_index
Offset of local plane.
Definition: domain_decomposition.hpp:84
int node_offset
Offset of first mesh node belonging to this MPI rank.
Definition: domain_decomposition.hpp:91
Definition: field_decomposition.hpp:12
Definition: NamelistReader.hpp:193
Definition: vertex_list.hpp:53
Kokkos::ViewAllocateWithoutInitializing NoInit
Definition: space_settings.hpp:69
Definition: domain_decomposition.hpp:15
int my_displ() const
Definition: domain_decomposition.hpp:32
View< int *, CLayout, HostType > displs
Definition: domain_decomposition.hpp:17
View< int *, CLayout, HostType > cnts
Definition: domain_decomposition.hpp:16
DistributionPlan(int my_rank_in, int nranks)
Definition: domain_decomposition.hpp:22
int my_count() const
Definition: domain_decomposition.hpp:28
int my_rank
Definition: domain_decomposition.hpp:18
DistributionPlan()
Definition: domain_decomposition.hpp:20
Definition: my_mpi.hpp:19
Definition: grid_weights.hpp:47