XGCa
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
my_mpi.hpp
Go to the documentation of this file.
1 #ifndef MY_MPI_HPP
2 #define MY_MPI_HPP
3 
4 #include <mpi.h>
5 #include <vector>
6 #include <cstdio>
7 
8 extern MPI_Comm SML_COMM_WORLD;
9 extern int SML_COMM_RANK;
10 extern int SML_COMM_SIZE;
11 
12 void create_comm_world(int color);
13 void destroy_comm_world();
14 
15 /* To illustrate the different communicators: Imagine SML_COMM_WORLD has 12 ranks:
16  * 0 1 2 3 4 5 6 7 8 9 10 11
17  */
18 
19 struct MyMPI{
20  // General communicator
21  MPI_Comm comm;
22  MPI_Group group;
23  int nranks;
24  int my_rank;
25  /* Reorders SML_COMM_WORLD e.g. (with 12 ranks, 4 planes):
26  * 0 3 6 9
27  * 1 4 7 10
28  * 2 5 8 11
29  * So e.g. processes with ranks 0 and 3 in SML_COMM_WORLD are reordered to be
30  * consecutive in comm
31  * Not sure why this transpose is done; maybe the order affects some MPI algorithms
32  * but not clear when/why this would be better.
33  * Also, in the Fortran code comm overwrites SML_COMM_WORLD in the setup (both
34  * called sml_comm), so something similar could be done here. - ALS
35  */
36 
37  // Intraplane communicator
38  MPI_Comm plane_comm;
41  /* Takes a column from comm, e.g. (with 12 ranks, 4 planes, this is rank 4):
42  * 3
43  * 4
44  * 5
45  */
46 
47  // Interplane communicator
48  MPI_Comm intpl_comm;
51  /* Takes a row from comm, e.g. (with 12 ranks, 4 planes, this is rank 4):
52  *
53  * 1 4 7 10
54  *
55  */
56 
57  MyMPI(){}
58 
59  // Spoofed MPI for dry run
60  MyMPI(int nranks_in)
61  : nranks(nranks_in),
62  my_rank(0) {}
63 
64  MyMPI(const MPI_Comm& comm_world, int nplanes){
65  // Get total number of processors
66  MPI_Comm_size( comm_world, &nranks );
67 
68  // Assumes check that nranks is divisible by nplanes has already happened - move check here?
69  int ranks_per_plane = nranks/nplanes;
70 
71  if(false/*sml_plane_major*/){
72  // get the group underlying sml_comm
73  //mpi_comm_group(sml_comm, sml_comm_group, ierr)
74  }else{
75  // redefine sml_comm pe ordering from consecutive within planes
76  // (Plane_Major) to consecutive across planes (Interplane-Major)
77  std::vector<int> new_sml_comm_ranks(nranks);
78  int k = 0;
79  for(int j=0; j<ranks_per_plane; j++){
80  for(int i=0; i<nplanes; i++){
81  new_sml_comm_ranks[ranks_per_plane*i + j] = k;
82  k++;
83  }
84  }
85 
86  // get the group underlying sml_comm (== mpi_comm_world)
87  // Use sml_comm_world instead of MPI_COMM_WORLD (for MPMD/staging execution)
88  MPI_Group mpi_comm_world_group;
89  MPI_Comm_group(comm_world, &mpi_comm_world_group);
90 
91  // create new group permuting sml_comm pe ordering to Interplane-Major
92  MPI_Group_incl(mpi_comm_world_group, nranks, new_sml_comm_ranks.data(), &group);
93 
94  // Create the new communicator
95  MPI_Comm_create(comm_world, group, &comm);
96  MPI_Comm_rank( comm, &my_rank );
97  }
98 
99  // INTRA-PLANE MPI COMMUNICATOR
100  //checkpoint("\nPlane mpi communication");
101  int plane_0_pe=int(my_rank/ranks_per_plane)*ranks_per_plane;
102  std::vector<int> plane_ranks(ranks_per_plane);
103  for(int j=0; j<ranks_per_plane; j++){
104  plane_ranks[j]=plane_0_pe + j;
105  }
106 
107  // Create the new plane group
108  MPI_Group plane_group;
109  MPI_Group_incl(group, ranks_per_plane, plane_ranks.data(), &plane_group);
110 
111  // Create the new plane communicator
112  MPI_Comm_create(comm, plane_group, &plane_comm);
113 
114  //call mpi_comm_size(plane_comm, plane_nranks, ierr)
115  MPI_Comm_rank(plane_comm, &my_plane_rank);
116  MPI_Comm_size(plane_comm, &n_plane_ranks);
117 
118  // INTER-PLANE MPI COMMUNICATOR
119  //checkpoint("\nInter-plane mpi communication");
120  std::vector<int> intpl_ranks(nplanes);
121  for(int i=0; i<nplanes; i++){
122  intpl_ranks[i]=my_plane_rank + i*ranks_per_plane;
123  }
124 
125  // Create the new inter-plane group
126  MPI_Group intpl_group;
127  MPI_Group_incl(group, nplanes, intpl_ranks.data(), &intpl_group);
128 
129  // Create the new inter-plane communicator
130  MPI_Comm_create(comm, intpl_group, &intpl_comm);
131 
132  MPI_Comm_rank(intpl_comm, &my_intpl_rank);
133  MPI_Comm_size(intpl_comm, &n_intpl_ranks );
134 
135  /*
136  call check_point('adios mpi communication?')
137  do i=0, sml_intpl_nranks-1
138  adios_ranks(i)= i*ranks_per_plane
139  enddo
140  call mpi_group_incl(sml_comm_group, sml_intpl_nranks, adios_ranks, sml_adios_group, ierr)
141  call mpi_comm_create(sml_comm, sml_adios_group, sml_adios_comm, ierr)
142 
143 
144  //#ifndef NO_TASKMAP
145  // output task-to-node mapping
146  write(c_color,'(i8)') sml_comm_color
147  if (sml_mype .eq. 0) then
148  open(unit=14, file='TASKMAP_Color'//trim(adjustl(c_color))//'.txt', &
149  status='OLD', access='SEQUENTIAL', position='APPEND' )
150  endif
151  call taskmap_write(14, sml_comm, &
152  'SML_COMM COLOR #'//trim(adjustl(c_color)), sml_plane_mype, sml_intpl_mype, .false.)
153  if (sml_mype .eq. 0) close(14)
154  //#endif
155  //#ifdef XGC1
156  //jyc print mpi placement info
157  if (sml_verbose) then
158  call mpi_get_processor_name(nodename, len, ierr)
159  print *, sml_mype, 'Process (plane,node,color):', sml_plane_index, trim(nodename), ' ', sml_comm_color
160  endif
161  //#endif
162  */
163  }
164 };
165 
166 // Some useful debugging tools
167 template <typename F>
168 void execute_in_rank_order(const MPI_Comm& comm, F func){
169  int my_rank;
170  int n_ranks;
171  MPI_Comm_rank( comm, &my_rank );
172  MPI_Comm_size( comm, &n_ranks );
173 
174  // double to make sure *every* rank is flushing at *every* barrier
175  fflush(stdout);
176  MPI_Barrier(comm);
177  fflush(stdout);
178  MPI_Barrier(comm);
179  for(int i=0; i<n_ranks; i++){
180  if(my_rank==i) func(i);
181  fflush(stdout);
182  MPI_Barrier(comm);
183  fflush(stdout);
184  MPI_Barrier(comm);
185  }
186 };
187 
188 #endif
MPI_Comm SML_COMM_WORLD
Definition: my_mpi.cpp:4
MPI_Comm plane_comm
Definition: my_mpi.hpp:38
MyMPI(const MPI_Comm &comm_world, int nplanes)
Definition: my_mpi.hpp:64
int my_intpl_rank
Definition: my_mpi.hpp:49
int my_rank
Definition: my_mpi.hpp:24
Definition: my_mpi.hpp:19
void create_comm_world(int color)
Definition: my_mpi.cpp:13
int SML_COMM_SIZE
Definition: my_mpi.cpp:6
MPI_Comm comm
Definition: my_mpi.hpp:21
int my_plane_rank
Definition: my_mpi.hpp:39
int n_plane_ranks
Definition: my_mpi.hpp:40
int SML_COMM_RANK
Definition: my_mpi.cpp:5
void execute_in_rank_order(const MPI_Comm &comm, F func)
Definition: my_mpi.hpp:168
MyMPI()
Definition: my_mpi.hpp:57
MPI_Comm intpl_comm
Definition: my_mpi.hpp:48
void destroy_comm_world()
Definition: my_mpi.cpp:28
MPI_Group group
Definition: my_mpi.hpp:22
int nranks
Definition: my_mpi.hpp:23
int n_intpl_ranks
Definition: my_mpi.hpp:50
MyMPI(int nranks_in)
Definition: my_mpi.hpp:60