XGCa
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
distribution.hpp
Go to the documentation of this file.
1 #ifndef DISTRIBUTION_HPP
2 #define DISTRIBUTION_HPP
3 
4 #include <Kokkos_Core.hpp>
5 #include "space_settings.hpp"
6 #include "globals.hpp"
7 #include "velocity_grid.hpp"
8 
9 #ifndef NO_FORTRAN
10 extern "C" double* get_f_loc();
11 extern "C" double* get_f0g_loc();
12 extern "C" double* get_temp_ev_loc();
13 extern "C" double* get_df0g_tmp_loc();
14 extern "C" double* get_delta_n_loc();
15 extern "C" double* get_delta_u_loc();
16 extern "C" double* get_delta_t_loc();
17 #endif
18 
19 template<class Device>
21 
22  public:
23 
24  // Let f view just wrap around fortran array instead of making a copy
25  // Used in collisions, but stay on the host
26  Kokkos::View<double***,Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>> f;
27  Kokkos::View<double***,Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>> df0g_tmp;
28 
29  // Used in getf0, copied to gpu so use _h to distinguish the host versions
30  // If F0_TOR_LINEAR is used, f0g has a 4th dimension of size 2 (to enable a toroidal interpolation)
31 #if defined(XGC1) && defined(F0_TOR_LINEAR)
32  Kokkos::View<double****,Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>> f0g_h;
33 #else
34  Kokkos::View<double***,Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>> f0g_h;
35 #endif
36  Kokkos::View<double*,Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>> temp_ev_h;
37 
38  // Arrays on Device:
39 #if defined(XGC1) && defined(F0_TOR_LINEAR)
40  Kokkos::View<double****,Kokkos::LayoutRight,Device> f0g;
41 #else
42  Kokkos::View<double***,Kokkos::LayoutRight,Device> f0g;
43 #endif
44  Kokkos::View<double*,Kokkos::LayoutRight,Device> temp_ev;
45 
46  // These arrays are for the entire mesh on the fortran side; could be decomposed
47  Kokkos::View<double*,Kokkos::LayoutRight,Device> delta_n;
48  Kokkos::View<double*,Kokkos::LayoutRight,Device> delta_u;
49  Kokkos::View<double*,Kokkos::LayoutRight,Device> delta_t;
50 
51  // Used in distribute_f0g
52  Kokkos::View<double*,Kokkos::LayoutRight,Device> grid_vol;
53 
54  // Used in update_f0
55  Kokkos::View<double*,Kokkos::LayoutRight,Device> inv_grid_vol;
56  Kokkos::View<double***,Kokkos::LayoutRight,Device> n;
57  Kokkos::View<double***,Kokkos::LayoutRight, Device> f_d;
58  Kokkos::View<double*,Kokkos::LayoutRight,Device> temp_ev_d;
59 
60  // Initialize these Views but don't allocate any memory yet; Instead, resize when needed
61  Distribution()
62 #if defined(XGC1) && defined(F0_TOR_LINEAR)
63  : f0g("f0g",0,0,0,0),
64 #else
65  : f0g("f0g",0,0,0),
66 #endif
67  temp_ev("temp_ev",0),
68  delta_n("delta_n",0),
69  delta_u("delta_u",0),
70  delta_t("delta_t",0),
71  grid_vol("grid_vol",0),
72  inv_grid_vol("inv_grid_vol",0),
73  temp_ev_d("temp_ev_d",0),
74  f_d("f_d",0,0,0),
75  n("n",0,0,0)
76  {}
77 
78 
81  // Allocate memory for the device arrays by resizing them
82  void resize_on_device(const VelocityGrid& vgrid, int nnode, int nnode_entire_plane){
83 #if defined(XGC1) && defined(F0_TOR_LINEAR)
84  Kokkos::resize(f0g, 2, vgrid.nvr, nnode, vgrid.nvz);
85 #else
86  Kokkos::resize(f0g, vgrid.nvr, nnode, vgrid.nvz);
87 #endif
88  Kokkos::resize(temp_ev, nnode);
89  Kokkos::resize(delta_n, nnode_entire_plane);
90  Kokkos::resize(delta_u, nnode_entire_plane);
91  Kokkos::resize(delta_t, nnode_entire_plane);
92  }
93 
94  // Deallocate memory of the device arrays by resizing back to 0
96 #if defined(XGC1) && defined(F0_TOR_LINEAR)
97  Kokkos::resize(f0g, 0,0,0,0);
98 #else
99  Kokkos::resize(f0g, 0,0,0);
100 #endif
101  Kokkos::resize(temp_ev, 0);
102  Kokkos::resize(delta_n, 0);
103  Kokkos::resize(delta_u, 0);
104  Kokkos::resize(delta_t, 0);
105  }
106 
107  // Send arrays needed for weight udpate
108  void send_to_device(double* delta_n_array,double* delta_u_array,double* delta_t_array,int isp)
109  {
110  subarray_deep_copy(delta_n, delta_n_array, isp);
111  subarray_deep_copy(delta_u, delta_u_array, isp);
112  subarray_deep_copy(delta_t, delta_t_array, isp);
113 
114  Kokkos::deep_copy(f0g, f0g_h);
115  Kokkos::deep_copy(temp_ev, temp_ev_h);
116  }
117 
120  // Allocate memory for the device arrays by resizing them
121  void resize_for_distribute_f0g_on_device(const VelocityGrid& vgrid, int nnode, int nnode_entire_plane){
122  Kokkos::resize(grid_vol, nnode);
123  Kokkos::resize(temp_ev_d, nnode);
124  }
125 
126  // Deallocate memory of the device arrays by resizing back to 0
128  Kokkos::resize(grid_vol, 0);
129  Kokkos::resize(temp_ev_d, 0);
130  }
131 
134  // Allocate memory for the device arrays by resizing them
135  void resize_for_update_f0_on_device(const VelocityGrid& vgrid, int nnode, int nnode_entire_plane){
136  Kokkos::resize(f0g, vgrid.nvr, nnode, vgrid.nvz);
137  Kokkos::resize(inv_grid_vol, nnode);
138  Kokkos::resize(f_d, vgrid.nvr, nnode, vgrid.nvz);
139  Kokkos::resize(n, vgrid.nvr, nnode, vgrid.nvz);
140  Kokkos::resize(temp_ev_d, nnode);
141  }
142 
143  // Deallocate memory of the device arrays by resizing back to 0
145  Kokkos::resize(f0g, 0,0,0);
146  Kokkos::resize(inv_grid_vol, 0);
147  Kokkos::resize(f_d, 0,0,0);
148  Kokkos::resize(n, 0,0,0);
149  Kokkos::resize(temp_ev_d, 0);
150  }
151 
152  /********************************/
153 
154  // isp here is the species index in f0, i.e. non-adiabatic species
155  void update_pointers(const VelocityGrid& vgrid, int nnode, int isp, double* f0g_ptr, double* temp_ev_ptr )
156  {
157  // Need to update pointers to fortran arrays containing grid info
158  // Since these pointers point to specific species, must add offset to get to the correct species
159  // This all goes away once the distributions are resident on the C++ side
160  int one_sp_size = vgrid.nvr*nnode*vgrid.nvz;
161  int ptr_offset = isp*one_sp_size;
162 
163 #if defined(XGC1) && defined(F0_TOR_LINEAR)
164  f0g_h = Kokkos::View<double****,Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>>
165  (f0g_ptr + ptr_offset, 2, vgrid.nvr, nnode, vgrid.nvz);
166 #else
167  f0g_h = Kokkos::View<double***,Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>>
168  (f0g_ptr + ptr_offset, vgrid.nvr, nnode, vgrid.nvz);
169 #endif
170 
171  // 1D array: temp_ev
172  one_sp_size = nnode;
173  ptr_offset = isp*one_sp_size;
174  temp_ev_h = Kokkos::View<double*,Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>>
175  (temp_ev_ptr + ptr_offset, nnode);
176  }
177 
178  // This pointer is to a temporary array allocated in fortran right before collisions, so it must be updated
179  // at the beginning of collisions
180  // isp here is the species index in f0, i.e. non-adiabatic species
181  void update_tmp_pointer(const VelocityGrid& vgrid, int nnode, int isp, double* f_ptr, double* df0g_tmp_ptr )
182  {
183  // Need to update pointers to fortran arrays containing grid info
184  // Since these pointers point to specific species, must add offset to get to the correct species
185  // This all goes away once the distributions are resident on the C++ side
186  int one_sp_size = vgrid.nvr*nnode*vgrid.nvz;
187  int ptr_offset = isp*one_sp_size;
188  f = Kokkos::View<double***,Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>>
189  (f_ptr + ptr_offset, vgrid.nvr, nnode, vgrid.nvz);
190  df0g_tmp = Kokkos::View<double***,Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>>
191  (df0g_tmp_ptr + ptr_offset, vgrid.nvr, nnode, vgrid.nvz);
192  }
193 
194 };
195 #endif
double * get_f0g_loc()
Kokkos::View< double ***, Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits< Kokkos::Unmanaged > > df0g_tmp
Definition: distribution.hpp:27
double * get_temp_ev_loc()
void resize_for_update_f0_on_device(const VelocityGrid &vgrid, int nnode, int nnode_entire_plane)
Definition: distribution.hpp:135
void send_to_device(double *delta_n_array, double *delta_u_array, double *delta_t_array, int isp)
Definition: distribution.hpp:108
Definition: velocity_grid.hpp:5
Kokkos::View< double *, Kokkos::LayoutRight, Device > delta_n
Definition: distribution.hpp:47
int nvr
Definition: velocity_grid.hpp:16
Kokkos::View< double *, Kokkos::LayoutRight, Device > temp_ev
Definition: distribution.hpp:44
n("n", 0, 0, 0)
Definition: distribution.hpp:75
Definition: distribution.hpp:20
void resize_for_update_f0_to_zero()
Definition: distribution.hpp:144
double * get_df0g_tmp_loc()
double * get_f_loc()
Kokkos::View< double *, Kokkos::LayoutRight, Device > delta_t
Definition: distribution.hpp:49
void resize_for_distribute_f0g_to_zero()
Definition: distribution.hpp:127
void resize_for_distribute_f0g_on_device(const VelocityGrid &vgrid, int nnode, int nnode_entire_plane)
Definition: distribution.hpp:121
void subarray_deep_copy(T *array, const Kokkos::View< T *, Kokkos::LayoutRight, DeviceType > &view, int isp)
Definition: array_deep_copy.hpp:102
void update_pointers(const VelocityGrid &vgrid, int nnode, int isp, double *f0g_ptr, double *temp_ev_ptr)
Definition: distribution.hpp:155
Kokkos::View< double *, Kokkos::HostSpace, Kokkos::MemoryTraits< Kokkos::Unmanaged > > temp_ev_h
Definition: distribution.hpp:36
Kokkos::View< double ***, Kokkos::LayoutRight, Device > f_d
Definition: distribution.hpp:57
Kokkos::View< double *, Kokkos::LayoutRight, Device > delta_u
Definition: distribution.hpp:48
double * get_delta_t_loc()
Kokkos::View< double ***, Kokkos::LayoutRight, Device > f0g
Definition: distribution.hpp:42
Kokkos::View< double ***, Kokkos::LayoutRight, Device > n
Definition: distribution.hpp:56
double * get_delta_u_loc()
Kokkos::View< double *, Kokkos::LayoutRight, Device > inv_grid_vol
Definition: distribution.hpp:55
Kokkos::View< double *, Kokkos::LayoutRight, Device > temp_ev_d
Definition: distribution.hpp:58
void update_tmp_pointer(const VelocityGrid &vgrid, int nnode, int isp, double *f_ptr, double *df0g_tmp_ptr)
Definition: distribution.hpp:181
int nvz
Definition: velocity_grid.hpp:17
void resize_on_device(const VelocityGrid &vgrid, int nnode, int nnode_entire_plane)
Definition: distribution.hpp:82
Kokkos::View< double ***, Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits< Kokkos::Unmanaged > > f0g_h
Definition: distribution.hpp:34
void resize_to_zero()
Definition: distribution.hpp:95
double * get_delta_n_loc()
Kokkos::View< double ***, Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits< Kokkos::Unmanaged > > f
Definition: distribution.hpp:26
Kokkos::View< double *, Kokkos::LayoutRight, Device > grid_vol
Definition: distribution.hpp:52