XGCa
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
distribution.hpp
Go to the documentation of this file.
1 #ifndef DISTRIBUTION_HPP
2 #define DISTRIBUTION_HPP
3 
4 #include <Kokkos_Core.hpp>
5 #include "space_settings.hpp"
6 #include "globals.hpp"
7 #include "velocity_grid.hpp"
8 
9 // Functions to fetch fortran array addresses
10 extern "C" double* get_f0g_loc();
11 extern "C" double* get_temp_ev_loc();
12 extern "C" double* get_delta_n_loc();
13 extern "C" double* get_delta_u_loc();
14 extern "C" double* get_delta_t_loc();
15 
16 template<class Device>
18 
19  public:
20 
21  // Used in collisions, but stay on the host
22  // Let f view just wrap around fortran array instead of making a copy
23  Kokkos::View<double***,Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>> f;
24 
25  // Used in getf0, copied to gpu so use _h to distinguish the host versions
26  Kokkos::View<double***,Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>> f0g_h;
27  Kokkos::View<double*,Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>> temp_ev_h;
28 
29  // Arrays on Device:
30  Kokkos::View<double***,Kokkos::LayoutRight,Device> f0g;
31  Kokkos::View<double*,Kokkos::LayoutRight,Device> temp_ev;
32 
33  // These arrays are for the entire mesh on the fortran side; could be decomposed
34  Kokkos::View<double*,Kokkos::LayoutRight,Device> delta_n;
35  Kokkos::View<double*,Kokkos::LayoutRight,Device> delta_u;
36  Kokkos::View<double*,Kokkos::LayoutRight,Device> delta_t;
37 
38  // Used in distribute_f0g
39  Kokkos::View<double*,Kokkos::LayoutRight,Device> grid_vol;
40 
41  // Used in update_f0
42  Kokkos::View<double*,Kokkos::LayoutRight,Device> inv_grid_vol;
43  Kokkos::View<double***,Kokkos::LayoutRight,Device> n;
44  Kokkos::View<double***,Kokkos::LayoutRight, Device> f_d;
45  Kokkos::View<double*,Kokkos::LayoutRight,Device> temp_ev_d;
46 
47  // Initialize these Views but don't allocate any memory yet; Instead, resize when needed
49  : f0g("f0g",0,0,0),
50  temp_ev("temp_ev",0),
51  delta_n("delta_n",0),
52  delta_u("delta_u",0),
53  delta_t("delta_t",0),
54  grid_vol("grid_vol",0),
55  inv_grid_vol("inv_grid_vol",0),
56  temp_ev_d("temp_ev_d",0),
57  f_d("f_d",0,0,0),
58  n("n",0,0,0)
59  {}
60 
61 
64  // Allocate memory for the device arrays by resizing them
65  void resize_on_device(const VelocityGrid& vgrid, int nnode, int nnode_entire_plane){
66  Kokkos::resize(f0g, vgrid.nvr, nnode, vgrid.nvz);
67  Kokkos::resize(temp_ev, nnode);
68  Kokkos::resize(delta_n, nnode_entire_plane);
69  Kokkos::resize(delta_u, nnode_entire_plane);
70  Kokkos::resize(delta_t, nnode_entire_plane);
71  }
72 
73  // Deallocate memory of the device arrays by resizing back to 0
75  Kokkos::resize(f0g, 0,0,0);
76  Kokkos::resize(temp_ev, 0);
77  Kokkos::resize(delta_n, 0);
78  Kokkos::resize(delta_u, 0);
79  Kokkos::resize(delta_t, 0);
80  }
81 
82  // Send arrays needed for weight udpate
83  void send_to_device(double* delta_n_array,double* delta_u_array,double* delta_t_array,int isp)
84  {
85  subarray_deep_copy(delta_n, delta_n_array, isp);
86  subarray_deep_copy(delta_u, delta_u_array, isp);
87  subarray_deep_copy(delta_t, delta_t_array, isp);
88 
89  Kokkos::deep_copy(f0g, f0g_h);
90  Kokkos::deep_copy(temp_ev, temp_ev_h);
91  }
92 
95  // Allocate memory for the device arrays by resizing them
96  void resize_for_distribute_f0g_on_device(const VelocityGrid& vgrid, int nnode){
97  Kokkos::resize(grid_vol, nnode);
98  Kokkos::resize(temp_ev_d, nnode);
99  }
100 
101  // Deallocate memory of the device arrays by resizing back to 0
103  Kokkos::resize(grid_vol, 0);
104  Kokkos::resize(temp_ev_d, 0);
105  }
106 
109  // Allocate memory for the device arrays by resizing them
110  void resize_for_update_f0_on_device(const VelocityGrid& vgrid, int nnode){
111  Kokkos::resize(f0g, vgrid.nvr, nnode, vgrid.nvz);
112  Kokkos::resize(inv_grid_vol, nnode);
113  Kokkos::resize(f_d, vgrid.nvr, nnode, vgrid.nvz);
114  Kokkos::resize(n, vgrid.nvr, nnode, vgrid.nvz);
115  Kokkos::resize(temp_ev_d, nnode);
116  }
117 
118  // Deallocate memory of the device arrays by resizing back to 0
120  Kokkos::resize(f0g, 0,0,0);
121  Kokkos::resize(inv_grid_vol, 0);
122  Kokkos::resize(f_d, 0,0,0);
123  Kokkos::resize(n, 0,0,0);
124  Kokkos::resize(temp_ev_d, 0);
125  }
126 
127  /********************************/
128 
129  // isp here is the species index in f0, i.e. non-adiabatic species
130  void update_pointers(const VelocityGrid& vgrid, int nnode, int isp, double* f0g_ptr, double* temp_ev_ptr )
131  {
132  // Need to update pointers to fortran arrays containing grid info
133  // Since these pointers point to specific species, must add offset to get to the correct species
134  // This all goes away once the distributions are resident on the C++ side
135  int one_sp_size = vgrid.nvr*nnode*vgrid.nvz;
136  int ptr_offset = isp*one_sp_size;
137 
138  f0g_h = Kokkos::View<double***,Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>>
139  (f0g_ptr + ptr_offset, vgrid.nvr, nnode, vgrid.nvz);
140 
141  // 1D array: temp_ev
142  one_sp_size = nnode;
143  ptr_offset = isp*one_sp_size;
144  temp_ev_h = Kokkos::View<double*,Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>>
145  (temp_ev_ptr + ptr_offset, nnode);
146  }
147 
148  // This pointer is to a temporary array allocated in fortran right before collisions, so it must be updated
149  // at the beginning of collisions
150  // isp here is the species index in f0, i.e. non-adiabatic species
151  void update_tmp_pointer(const VelocityGrid& vgrid, int nnode, int isp, double* f_ptr)
152  {
153  // Need to update pointers to fortran arrays containing grid info
154  // Since these pointers point to specific species, must add offset to get to the correct species
155  // This all goes away once the distributions are resident on the C++ side
156  int one_sp_size = vgrid.nvr*nnode*vgrid.nvz;
157  int ptr_offset = isp*one_sp_size;
158  f = Kokkos::View<double***,Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>>
159  (f_ptr + ptr_offset, vgrid.nvr, nnode, vgrid.nvz);
160  }
161 
162 };
163 #endif
double * get_f0g_loc()
double * get_temp_ev_loc()
void send_to_device(double *delta_n_array, double *delta_u_array, double *delta_t_array, int isp)
Definition: distribution.hpp:83
Definition: velocity_grid.hpp:5
Kokkos::View< double *, Kokkos::LayoutRight, Device > delta_n
Definition: distribution.hpp:34
int nvr
full grid size (including zero)
Definition: velocity_grid.hpp:16
Kokkos::View< double *, Kokkos::LayoutRight, Device > temp_ev
Definition: distribution.hpp:31
void resize_for_update_f0_on_device(const VelocityGrid &vgrid, int nnode)
Definition: distribution.hpp:110
Definition: distribution.hpp:17
void resize_for_update_f0_to_zero()
Definition: distribution.hpp:119
Kokkos::View< double *, Kokkos::LayoutRight, Device > delta_t
Definition: distribution.hpp:36
void resize_for_distribute_f0g_to_zero()
Definition: distribution.hpp:102
void subarray_deep_copy(T *array, const Kokkos::View< T *, Kokkos::LayoutRight, DeviceType > &view, int isp)
Definition: array_deep_copy.hpp:102
void update_pointers(const VelocityGrid &vgrid, int nnode, int isp, double *f0g_ptr, double *temp_ev_ptr)
Definition: distribution.hpp:130
Kokkos::View< double *, Kokkos::HostSpace, Kokkos::MemoryTraits< Kokkos::Unmanaged > > temp_ev_h
Definition: distribution.hpp:27
Kokkos::View< double ***, Kokkos::LayoutRight, Device > f_d
Definition: distribution.hpp:44
Kokkos::View< double *, Kokkos::LayoutRight, Device > delta_u
Definition: distribution.hpp:35
Distribution()
Definition: distribution.hpp:48
double * get_delta_t_loc()
Kokkos::View< double ***, Kokkos::LayoutRight, Device > f0g
Definition: distribution.hpp:30
void resize_for_distribute_f0g_on_device(const VelocityGrid &vgrid, int nnode)
Definition: distribution.hpp:96
Kokkos::View< double ***, Kokkos::LayoutRight, Device > n
Definition: distribution.hpp:43
double * get_delta_u_loc()
Kokkos::View< double *, Kokkos::LayoutRight, Device > inv_grid_vol
Definition: distribution.hpp:42
void update_tmp_pointer(const VelocityGrid &vgrid, int nnode, int isp, double *f_ptr)
Definition: distribution.hpp:151
Kokkos::View< double *, Kokkos::LayoutRight, Device > temp_ev_d
Definition: distribution.hpp:45
int nvz
full grid size (including negative and zero)
Definition: velocity_grid.hpp:17
void resize_on_device(const VelocityGrid &vgrid, int nnode, int nnode_entire_plane)
Definition: distribution.hpp:65
Kokkos::View< double ***, Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits< Kokkos::Unmanaged > > f0g_h
Definition: distribution.hpp:26
void resize_to_zero()
Definition: distribution.hpp:74
double * get_delta_n_loc()
Kokkos::View< double ***, Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits< Kokkos::Unmanaged > > f
Definition: distribution.hpp:23
Kokkos::View< double *, Kokkos::LayoutRight, Device > grid_vol
Definition: distribution.hpp:39