XGCa
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
distribution.hpp
Go to the documentation of this file.
1 #ifndef DISTRIBUTION_HPP
2 #define DISTRIBUTION_HPP
3 
4 #include <Kokkos_Core.hpp>
5 #include "space_settings.hpp"
6 #include "globals.hpp"
7 #include "velocity_grid.hpp"
8 
9 // Functions to fetch fortran array addresses
10 extern "C" double* get_delta_n_loc();
11 extern "C" double* get_delta_u_loc();
12 extern "C" double* get_delta_t_loc();
13 
14 template<class Device>
15 struct Distribution{
16 
17  // Used in getf0, copied to gpu so use _h to distinguish the host versions
18  Kokkos::View<double***,Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>> f0g_h;
19  Kokkos::View<double*,Kokkos::HostSpace, Kokkos::MemoryTraits<Kokkos::Unmanaged>> temp_ev_h;
20 
21  View<double*,CLayout, HostType, Kokkos::MemoryTraits<Kokkos::Unmanaged>> den_global_h;
22  View<double*,CLayout, HostType, Kokkos::MemoryTraits<Kokkos::Unmanaged>> temp_global_h;
23 
24  // Arrays on Device:
25  Kokkos::View<double***,Kokkos::LayoutRight,Device> f0g;
26  Kokkos::View<double*,Kokkos::LayoutRight,Device> temp_ev;
27 
28  // These arrays are for the entire mesh on the fortran side; could be decomposed
29  Kokkos::View<double*,Kokkos::LayoutRight,Device> delta_n;
30  Kokkos::View<double*,Kokkos::LayoutRight,Device> delta_u;
31  Kokkos::View<double*,Kokkos::LayoutRight,Device> delta_t;
32 
33  // Used in distribute_f0g
34  Kokkos::View<double*,Kokkos::LayoutRight,Device> grid_vol;
35 
36  // Used in update_f0
37  Kokkos::View<double*,Kokkos::LayoutRight,Device> inv_grid_vol;
38  Kokkos::View<double***,Kokkos::LayoutRight,Device> n;
39  Kokkos::View<double***,Kokkos::LayoutRight, Device> f_d;
40  Kokkos::View<double*,Kokkos::LayoutRight,Device> temp_ev_d;
41 
42  // Initialize these Views but don't allocate any memory yet; Instead, resize when needed
44  : f0g("f0g",0,0,0),
45  temp_ev("temp_ev",0),
46  delta_n("delta_n",0),
47  delta_u("delta_u",0),
48  delta_t("delta_t",0),
49  grid_vol("grid_vol",0),
50  inv_grid_vol("inv_grid_vol",0),
51  temp_ev_d("temp_ev_d",0),
52  f_d("f_d",0,0,0),
53  n("n",0,0,0)
54  {}
55 
56 
59  // Allocate memory for the device arrays by resizing them
60  void resize_on_device(const VelocityGrid& vgrid, int nnode, int nnode_entire_plane){
61  Kokkos::resize(f0g, vgrid.nvr, nnode, vgrid.nvz);
62  Kokkos::resize(temp_ev, nnode);
63  Kokkos::resize(delta_n, nnode_entire_plane);
64  Kokkos::resize(delta_u, nnode_entire_plane);
65  Kokkos::resize(delta_t, nnode_entire_plane);
66  }
67 
68  // Deallocate memory of the device arrays by resizing back to 0
70  Kokkos::resize(f0g, 0,0,0);
71  Kokkos::resize(temp_ev, 0);
72  Kokkos::resize(delta_n, 0);
73  Kokkos::resize(delta_u, 0);
74  Kokkos::resize(delta_t, 0);
75  }
76 
77  // Send arrays needed for weight udpate
78  void send_to_device(double* delta_n_array,double* delta_u_array,double* delta_t_array,int isp)
79  {
80  subarray_deep_copy(delta_n, delta_n_array, isp);
81  subarray_deep_copy(delta_u, delta_u_array, isp);
82  subarray_deep_copy(delta_t, delta_t_array, isp);
83 
84  Kokkos::deep_copy(f0g, f0g_h);
85  Kokkos::deep_copy(temp_ev, temp_ev_h);
86  }
87 
90  // Allocate memory for the device arrays by resizing them
91  void resize_for_distribute_f0g_on_device(const VelocityGrid& vgrid, int nnode){
92  Kokkos::resize(grid_vol, nnode);
93  Kokkos::resize(temp_ev_d, nnode);
94  }
95 
96  // Deallocate memory of the device arrays by resizing back to 0
98  Kokkos::resize(grid_vol, 0);
99  Kokkos::resize(temp_ev_d, 0);
100  }
101 
104  // Allocate memory for the device arrays by resizing them
105  void resize_for_update_f0_on_device(const VelocityGrid& vgrid, int nnode){
106  Kokkos::resize(f0g, vgrid.nvr, nnode, vgrid.nvz);
107  Kokkos::resize(inv_grid_vol, nnode);
108  Kokkos::resize(f_d, vgrid.nvr, nnode, vgrid.nvz);
109  Kokkos::resize(n, vgrid.nvr, nnode, vgrid.nvz);
110  Kokkos::resize(temp_ev_d, nnode);
111  }
112 
113  // Deallocate memory of the device arrays by resizing back to 0
115  Kokkos::resize(f0g, 0,0,0);
116  Kokkos::resize(inv_grid_vol, 0);
117  Kokkos::resize(f_d, 0,0,0);
118  Kokkos::resize(n, 0,0,0);
119  Kokkos::resize(temp_ev_d, 0);
120  }
121 };
122 #endif
Kokkos::View< double *, Kokkos::LayoutRight, Device > temp_ev
Definition: distribution.hpp:26
Kokkos::View< double *, Kokkos::LayoutRight, Device > temp_ev_d
Definition: distribution.hpp:40
Definition: velocity_grid.hpp:7
Distribution()
Definition: distribution.hpp:43
Kokkos::View< double *, Kokkos::LayoutRight, Device > delta_n
Definition: distribution.hpp:29
int nvr
full grid size (including zero)
Definition: velocity_grid.hpp:18
Kokkos::View< double ***, Kokkos::LayoutRight, Device > n
Definition: distribution.hpp:38
void subarray_deep_copy(T *array, const Kokkos::View< T *, Kokkos::LayoutRight, DeviceType > &view, int isp)
Definition: array_deep_copy.hpp:102
Kokkos::View< double *, Kokkos::LayoutRight, Device > delta_u
Definition: distribution.hpp:30
void resize_on_device(const VelocityGrid &vgrid, int nnode, int nnode_entire_plane)
Definition: distribution.hpp:60
View< double *, CLayout, HostType, Kokkos::MemoryTraits< Kokkos::Unmanaged > > temp_global_h
Definition: distribution.hpp:22
Kokkos::View< double ***, Kokkos::LayoutRight, Device > f_d
Definition: distribution.hpp:39
View< double *, CLayout, HostType, Kokkos::MemoryTraits< Kokkos::Unmanaged > > den_global_h
Definition: distribution.hpp:21
double * get_delta_t_loc()
void resize_for_update_f0_on_device(const VelocityGrid &vgrid, int nnode)
Definition: distribution.hpp:105
void resize_for_distribute_f0g_to_zero()
Definition: distribution.hpp:97
Kokkos::View< double *, Kokkos::HostSpace, Kokkos::MemoryTraits< Kokkos::Unmanaged > > temp_ev_h
Definition: distribution.hpp:19
Kokkos::View< double *, Kokkos::LayoutRight, Device > delta_t
Definition: distribution.hpp:31
double * get_delta_u_loc()
Kokkos::View< double *, Kokkos::LayoutRight, Device > grid_vol
Definition: distribution.hpp:34
Kokkos::View< double *, Kokkos::LayoutRight, Device > inv_grid_vol
Definition: distribution.hpp:37
void resize_for_update_f0_to_zero()
Definition: distribution.hpp:114
int nvz
full grid size (including negative and zero)
Definition: velocity_grid.hpp:19
void resize_to_zero()
Definition: distribution.hpp:69
Kokkos::View< double ***, Kokkos::LayoutRight, Device > f0g
Definition: distribution.hpp:25
double * get_delta_n_loc()
Definition: distribution.hpp:15
Kokkos::View< double ***, Kokkos::LayoutRight, Kokkos::HostSpace, Kokkos::MemoryTraits< Kokkos::Unmanaged > > f0g_h
Definition: distribution.hpp:18
void resize_for_distribute_f0g_on_device(const VelocityGrid &vgrid, int nnode)
Definition: distribution.hpp:91
void send_to_device(double *delta_n_array, double *delta_u_array, double *delta_t_array, int isp)
Definition: distribution.hpp:78