XGCa
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
electric_field.hpp
Go to the documentation of this file.
1 #ifndef ELECTRIC_FIELD_HPP
2 #define ELECTRIC_FIELD_HPP
3 #include "space_settings.hpp"
4 #include "NamelistReader.hpp"
5 #include "grid.hpp"
6 #include "linear_weights.hpp"
7 #include "local_fields.hpp"
8 
9 //Struct used for E_phi_ff, dAs_phi_ff, dAh_phi_ff, As_phi_ff, Ah_phi_ff. 3 vector components, 2 contributing planes
11  double V[3][2];
12 };
13 
14 //Struct used for E00_ff. 2 vector components, 2 contributing planes
16  double V[2][2];
17 };
18 
19 //Struct used for ddpotdt_phi, As_phi_ff, Ah_phi_ff, Ah_cv for EM. 2 contributing planes
21  double S[2];
22 };
23 
24 // Struct used for E_rho. 2 vector components (r,z) --> can be combined to ScalarFieldPlanes
25 struct FieldXGCa {
26  double E[2];
27 };
28 
29 // Fortran calls
30 #ifdef XGC1
31 extern "C" void set_rho_ff_pointers(int nrho, int n, double* dpot, ScalarFieldPlanes* dpot_ff, ScalarFieldPlanes* pot_rho_ff, VectorFieldPlanes* E_rho_ff
32 # ifdef EXPLICIT_EM
33  , double* Ah, ScalarFieldPlanes* Ah_rho_ff, VectorFieldPlanes* dAh_rho_ff, double* As, ScalarFieldPlanes* As_rho_ff, VectorFieldPlanes* dAs_rho_ff, ScalarFieldPlanes* E_para_em_rho_ff
34 # endif
35  );
36 #else
37 extern "C" void set_rho_pointers(int nrho, int n, double* dpot, FieldXGCa* E_rho);
38 #endif
39 
40 enum class GridFieldOpts{
41  WithBothFF = 0,
44 };
45 
46 // Struct for each field that contains different variations of the data: global vs one plane, field-following vs not, maybe host vs device, maybe XGCA vs XGC1
47 template<typename T>
48 struct GridField{
49 #ifdef XGC1
50  Kokkos::View<T**,Kokkos::LayoutRight,HostType> ff_h;
51  Kokkos::View<T**,Kokkos::LayoutRight,HostType> phi_ff_h;
52  Kokkos::View<T**,Kokkos::LayoutRight,HostType> rho_ff_h;
53 
54  GridField(int nphi, int nrho, int nnode, GridFieldOpts opt=GridFieldOpts::WithBothFF) {
56  rho_ff_h = Kokkos::View<T**,Kokkos::LayoutRight,HostType>("rho_ff_h",nnode,nrho+1);
57  }
59  phi_ff_h = Kokkos::View<T**,Kokkos::LayoutRight,HostType>("phi_ff_h",nphi,nnode);
60  }
61  }
62 #else
63  Kokkos::View<T**,Kokkos::LayoutRight,HostType> rho_h;
64 
65  GridField(int nrho, int nnode)
66  : rho_h("rho_h",nnode, nrho+1) {}
67 #endif
68 };
69 
70 // Electric field class
71 template<class Device>
72 struct ElectricField {
73 
74  // Dimensions
75  int nnode;
76  int nphi;
77  int nrho;
78 
79  // Field variables
80  bool turb_efield;
81 
82  // Host Views
83 #ifdef XGC1
86  Kokkos::View<double**,Kokkos::LayoutRight,HostType> dpot_h;
87  Kokkos::View<ScalarFieldPlanes*,Kokkos::LayoutRight,HostType> dpot_ff_h;
88 #ifdef DELTAF_CONV
89  Kokkos::View<Vector2DFieldPlanes*,Kokkos::LayoutRight,HostType> E00_ff_h;
90  GridField<ScalarFieldPlanes> ddpotdt; // Not field-following?
91 #endif
92 #ifdef EXPLICIT_EM
93  //For EM
94  Kokkos::View<double**,Kokkos::LayoutRight,HostType> Ah_h;
97  Kokkos::View<double**,Kokkos::LayoutRight,HostType> As_h;
102 
103  //Kokkos::View<double*,Kokkos::LayoutRight,HostType> dpot_n0_h;
104 #endif
105 #else
107  Kokkos::View<double*,Kokkos::LayoutRight,HostType> dpot_h;
108 #endif
109 
110  // Device Views
111 #ifdef XGC1
112  Kokkos::View<VectorFieldPlanes**,Kokkos::LayoutRight,Device> E_phi_ff;
113  Kokkos::View<VectorFieldPlanes**,Kokkos::LayoutRight,Device> E_rho_ff;
114  Kokkos::View<ScalarFieldPlanes*,Kokkos::LayoutRight,Device> dpot_ff;
115 #ifdef DELTAF_CONV
116  Kokkos::View<Vector2DFieldPlanes*,Kokkos::LayoutRight,Device> E00_ff;
117  Kokkos::View<ScalarFieldPlanes**,Kokkos::LayoutRight,Device> ddpotdt_phi;
118 #endif
119 #ifdef EXPLICIT_EM
120  //For EM
121  Kokkos::View<VectorFieldPlanes**,Kokkos::LayoutRight,Device> dAh_phi_ff;
122  Kokkos::View<VectorFieldPlanes**,Kokkos::LayoutRight,Device> dAh_rho_ff;
123  Kokkos::View<ScalarFieldPlanes**,Kokkos::LayoutRight,Device> Ah_phi_ff;
124  Kokkos::View<ScalarFieldPlanes**,Kokkos::LayoutRight,Device> Ah_rho_ff;
125  Kokkos::View<VectorFieldPlanes**,Kokkos::LayoutRight,Device> dAs_phi_ff;
126  Kokkos::View<VectorFieldPlanes**,Kokkos::LayoutRight,Device> dAs_rho_ff;
127  Kokkos::View<ScalarFieldPlanes**,Kokkos::LayoutRight,Device> As_phi_ff;
128  Kokkos::View<ScalarFieldPlanes**,Kokkos::LayoutRight,Device> As_rho_ff;
129  Kokkos::View<ScalarFieldPlanes**,Kokkos::LayoutRight,Device> Ah_cv_phi_ff;
130  Kokkos::View<ScalarFieldPlanes**,Kokkos::LayoutRight,Device> Epar_em_phi_ff;
131  Kokkos::View<ScalarFieldPlanes**,Kokkos::LayoutRight,Device> Epar_em_rho_ff;
132 
133  //Kokkos::View<double*,Kokkos::LayoutRight,Device> dpot_n0;
134 #endif
135 #else
136  Kokkos::View<FieldXGCa**,Kokkos::LayoutRight,Device> E_rho;
137  Kokkos::View<double*,Kokkos::LayoutRight,Device> dpot;
138 #endif
139 
140  // Default constructor
142 
143  // Makes sense to use the Grid in the constructor here, but the Grid still copies from fortran so until that changes,
144  // it's simpler to not use it here so that ElectricField can be defined before the restart, which uses psn%As
145  ElectricField(NLReader::NamelistReader& nlr, int nnode_in, int nplanes_in, int nrho_in) : //Grid<DeviceType>& grid) :
146  /*nnode(grid.nnode),
147  nphi(grid.nplanes),
148  nrho(grid.nrho),*/
149  nnode(nnode_in),
150  nphi(nplanes_in),
151  nrho(nrho_in),
152  // Host Views (only phi_ff during this intermediate PR)
153 #ifdef XGC1
155  E(nphi,nrho,nnode),
156 # ifdef DELTAF_CONV
157  E00_ff_h("E00_ff_h",nnode),
159 # endif
160 # ifdef EXPLICIT_EM
161  dAs(nphi,nrho,nnode),
162  dAh(nphi,nrho,nnode),
163  As(nphi,nrho,nnode),
164  Ah(nphi,nrho,nnode),
166  Epar_em(nphi,nrho,nnode),
167  Ah_h("Ah_h",4,nnode), // -1:2 in fortran
168  As_h("As_h",4,nnode), // -1:2 in fortran
169 # endif
170  dpot_h("dpot_h",4,nnode), // -1:2 in fortran
171  dpot_ff_h("dpot_ff_h",nnode),
172 #else
173  E(nrho, nnode),
174  dpot_h("dpot_h",nnode),
175 #endif
176 
177  // Device Views
178 #ifdef XGC1
179  E_phi_ff("E_phi_ff",0,0),
180  E_rho_ff("E_rho_ff",0,0),
181 # ifdef DELTAF_CONV
182  E00_ff("E00_ff",0),
183  ddpotdt_phi("ddpotdt_phi",0,0),
184 # endif
185 # ifdef EXPLICIT_EM
186  dAs_phi_ff("dAs_phi_ff",0,0),
187  dAs_rho_ff("dAs_rho_ff",0,0),
188  dAh_phi_ff("dAh_phi_ff",0,0),
189  dAh_rho_ff("dAh_rho_ff",0,0),
190  As_phi_ff("As_phi_ff",0,0),
191  As_rho_ff("As_rho_ff",0,0),
192  Ah_phi_ff("Ah_phi_ff",0,0),
193  Ah_rho_ff("Ah_rho_ff",0,0),
194  Ah_cv_phi_ff("Ah_cv_phi_ff",0,0),
195  Epar_em_phi_ff("Epar_em_phi_ff",0,0),
196  Epar_em_rho_ff("Epar_em_rho_ff",0,0),
197  //dpot_n0("dpot_n0",nnode),
198 # endif
199  dpot_ff("dpot_ff",0)
200 #else
201  E_rho("E_rho",0,0),
202  dpot("dpot",0)
203 #endif
204  {
205  nlr.use_namelist("sml_param");
206  turb_efield = nlr.get<bool>("sml_turb_efield",true);
207 #ifdef XGC1
208  // Set pointers in Fortran to rho_ff arrays
209  set_rho_ff_pointers(nrho, nnode, dpot_h.data(), dpot_ff_h.data(), pot.rho_ff_h.data(), E.rho_ff_h.data()
210 # ifdef EXPLICIT_EM
211  , Ah_h.data(), Ah.rho_ff_h.data(), dAh.rho_ff_h.data(), As_h.data(), As.rho_ff_h.data(), dAs.rho_ff_h.data(), Epar_em.rho_ff_h.data()
212 # endif
213  );
214 #else
215  set_rho_pointers(nrho, nnode, dpot_h.data(), E.rho_h.data());
216 #endif
217  }
218 
219 #ifdef XGC1
220  // Resizes allocations for large arrays that have a phi dimension (i.e used for electron push)
221  void copy_phi_fields_to_device(){
222  Kokkos::resize(E_phi_ff,nphi, nnode);
223  Kokkos::deep_copy(E_phi_ff, E.phi_ff_h);
224 # ifdef DELTAF_CONV
225  Kokkos::resize(ddpotdt_phi,nphi,nnode);
226  Kokkos::deep_copy(ddpotdt_phi, ddpotdt.phi_ff_h);
227  Kokkos::resize(E00_ff, nnode);
228  Kokkos::deep_copy(E00_ff, E00_ff_h);
229 # endif
230 # ifdef EXPLICIT_EM
231  Kokkos::resize(dAs_phi_ff,nphi,nnode);
232  Kokkos::resize(dAh_phi_ff,nphi,nnode);
233  Kokkos::resize(As_phi_ff,nphi,nnode);
234  Kokkos::resize(Ah_phi_ff,nphi,nnode);
235  Kokkos::resize(Ah_cv_phi_ff,nphi,nnode);
236  Kokkos::resize(Epar_em_phi_ff,nphi,nnode);
237  Kokkos::deep_copy(dAs_phi_ff, dAs.phi_ff_h);
238  Kokkos::deep_copy(dAh_phi_ff, dAh.phi_ff_h);
239  Kokkos::deep_copy(As_phi_ff, As.phi_ff_h);
240  Kokkos::deep_copy(Ah_phi_ff, Ah.phi_ff_h);
241  Kokkos::deep_copy(Ah_cv_phi_ff, Ah_cv.phi_ff_h);
242  Kokkos::deep_copy(Epar_em_phi_ff, Epar_em.phi_ff_h);
243 # endif
244  }
245 
246  // Resizes allocations that have a phi dimension to 0, to make room for other data on the device
247  void deallocate_phi_fields_on_device(){
248  Kokkos::resize(E_phi_ff,0,0);
249 # ifdef DELTAF_CONV
250  Kokkos::resize(ddpotdt_phi,0,0);
251  Kokkos::resize(E00_ff, 0);
252 # endif
253 # ifdef EXPLICIT_EM
254  Kokkos::resize(dAs_phi_ff,0,0);
255  Kokkos::resize(dAh_phi_ff,0,0);
256  Kokkos::resize(As_phi_ff,0,0);
257  Kokkos::resize(Ah_phi_ff,0,0);
258  Kokkos::resize(Ah_cv_phi_ff,0,0);
259  Kokkos::resize(Epar_em_phi_ff,0,0);
260 # endif
261  }
262 
263  // Resizes allocations for large arrays that have a rho dimension (i.e used for ions)
265  Kokkos::resize(E_rho_ff, nnode,nrho+1);
266  Kokkos::deep_copy(E_rho_ff, E.rho_ff_h);
267 # ifdef EXPLICIT_EM
268  Kokkos::resize(dAs_rho_ff,nnode,nrho+1);
269  Kokkos::resize(dAh_rho_ff,nnode,nrho+1);
270  Kokkos::resize(As_rho_ff,nnode,nrho+1);
271  Kokkos::resize(Ah_rho_ff,nnode,nrho+1);
272  Kokkos::resize(Epar_em_rho_ff,nnode,nrho+1);
273  Kokkos::deep_copy(dAs_rho_ff, dAs.rho_ff_h);
274  Kokkos::deep_copy(dAh_rho_ff, dAh.rho_ff_h);
275  Kokkos::deep_copy(As_rho_ff, As.rho_ff_h);
276  Kokkos::deep_copy(Ah_rho_ff, Ah.rho_ff_h);
277  Kokkos::deep_copy(Epar_em_rho_ff, Epar_em.rho_ff_h);
278 # endif
279  }
280 
281  // Resizes allocations that have a rho dimension to 0, to make room for other data on the device
283  Kokkos::resize(E_rho_ff,0,0);
284 # ifdef EXPLICIT_EM
285  Kokkos::resize(dAs_rho_ff,0,0);
286  Kokkos::resize(dAh_rho_ff,0,0);
287  Kokkos::resize(As_rho_ff,0,0);
288  Kokkos::resize(Ah_rho_ff,0,0);
289  Kokkos::resize(Epar_em_rho_ff,0,0);
290 # endif
291  }
292 
293 # ifdef EXPLICIT_EM
294  /* Set of functions used to allocate/deallocate gpu memory for Ah during particle scatters */
295  // Resizes allocation of Ah_cv
296  void copy_Ah_cv_to_device(){
297  Kokkos::resize(Ah_cv_phi_ff,nphi,nnode);
298  Kokkos::deep_copy(Ah_cv_phi_ff, Ah_cv.phi_ff_h);
299  }
300 
301  // Resizes Ah_cv to 0
302  void deallocate_Ah_cv_on_device(){
303  Kokkos::resize(Ah_cv_phi_ff,0,0);
304  }
305 
306  void copy_Ah_phi_ff_to_device(){
307  Kokkos::resize(Ah_phi_ff,nphi,nnode);
308  Kokkos::deep_copy(Ah_phi_ff, Ah.phi_ff_h);
309  }
310 
311  // Resizes Ah_phi_ff to 0
312  void deallocate_Ah_phi_ff_on_device(){
313  Kokkos::resize(Ah_phi_ff,0,0);
314  }
315 
316  // Resizes allocation of Ah_rho_ff
317  void copy_Ah_rho_ff_to_device(){
318  Kokkos::resize(Ah_rho_ff,nnode,nrho+1);
319  Kokkos::deep_copy(Ah_rho_ff, Ah.rho_ff_h);
320  }
321 
322  // Resizes Ah_rho_ff to 0
323  void deallocate_Ah_rho_ff_on_device(){
324  Kokkos::resize(Ah_rho_ff,0,0);
325  }
326 # endif
327 
328  // Resizes allocation of dpot_ff and copies to device
329  void copy_dpot_to_device(){
330  Kokkos::resize(dpot_ff,nnode);
331  Kokkos::deep_copy(dpot_ff, dpot_ff_h);
332  }
333 
334  // Resizes dpot_ff to 0
336  Kokkos::resize(dpot_ff,0);
337  }
338 #else
339  // Resizes allocations for large arrays that have a rho dimension (i.e used for ions)
341  Kokkos::resize(E_rho, nnode,nrho+1);
342  Kokkos::deep_copy(E_rho, E.rho_h);
343  }
344 
345  // Resizes allocations that have a rho dimension to 0, to make room for other data on the device
347  Kokkos::resize(E_rho,0,0);
348  }
349 
350  // Resizes allocation of dpot and copies to device
352  Kokkos::resize(dpot,nnode);
353  Kokkos::deep_copy(dpot, dpot_h);
354  }
355 
356  // Resizes dpot to 0
358  Kokkos::resize(dpot,0);
359  }
360 #endif
361 
362 };
363 
364 #endif
void set_rho_pointers(int nrho, int n, double *dpot, FieldXGCa *E_rho)
void deallocate_dpot_on_device()
Definition: electric_field.hpp:357
Definition: electric_field.hpp:10
ElectricField(NLReader::NamelistReader &nlr, int nnode_in, int nplanes_in, int nrho_in)
Definition: electric_field.hpp:145
T get(const string &param, const T default_val, int val_ind=0)
Definition: NamelistReader.hpp:353
double V[3][2]
Definition: electric_field.hpp:11
GridField(int nrho, int nnode)
Definition: electric_field.hpp:65
int nrho
Definition: electric_field.hpp:77
Definition: NamelistReader.hpp:163
Definition: electric_field.hpp:72
Definition: electric_field.hpp:20
void copy_dpot_to_device()
Definition: electric_field.hpp:351
double S[2]
Definition: electric_field.hpp:21
Kokkos::View< FieldXGCa **, Kokkos::LayoutRight, Device > E_rho
Definition: electric_field.hpp:136
Definition: electric_field.hpp:48
ElectricField()
Definition: electric_field.hpp:141
void use_namelist(const string &namelist)
Definition: NamelistReader.hpp:322
void deallocate_rho_fields_on_device()
Definition: electric_field.hpp:346
Definition: electric_field.hpp:25
Kokkos::View< double *, Kokkos::LayoutRight, HostType > dpot_h
Definition: electric_field.hpp:107
double E[2]
Definition: electric_field.hpp:26
void copy_rho_fields_to_device()
Definition: electric_field.hpp:340
Definition: electric_field.hpp:15
bool turb_efield
Whether the electric field is turbulent?
Definition: electric_field.hpp:80
int nphi
Definition: electric_field.hpp:76
int nnode
Definition: electric_field.hpp:75
Kokkos::View< T **, Kokkos::LayoutRight, HostType > rho_h
Definition: electric_field.hpp:63
GridFieldOpts
Definition: electric_field.hpp:40
GridField< FieldXGCa > E
Definition: electric_field.hpp:106
double V[2][2]
Definition: electric_field.hpp:16
Kokkos::View< double *, Kokkos::LayoutRight, Device > dpot
Definition: electric_field.hpp:137