XGCa
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
electric_field.hpp
Go to the documentation of this file.
1 #ifndef ELECTRIC_FIELD_HPP
2 #define ELECTRIC_FIELD_HPP
3 #include "space_settings.hpp"
4 #include "NamelistReader.hpp"
5 #include "grid.hpp"
6 #include "grid_field_pack.hpp"
7 #include "timer_macro.hpp"
9 
10 // Electric field class
11 template<class Device>
12 struct ElectricField {
13 
14  // Dimensions
15  int nnode;
16  int nphi;
17  int nrho;
18 
19  // Field variables
20  bool turb_efield;
21  bool E00_efield;
22 
23  // Host Views
24  Kokkos::View<Field<VarType::Vector2D,PhiInterpType::Planes>*,Kokkos::LayoutRight,HostType> E00_ff_h;
26 
27  //For EM
28  Kokkos::View<double**,Kokkos::LayoutRight,HostType> Ah_h;
31  Kokkos::View<double**,Kokkos::LayoutRight,HostType> As_h;
36 
37  //Kokkos::View<double*,Kokkos::LayoutRight,HostType> dpot_n0_h;
38 
39 #ifdef XGC1
42  Kokkos::View<double**,Kokkos::LayoutRight,HostType> dpot_h;
43  Kokkos::View<Field<VarType::Scalar,PhiInterpType::Planes>*,Kokkos::LayoutRight,HostType> dpot_ff_h;
44 #else
47  Kokkos::View<double*,Kokkos::LayoutRight,HostType> dpot_h;
48 # ifdef SONIC_GK
52 # endif
53 #endif
54  Kokkos::View<double*,Kokkos::LayoutRight,HostType> pot0_h;
55 
56  // Default constructor
58 
59  // Makes sense to use the Grid in the constructor here, but the Grid still copies from fortran so until that changes,
60  // it's simpler to not use it here so that ElectricField can be defined before the restart, which uses psn%As
61  ElectricField(NLReader::NamelistReader& nlr, int nnode_in, int nplanes_in, int nrho_in) : //Grid<DeviceType>& grid) :
62  /*nnode(grid.nnode),
63  nphi(grid.nplanes),
64  nrho(grid.nrho),*/
65  nnode(nnode_in),
66  nphi(nplanes_in),
67  nrho(nrho_in),
68  // Host Views
69 #ifdef XGC1
70  pot(nphi,nrho,nnode),
71  E(nphi,nrho,nnode),
72 # ifdef DELTAF_CONV
73  E00_ff_h("E00_ff_h",nnode),
75 # endif
76 # ifdef EXPLICIT_EM
77  dAh(nphi,nrho,nnode),
78  Ah(nphi,nrho,nnode),
79  Ah_h("Ah_h",4,nnode), // -1:2 in fortran
80  As_h("As_h",4,nnode), // -1:2 in fortran
81 # endif
82  dpot_h("dpot_h",4,nnode), // -1:2 in fortran
83  dpot_ff_h("dpot_ff_h",nnode),
84 #else
85  E(nrho, nnode),
86  dpot_h("dpot_h",nnode),
87 # ifdef SONIC_GK
88  dEr_B2(nrho, nnode),
89  dEz_B2(nrho, nnode),
90  du2_E(nrho, nnode),
91 # endif
92 #endif
93  pot0_h("pot0_h",nnode)
94  {
95  nlr.use_namelist("sml_param");
96  turb_efield = nlr.get<bool>("sml_turb_efield",true);
97  E00_efield = nlr.get<bool>("sml_00_efield",true);
98 
99 #ifdef EXPLICIT_EM
100  // Pass these (or sml) as arguments
101  bool em_mixed_variable = nlr.get<bool>("sml_em_mixed_variable",false);
102  bool em_control_variate = nlr.get<bool>("sml_em_control_variate",false);
103  int em_pullback_mode = nlr.get<int>("sml_em_pullback_mode",3);
104  if (em_control_variate){
106  }
107 
108  if (em_mixed_variable){
111  if (em_pullback_mode==4){
113  }
114  }
115 #endif
116 
117  // Set pointers in Fortran to rho arrays
118 #ifdef XGC1
119  set_rho_ff_pointers(nrho, nnode, pot0_h.data(), dpot_h.data(), dpot_ff_h.data(), pot.rho_ff_h.data(), E.rho_ff_h.data()
120 # ifdef EXPLICIT_EM
121  , Ah_h.data(), Ah.rho_ff_h.data(), dAh.rho_ff_h.data(), As_h.data(), As.rho_ff_h.data(), dAs.rho_ff_h.data(), Epar_em.rho_ff_h.data()
122 # endif
123  );
124 #else
125  set_rho_pointers(nrho, nnode, pot0_h.data(), dpot_h.data(), E.rho_h.data()
126 # ifdef SONIC_GK
127  , dEr_B2.rho_h.data()
128  , dEz_B2.rho_h.data()
129  , du2_E.rho_h.data()
130 # endif
131  );
132 #endif
133  }
134 
135 
136  /**************************************************/
137  /************ Copy to GridFieldPack *************/
138  /**************************************************/
139 
140  // Should these functions be in GridFieldPack instead? - ALS
141 
142 
143  // Shallow copy if DeviceType and HostType are the same
144  template<typename T>
145  void copy_to_pack(T& device_view, const T& host_view) const{
146  device_view = host_view;
147  }
148 
149  // Allocate device view and copy data from host
150  template<typename T1, typename T2>
151  void copy_to_pack(T1& device_view, const T2& host_view) const{
152  // Allocate device array
153  device_view = T1("gfpack_view",host_view.layout());
154  // Deep copy from host to device
155  Kokkos::deep_copy(device_view, host_view);
156  }
157 
158  // Resizes allocations for large arrays that have a phi dimension (i.e used for electron push)
159  template<PhiInterpType PIT>
161  // Re-initialize new gridpack - deallocates whatever Views were already allocated
162  GridFieldPack<Device, PIT> gfpack(kintype_in, turb_efield);
163 #ifdef XGC1
164  // Copy field data to device (or shallow copy if no device)
165  if(kintype_in==DriftKin){
166  GPTLstart("GAT_FIELD_INFO");
167  if(pol_decomp.decompose_fields){
168  gfpack.phi_offset = pol_decomp.field_decomp.first_plane;
169  gfpack.node_offset = pol_decomp.field_decomp.first_node;
170 //printf("\ngfpack.phi_offset: %d, gfpack.node_offset: %d\n", gfpack.phi_offset, gfpack.node_offset);
171  }
172  PlaneFieldGatherer pfg(pol_decomp, grid);
173 //checkpoint("\n\n");
174 //checkpoint("Starting GATHER");
175 //checkpoint("\n\n");
176  pfg.gather_phi_ff_on_device(E.rho_ff_h, gfpack.E_phi_ff);
177 //Kokkos::fence();
178 //checkpoint("\n\n");
179 //checkpoint("Finished GATHER");
180 //checkpoint("\n\n");
181 # ifdef DELTAF_CONV
182  if(ddpotdt.ff_h.size()>0) pfg.gather_phi_ff_on_device(ddpotdt.ff_h, gfpack.ddpotdt_phi_ff);
183  copy_to_pack(gfpack.E00_ff, E00_ff_h); // No gather_phi needed due to axisymmetry
184 # endif
185 # ifdef EXPLICIT_EM
186  pfg.gather_phi_ff_on_device(dAh.rho_ff_h, gfpack.dAh_phi_ff);
187  pfg.gather_phi_ff_on_device(Ah.rho_ff_h, gfpack.Ah_phi_ff);
188  if(dAs.rho_ff_h.size()>0) pfg.gather_phi_ff_on_device(dAs.rho_ff_h, gfpack.dAs_phi_ff);
189  if(As.rho_ff_h.size()>0) pfg.gather_phi_ff_on_device(As.rho_ff_h, gfpack.As_phi_ff);
190  if(Ah_cv.rho_ff_h.size()>0) pfg.gather_phi_ff_on_device(Ah_cv.rho_ff_h, gfpack.Ah_cv_phi_ff);
191  if(Epar_em.rho_ff_h.size()>0) pfg.gather_phi_ff_on_device(Epar_em.rho_ff_h, gfpack.Epar_em_phi_ff);
192 # endif
193  GPTLstop("GAT_FIELD_INFO");
194  }else{
195  GPTLstart("Copy_rho_ff_fields_to_device");
196  copy_to_pack(gfpack.E_rho_ff, E.rho_ff_h);
197 # ifdef EXPLICIT_EM
198  copy_to_pack(gfpack.dAs_rho_ff, dAs.rho_ff_h);
199  copy_to_pack(gfpack.dAh_rho_ff, dAh.rho_ff_h);
200  copy_to_pack(gfpack.As_rho_ff, As.rho_ff_h);
201  copy_to_pack(gfpack.Ah_rho_ff, Ah.rho_ff_h);
202  copy_to_pack(gfpack.Epar_em_rho_ff, Epar_em.rho_ff_h);
203 # endif
204  GPTLstop("Copy_rho_ff_fields_to_device");
205  }
206 #else
207  copy_to_pack(gfpack.E_rho, E.rho_h);
208 # ifdef SONIC_GK
209  copy_to_pack(gfpack.dEr_B2_rho, dEr_B2.rho_h);
210  copy_to_pack(gfpack.dEz_B2_rho, dEz_B2.rho_h);
211  copy_to_pack(gfpack.du2_E_rho, du2_E.rho_h);
212 # endif
213 #endif
214  return gfpack;
215  }
216 
217  /* Set of functions used to allocate/deallocate gpu memory for Ah during particle scatters */
218  // Resizes allocation of Ah_cv
219  template<PhiInterpType PIT>
221  GridFieldPack<Device, PIT> gfpack(kintype_in, turb_efield);
222 #ifdef EXPLICIT_EM
223  PlaneFieldGatherer pfg(pol_decomp, grid);
224  pfg.gather_phi_ff_on_device(Ah_cv.rho_ff_h, gfpack.Ah_cv_phi_ff);
225 #endif
226  return gfpack;
227  }
228 
229  template<PhiInterpType PIT>
231  GridFieldPack<Device, PIT> gfpack(kintype_in, turb_efield);
232 #ifdef EXPLICIT_EM
233  if(kintype_in==DriftKin){
234  PlaneFieldGatherer pfg(pol_decomp, grid);
235  pfg.gather_phi_ff_on_device(Ah.rho_ff_h, gfpack.Ah_phi_ff);
236  }else{
237  copy_to_pack(gfpack.Ah_rho_ff, Ah.rho_ff_h);
238  }
239 #endif
240  return gfpack;
241  }
242 
243 
244  // Resizes allocation of dpot_ff and copies to device
245  template<PhiInterpType PIT>
247  GridFieldPack<Device, PIT> gfpack(kintype_in, turb_efield);
248 #ifdef XGC1
249  copy_to_pack(gfpack.dpot_ff, dpot_ff_h);
250 #else
251  copy_to_pack(gfpack.dpot, dpot_h);
252 #endif
253  return gfpack;
254  }
255 
256 };
257 
258 #endif
GridField< VarType::Scalar, PhiInterpType::Planes > Ah_cv
Definition: electric_field.hpp:34
bool decompose_fields
Whether to decompose fields.
Definition: domain_decomposition.hpp:34
GridField< VarType::Scalar, PhiInterpType::None > pot
Definition: electric_field.hpp:45
static int GPTLstart(const char *name)
Definition: timer_macro.hpp:9
void set_rho_pointers(int nrho, int n, double *pot0, double *dpot, Field< VarType::Vector2D, PhiInterpType::None > *E_rho)
int first_plane
First plane belonging to this rank, including ghost planes.
Definition: field_decomposition.hpp:35
void copy_to_pack(T &device_view, const T &host_view) const
Definition: electric_field.hpp:145
ElectricField(NLReader::NamelistReader &nlr, int nnode_in, int nplanes_in, int nrho_in)
Definition: electric_field.hpp:61
Kokkos::View< Field< VarType::Vector2D, PIT > **, Kokkos::LayoutRight, Device > E_rho
Definition: grid_field_pack.hpp:277
GridFieldPack< Device, PIT > copy_Ah_cv_to_device(KinType kintype_in, const DomainDecomposition< DeviceType > &pol_decomp, const Grid< DeviceType > &grid) const
Definition: electric_field.hpp:220
T get(const string &param, const T default_val, int val_ind=0)
Definition: NamelistReader.hpp:353
Kokkos::Device< HostExSpace, HostMemSpace > HostType
Definition: space_settings.hpp:56
Definition: globals.hpp:82
GridFieldOpts
Definition: grid_field_pack.hpp:209
Definition: plane_field_gatherer.hpp:9
GridFieldPack< Device, PIT > copy_push_fields_to_device(KinType kintype_in, const DomainDecomposition< DeviceType > &pol_decomp, const Grid< DeviceType > &grid) const
Definition: electric_field.hpp:160
int nrho
Definition: electric_field.hpp:17
Definition: NamelistReader.hpp:163
int phi_offset
Offset for phi_ff field decomposition.
Definition: grid_field_pack.hpp:250
Definition: electric_field.hpp:12
Kokkos::View< double *, Kokkos::LayoutRight, Device > dpot
Definition: grid_field_pack.hpp:278
GridFieldPack< Device, PIT > copy_Ah_ff_to_device(KinType kintype_in, const DomainDecomposition< DeviceType > &pol_decomp, const Grid< DeviceType > &grid) const
Definition: electric_field.hpp:230
Definition: grid_field_pack.hpp:241
Kokkos::View< Field< VT, PIT > **, Kokkos::LayoutRight, HostType > rho_h
Definition: grid_field_pack.hpp:232
GridField< VarType::Vector2D, PhiInterpType::None > E
Definition: electric_field.hpp:46
Kokkos::View< double **, Kokkos::LayoutRight, HostType > Ah_h
Definition: electric_field.hpp:28
ElectricField()
Definition: electric_field.hpp:57
GridField< VarType::Scalar, PhiInterpType::Planes > ddpotdt
Time derivative of phi-&lt;phi&gt; - Not field-following?
Definition: electric_field.hpp:25
Kokkos::View< Field< VarType::Vector2D, PhiInterpType::Planes > *, Kokkos::LayoutRight, HostType > E00_ff_h
Radial electric field from &lt;phi&gt; in field-following format.
Definition: electric_field.hpp:24
void use_namelist(const string &namelist)
Definition: NamelistReader.hpp:322
GridField< VarType::Scalar, PhiInterpType::Planes > Ah
Definition: electric_field.hpp:30
GridField< VarType::Vector, PhiInterpType::Planes > dAs
Definition: electric_field.hpp:32
GridField< VarType::Scalar, PhiInterpType::Planes > Epar_em
Definition: electric_field.hpp:35
Kokkos::View< double **, Kokkos::LayoutRight, HostType > As_h
Definition: electric_field.hpp:31
void gather_phi_ff_on_device(View< FT **, CLayout, HostType > &tmp, View< FT *, CLayout, HostType > &tmp_full, const T_h &rho_ff_h, View< FT **, CLayout, DeviceType > &phi_ff)
Definition: plane_field_gatherer.hpp:133
Kokkos::View< double *, Kokkos::LayoutRight, HostType > pot0_h
Definition: electric_field.hpp:54
bool E00_efield
Flux-surface averaged potential not used for calculating the electric field if .false.
Definition: electric_field.hpp:21
GridField< VarType::Scalar, PhiInterpType::Planes > As
Definition: electric_field.hpp:33
int node_offset
Offset for phi_ff field decomposition.
Definition: grid_field_pack.hpp:251
Kokkos::View< double *, Kokkos::LayoutRight, HostType > dpot_h
Definition: electric_field.hpp:47
KinType
Definition: globals.hpp:81
Kokkos::View< Field< VT, PIT > **, Kokkos::LayoutRight, HostType > rho_ff_h
Contains the local plane of gyroaveraged, field-following, host field.
Definition: grid_field_pack.hpp:221
Kokkos::View< Field< VT, PIT > *, Kokkos::LayoutRight, HostType > ff_h
Contains the local plane, field-following, host field.
Definition: grid_field_pack.hpp:220
bool turb_efield
E-field calculated only with if .false., psndpot will still contain all (n=0,|m|&gt;0) and (|n|&gt;0...
Definition: electric_field.hpp:20
int nphi
Definition: electric_field.hpp:16
void copy_to_pack(T1 &device_view, const T2 &host_view) const
Definition: electric_field.hpp:151
int nnode
Definition: electric_field.hpp:15
FieldDecomposition< Device > field_decomp
Definition: domain_decomposition.hpp:35
GridFieldPack< Device, PIT > copy_dpot_to_device(KinType kintype_in) const
Definition: electric_field.hpp:246
GridField< VarType::Vector, PhiInterpType::Planes > dAh
Definition: electric_field.hpp:29
int first_node
First mesh node belonging to this rank, including ghost nodes.
Definition: field_decomposition.hpp:32
static int GPTLstop(const char *name)
Definition: timer_macro.hpp:10