1 #ifndef GRID_FIELD_PACK_HPP
2 #define GRID_FIELD_PACK_HPP
22 template<VarType VT, PhiInterpType PIT>
38 KOKKOS_INLINE_FUNCTION
void gather(
SimdVector& vec,
int i_simd,
double wp,
const double (&wphi)[2],
const double (&rvec)[2],
const double (&zvec)[2])
const{
39 vec.
r[i_simd] += wp*(wphi[0]*( rvec[0]*V[
PIR][0] + zvec[0]*V[
PIZ][0] )
40 + wphi[1]*( rvec[0]*V[
PIR][1] + zvec[0]*V[
PIZ][1] ) );
41 vec.
z[i_simd] += wp*(wphi[0]*( rvec[1]*V[
PIR][0] + zvec[1]*V[
PIZ][0] )
42 + wphi[1]*( rvec[1]*V[
PIR][1] + zvec[1]*V[PIZ][1] ) );
43 vec.
phi[i_simd] += wp*(wphi[0]* V[
PIP][0]
44 + wphi[1]* V[
PIP][1] );
58 vec.
r[i_simd] += wp*(wts.
phi.
w[0]*( rvec[0]*(wts.
rho[0].
w[0]*V[
PIR][0] + wts.
rho[0].
w[1]*field2.
V[
PIR][0])
59 + zvec[0]*(wts.
rho[0].
w[0]*V[
PIZ][0] + wts.
rho[0].
w[1]*field2.
V[
PIZ][0]) )
61 + zvec[0]*(wts.
rho[1].
w[0]*V[
PIZ][1] + wts.
rho[1].
w[1]*field2.
V[
PIZ][1]) ) );
62 vec.
z[i_simd] += wp*(wts.
phi.
w[0]*( rvec[1]*(wts.
rho[0].
w[0]*V[
PIR][0] + wts.
rho[0].
w[1]*field2.
V[
PIR][0])
63 + zvec[1]*(wts.
rho[0].
w[0]*V[
PIZ][0] + wts.
rho[0].
w[1]*field2.
V[
PIZ][0]) )
65 + zvec[1]*(wts.
rho[1].
w[0]*V[
PIZ][1] + wts.
rho[1].
w[1]*field2.
V[
PIZ][1]) ) );
69 vec.
r[i_simd] += wp*(wts.
phi.
w[0]*( rvec[0]*(wts.
rho.
w[0]*V[
PIR][0] + wts.
rho.
w[1]*field2.
V[
PIR][0])
73 vec.
z[i_simd] += wp*(wts.
phi.
w[0]*( rvec[1]*(wts.
rho.
w[0]*V[
PIR][0] + wts.
rho.
w[1]*field2.
V[
PIR][0])
97 KOKKOS_INLINE_FUNCTION
void gather(
SimdVector& vec,
int i_simd,
double wp,
const double (&wphi)[2],
const double (&rvec)[2],
const double (&zvec)[2])
const{
98 vec.
r[i_simd] += wp*(wphi[0]*( rvec[0]*V[
PIR][0] + zvec[0]*V[
PIZ][0] )
99 + wphi[1]*( rvec[0]*V[
PIR][1] + zvec[0]*V[
PIZ][1] ) );
100 vec.
z[i_simd] += wp*(wphi[0]*( rvec[1]*V[
PIR][0] + zvec[1]*V[
PIZ][0] )
101 + wphi[1]*( rvec[1]*V[
PIR][1] + zvec[1]*V[PIZ][1] ) );
116 KOKKOS_INLINE_FUNCTION
void gather(
Simd<double>& sca,
int i_simd,
double wp,
const double (&wphi)[2])
const{
117 sca[i_simd] += wp* ( wphi[0]*S[0] + wphi[1]*S[1] );
128 # ifdef NEWGYROMATRIX
129 sca[i_simd] += wp* ( wts.
phi.
w[0]*wts.
rho[0].
w[0]*S[0]
131 + wts.
phi.
w[0]*wts.
rho[0].
w[1]*field2.
S[0]
132 + wts.
phi.
w[1]*wts.
rho[1].
w[1]*field2.
S[1]);
134 sca[i_simd] += wp* ( wts.
phi.
w[0]*wts.
rho.
w[0]*S[0]
137 + wts.
phi.
w[1]*wts.
rho.
w[1]*field2.
S[1]);
154 KOKKOS_INLINE_FUNCTION
void gather(
SimdVector& vec,
int i_simd,
double wp,
const double (&rvec)[2],
const double (&zvec)[2])
const{
155 vec.
r[i_simd] += wp*( rvec[0]*E[
PIR] + zvec[0]*E[
PIZ] );
156 vec.
z[i_simd] += wp*( rvec[1]*E[
PIR] + zvec[1]*E[
PIZ] );
169 vec.
r[i_simd] += wp*( rvec[0]*(wts.
rho.
w[0]*E[
PIR] + wts.
rho.
w[1]*field2.
E[
PIR])
171 vec.
z[i_simd] += wp*( rvec[1]*(wts.
rho.
w[0]*E[
PIR] + wts.
rho.
w[1]*field2.
E[
PIR])
196 ,
double* Ah,
Field<VarType::Scalar,PhiInterpType::Planes>* Ah_rho_ff,
Field<VarType::Vector,PhiInterpType::Planes>* dAh_rho_ff,
double* As,
Field<VarType::Scalar,PhiInterpType::Planes>* As_rho_ff,
Field<VarType::Vector,PhiInterpType::Planes>* dAs_rho_ff,
Field<VarType::Scalar,PhiInterpType::Planes>* E_para_em_rho_ff
213 template<VarType VT, PhiInterpType PIT>
223 ff_h = Kokkos::View<Field<VT,PIT>*,Kokkos::LayoutRight,
HostType>(
"ff_h",nnode);
225 rho_ff_h = Kokkos::View<Field<VT,PIT>**,Kokkos::LayoutRight,
HostType>(
"rho_ff_h",nnode,nrho+1);
234 :
rho_h(
"rho_h",nnode, nrho+1) {}
238 template<
class Device, PhiInterpType PIT>
253 Kokkos::View<Field<VarType::Vector,PIT>**,Kokkos::LayoutRight,Device> E_phi_ff;
254 Kokkos::View<Field<VarType::Vector,PIT>**,Kokkos::LayoutRight,Device> E_rho_ff;
255 Kokkos::View<Field<VarType::Scalar,PIT>*,Kokkos::LayoutRight,Device> dpot_ff;
257 Kokkos::View<Field<VarType::Vector2D,PIT>*,Kokkos::LayoutRight,Device> E00_ff;
258 Kokkos::View<Field<VarType::Scalar,PIT>**,Kokkos::LayoutRight,Device> ddpotdt_phi_ff;
262 Kokkos::View<Field<VarType::Vector,PIT>**,Kokkos::LayoutRight,Device> dAh_phi_ff;
263 Kokkos::View<Field<VarType::Vector,PIT>**,Kokkos::LayoutRight,Device> dAh_rho_ff;
264 Kokkos::View<Field<VarType::Scalar,PIT>**,Kokkos::LayoutRight,Device> Ah_phi_ff;
265 Kokkos::View<Field<VarType::Scalar,PIT>**,Kokkos::LayoutRight,Device> Ah_rho_ff;
266 Kokkos::View<Field<VarType::Vector,PIT>**,Kokkos::LayoutRight,Device> dAs_phi_ff;
267 Kokkos::View<Field<VarType::Vector,PIT>**,Kokkos::LayoutRight,Device> dAs_rho_ff;
268 Kokkos::View<Field<VarType::Scalar,PIT>**,Kokkos::LayoutRight,Device> As_phi_ff;
269 Kokkos::View<Field<VarType::Scalar,PIT>**,Kokkos::LayoutRight,Device> As_rho_ff;
270 Kokkos::View<Field<VarType::Scalar,PIT>**,Kokkos::LayoutRight,Device> Ah_cv_phi_ff;
271 Kokkos::View<Field<VarType::Scalar,PIT>**,Kokkos::LayoutRight,Device> Epar_em_phi_ff;
272 Kokkos::View<Field<VarType::Scalar,PIT>**,Kokkos::LayoutRight,Device> Epar_em_rho_ff;
275 Kokkos::View<Field<VarType::Vector2D,PIT>**,Kokkos::LayoutRight,Device>
E_rho;
276 Kokkos::View<double*,Kokkos::LayoutRight,Device>
dpot;
278 View<Field<VarType::Vector2D,PIT>**,
CLayout,Device> du2_E_rho;
311 KOKKOS_INLINE_FUNCTION
void field_correction(
int i_simd,
double basis,
double gamma_psi,
const SimdVector2D &gradpsi,
double* rvec,
double* zvec)
const {
312 rvec[0]=basis + (1.0-basis) * gamma_psi * gradpsi.
r[i_simd];
313 rvec[1]= (1.0-basis) * gamma_psi * gradpsi.
z[i_simd];
314 zvec[0]= (1.0-basis) * gamma_psi *(-gradpsi.
z[i_simd]);
315 zvec[1]=basis + (1.0-basis) * gamma_psi * gradpsi.
r[i_simd];
319 vec.
phi[i_simd]=(vec.
phi[i_simd]*Bmag - vec.
r[i_simd]*B.
r[i_simd] - vec.
z[i_simd]*B.
z[i_simd]) / B.
phi[i_simd];
327 E_rho(node,0).gather(fld.
E,i_simd, wp, rvec, zvec);
329 const int i_phi = (wts.phi.i -
phi_offset)%E_phi_ff.extent(0);
330 E_phi_ff(i_phi,node).gather(fld.
E,i_simd, wp, wts.phi.w, rvec, zvec);
332 Ah_phi_ff(i_phi,node).gather(fld.Ah,i_simd, wp, wts.phi.w);
333 dAh_phi_ff(i_phi,node).gather(fld.dAh,i_simd, wp, wts.phi.w, rvec, zvec);
335 if(push_controls.em_control_variate) {
336 Ah_cv_phi_ff(i_phi,node).gather(fld.Ah_cv,i_simd, wp, wts.phi.w);
339 if(push_controls.em_mixed_variable) {
340 dAs_phi_ff(i_phi,node).gather(fld.dAs,i_simd, wp, wts.phi.w, rvec, zvec);
341 As_phi_ff(i_phi,node).gather(fld.As,i_simd, wp, wts.phi.w);
343 if(push_controls.em_mixed_variable && (push_controls.em_pullback_mode==4)) {
344 Epar_em_phi_ff(i_phi,node).gather(fld.Epar_em,i_simd,wp,wts.phi.w);
349 #if defined(XGC1) && defined(DELTAF_CONV)
350 E00_ff(node).gather(fld.E00,i_simd, wp, wts.phi.w, rvec, zvec);
351 ddpotdt_phi_ff(i_phi,node).gather(fld.ddpotdt,i_simd, wp, wts.phi.w);
359 int irho = wts.rho[0].i;
361 int irho = wts.rho.i;
364 E_rho(node,irho).gather(fld.
E,i_simd, wp, wts, rvec, zvec,
E_rho(node,irho+1));
366 du2_E_rho(node,0).gather(fld.du2_E,i_simd, wp, rvec, zvec);
369 E_rho_ff(node,irho).gather(fld.
E,i_simd, wp, wts, rvec, zvec, E_rho_ff(node,irho+1));
371 Ah_rho_ff(node,irho).gather(fld.Ah,i_simd, wp, wts, Ah_rho_ff(node,irho+1));
372 dAh_rho_ff(node,irho).gather(fld.dAh,i_simd, wp, wts, rvec, zvec, dAh_rho_ff(node,irho+1));
374 if(push_controls.em_mixed_variable) {
375 dAs_rho_ff(node,irho).gather(fld.dAs,i_simd, wp, wts, rvec, zvec, dAs_rho_ff(node,irho+1));
376 As_rho_ff(node,irho).gather(fld.As,i_simd, wp, wts, As_rho_ff(node,irho+1));
378 if(push_controls.em_mixed_variable && (push_controls.em_pullback_mode==4)) {
379 Epar_em_rho_ff(node,irho).gather(fld.Epar_em,i_simd,wp,wts, Epar_em_rho_ff(node,irho+1));
418 for (
int i_simd = 0; i_simd<SIMD_SIZE; i_simd++){
420 int itr_work = itr[i_simd]>0 ? itr[i_simd] : 1;
422 double phi= itr[i_simd]>0 ? fld_phi[i_simd] : 0.0;
427 double gamma_psi=1.0/sqrt(gradpsi.
r[i_simd]*gradpsi.
r[i_simd] + gradpsi.
z[i_simd]*gradpsi.
z[i_simd]);
433 for (
int ip = 0; ip<3; ip++){
434 int node = itr[i_simd]>0 ? (grid.
nodes(itr_work - 1).vertex[ip] - 1) : 0;
435 double wp = itr[i_simd]>0 ? p[ip][i_simd] : 0.0;
442 #ifdef NEOCLASSICAL_TEST
443 fld.
E.
phi[i_simd]=0.0;
448 double Bmag=sqrt(B.
r[i_simd]*B.
r[i_simd] + B.
z[i_simd]*B.
z[i_simd] + B.
phi[i_simd]*B.
phi[i_simd]);
457 fld.
E.
phi[i_simd]=0.0;
459 fld.dAh.phi[i_simd]=0.0;
460 fld.dAs.phi[i_simd]=0.0;
475 for (
int i_simd = 0; i_simd<SIMD_SIZE; i_simd++){
477 int itr_work = itr[i_simd]>0 ? itr[i_simd] : 1;
478 double phi= itr[i_simd]>0 ? fld_phi[i_simd] : 0.0;
484 int irho = wts.rho[0].i;
486 int irho = wts.rho.i;
490 for (
int ip = 0; ip<3; ip++){
491 int node = itr[i_simd]>0 ? (grid.
nodes(itr_work - 1).vertex[ip] - 1) : 0;
492 double wp = itr[i_simd]>0 ? p[ip][i_simd] : 0.0;
494 Ah_rho_ff(node,irho).gather(Ah,i_simd, wp, wts, Ah_rho_ff(node,irho+1));
506 for (
int i_simd = 0; i_simd<SIMD_SIZE; i_simd++){
508 int itr_work = itr[i_simd]>0 ? itr[i_simd] : 1;
509 double phi= itr[i_simd]>0 ? fld_phi[i_simd] : 0.0;
515 for (
int ip = 0; ip<3; ip++){
516 int i_node = itr[i_simd]>0 ? (grid.
nodes(itr_work - 1).vertex[ip] - 1) : 0;
517 double wp = itr[i_simd]>0 ? p[ip][i_simd] : 0.0;
522 Ah_phi_ff(i_phi,node).gather(Ah,i_simd, wp, wts.phi.w);
534 for (
int i_simd = 0; i_simd<SIMD_SIZE; i_simd++){
536 int itr_work = itr[i_simd]>0 ? itr[i_simd] : 1;
537 double phi= itr[i_simd]>0 ? fld_phi[i_simd] : 0.0;
543 for (
int ip = 0; ip<3; ip++){
544 int i_node = itr[i_simd]>0 ? (grid.
nodes(itr_work - 1).vertex[ip] - 1) : 0;
545 double wp = itr[i_simd]>0 ? p[ip][i_simd] : 0.0;
550 Ah_cv_phi_ff(i_phi,node).gather(Ah_cv,i_simd, wp, wts.phi.w);
560 for (
int i_simd = 0; i_simd<SIMD_SIZE; i_simd++){
562 int itr_work = itr[i_simd]>0 ? itr[i_simd] : 1;
564 double phi = itr[i_simd]>0 ? fld_phi[i_simd] : 0.0;
568 for(
int ip=0; ip<3; ip++){
569 int node = itr[i_simd]>0 ? (grid.
nodes(itr_work - 1).vertex[ip] - 1) : 0;
570 double wp = itr[i_simd]>0 ? p[ip][i_simd] : 0.0;
575 dpot_ff(node).gather(dpot_out,i_simd, wp, wts.phi.w);
581 dpot_out[i_simd]+=wp*
dpot(node);
Simd< double > r
Definition: simd.hpp:145
KOKKOS_INLINE_FUNCTION void gather(SimdVector &vec, int i_simd, double wp, const double(&rvec)[2], const double(&zvec)[2]) const
Definition: grid_field_pack.hpp:154
GridFieldPack()
Definition: grid_field_pack.hpp:282
void set_rho_pointers(int nrho, int n, double *pot0, double *dpot, Field< VarType::Vector2D, PhiInterpType::None > *E_rho)
Definition: gyro_radius.hpp:24
KOKKOS_INLINE_FUNCTION void get_dpot(const Grid< Device > &grid, const Simd< double > &fld_phi, const Simd< int > &itr, const SimdGridVec &p, Simd< double > &dpot_out) const
Definition: grid_field_pack.hpp:556
KOKKOS_INLINE_FUNCTION void get_Ah(const Grid< Device > &grid, const Simd< double > &fld_phi, const Simd< int > &itr, const SimdGridVec &p, SimdGyroRadius< GyroKin > &rho, Simd< double > &Ah) const
Definition: grid_field_pack.hpp:469
Kokkos::View< Field< VarType::Vector2D, PIT > **, Kokkos::LayoutRight, Device > E_rho
Definition: grid_field_pack.hpp:275
Definition: grid_field_pack.hpp:107
bool turb_efield
Definition: grid_field_pack.hpp:246
Definition: field_weights.hpp:13
Kokkos::Device< HostExSpace, HostMemSpace > HostType
Definition: space_settings.hpp:56
GridFieldOpts
Definition: grid_field_pack.hpp:207
LinearWeights phi
Definition: field_weights.hpp:35
Definition: push_controls.hpp:8
int phi_offset
Offset for phi_ff field decomposition.
Definition: grid_field_pack.hpp:248
Kokkos::View< double *, Kokkos::LayoutRight, Device > dpot
Definition: grid_field_pack.hpp:276
int nplanes
Number of planes.
Definition: grid.hpp:84
GridField(int nphi, int nrho, int nnode, GridFieldOpts opt=GridFieldOpts::WithRhoFF)
Definition: grid_field_pack.hpp:221
Definition: grid_field_pack.hpp:239
Kokkos::LayoutRight CLayout
Definition: space_settings.hpp:60
Kokkos::View< Field< VT, PIT > **, Kokkos::LayoutRight, HostType > rho_h
Definition: grid_field_pack.hpp:230
Definition: local_fields.hpp:7
Definition: grid_field_pack.hpp:214
bool can_reuse(KinType kintype_in)
Definition: grid_field_pack.hpp:292
double S[2]
Definition: grid_field_pack.hpp:108
KOKKOS_INLINE_FUNCTION void get_Ah(const Grid< Device > &grid, const Simd< double > &fld_phi, const Simd< int > &itr, const SimdGridVec &p, SimdGyroRadius< DriftKin > &rho, Simd< double > &Ah) const
Definition: grid_field_pack.hpp:500
Definition: field_weights.hpp:33
Definition: grid_field_pack.hpp:27
LinearWeights rho
Definition: field_weights.hpp:36
KOKKOS_INLINE_FUNCTION void get_Ah_cv(const Grid< Device > &grid, const Simd< double > &fld_phi, const Simd< int > &itr, const SimdGridVec &p, SimdGyroRadius< DriftKin > &rho, Simd< double > &Ah_cv) const
Definition: grid_field_pack.hpp:528
r coordinate
Definition: globals.hpp:102
KOKKOS_INLINE_FUNCTION void gather_all_fields(const PushControls &push_controls, int i_simd, int i_node, double wp, const double(&rvec)[2], const double(&zvec)[2], const FieldWeights< DriftKin, PIT > &wts, LocalFields &fld) const
Definition: grid_field_pack.hpp:324
KinType kintype
Definition: grid_field_pack.hpp:241
Kokkos::View< int *, Kokkos::LayoutRight, Device > basis
A basis for the guesses?
Definition: grid.hpp:91
Definition: field_weights.hpp:62
KOKKOS_INLINE_FUNCTION void gather(SimdVector &vec, int i_simd, double wp, const double(&wphi)[2], const double(&rvec)[2], const double(&zvec)[2]) const
Definition: grid_field_pack.hpp:97
double S
Definition: grid_field_pack.hpp:180
Simd< double > z
Definition: simd.hpp:141
KOKKOS_INLINE_FUNCTION void phi_from_para(SimdVector &vec, int i_simd, const SimdVector &B, double Bmag) const
Definition: grid_field_pack.hpp:318
double w[2]
Definition: linear_weights.hpp:9
KOKKOS_INLINE_FUNCTION void gather(SimdVector &vec, int i_simd, double wp, const FieldWeights< GyroKin, PhiInterpType::Planes > &wts, const double(&rvec)[2], const double(&zvec)[2], const Field< VarType::Vector, PhiInterpType::Planes > &field2) const
Definition: grid_field_pack.hpp:56
KOKKOS_INLINE_FUNCTION void gather(Simd< double > &sca, int i_simd, double wp, const FieldWeights< GyroKin, PhiInterpType::Planes > &wts, const Field< VarType::Scalar, PhiInterpType::Planes > &field2) const
Definition: grid_field_pack.hpp:127
GridFieldPack(KinType kintype_in, bool turb_efield_in)
Definition: grid_field_pack.hpp:285
KOKKOS_INLINE_FUNCTION void gather(Simd< double > &sca, int i_simd, double wp)
Definition: grid_field_pack.hpp:187
Simd< double > phi
Definition: simd.hpp:147
int node_offset
Offset for phi_ff field decomposition.
Definition: grid_field_pack.hpp:249
Simd< double > z
Definition: simd.hpp:146
KinType
Definition: globals.hpp:81
Definition: grid_field_pack.hpp:23
Kokkos::View< Field< VT, PIT > **, Kokkos::LayoutRight, HostType > rho_ff_h
Contains the local plane of gyroaveraged, field-following, host field.
Definition: grid_field_pack.hpp:219
KOKKOS_INLINE_FUNCTION void zero()
Definition: simd.hpp:76
Definition: grid_structs.hpp:7
VarType
Definition: grid_field_pack.hpp:16
Kokkos::View< Field< VT, PIT > *, Kokkos::LayoutRight, HostType > ff_h
Contains the local plane, field-following, host field.
Definition: grid_field_pack.hpp:218
Simd< double > r
Definition: simd.hpp:140
KOKKOS_INLINE_FUNCTION void gather(SimdVector &vec, int i_simd, double wp, const FieldWeights< GyroKin, PhiInterpType::None > &wts, const double(&rvec)[2], const double(&zvec)[2], const Field< VarType::Vector2D, PhiInterpType::None > &field2) const
Definition: grid_field_pack.hpp:168
GridField(int nrho, int nnode)
Definition: grid_field_pack.hpp:233
KOKKOS_INLINE_FUNCTION void gather(Simd< double > &sca, int i_simd, double wp, const double(&wphi)[2]) const
Definition: grid_field_pack.hpp:116
KOKKOS_INLINE_FUNCTION void zero()
Definition: simd.hpp:162
phi coordinate
Definition: globals.hpp:104
bool is_initialized
Definition: grid_field_pack.hpp:242
Definition: gyro_radius.hpp:96
LinearWeights rho
Definition: field_weights.hpp:64
GridField()
Definition: grid_field_pack.hpp:215
Definition: gyro_radius.hpp:74
KOKKOS_INLINE_FUNCTION void gather(SimdVector &vec, int i_simd, double wp, const double(&wphi)[2], const double(&rvec)[2], const double(&zvec)[2]) const
Definition: grid_field_pack.hpp:38
z coordinate
Definition: globals.hpp:103
double E[2]
Definition: grid_field_pack.hpp:145
KOKKOS_INLINE_FUNCTION void gather_all_fields(const PushControls &push_controls, int i_simd, int node, double wp, const double(&rvec)[2], const double(&zvec)[2], const FieldWeights< GyroKin, PIT > &wts, LocalFields &fld) const
Definition: grid_field_pack.hpp:356
SimdVector E
Definition: local_fields.hpp:8
double V[3][2]
Definition: grid_field_pack.hpp:28
KOKKOS_INLINE_FUNCTION void field_correction(int i_simd, double basis, double gamma_psi, const SimdVector2D &gradpsi, double *rvec, double *zvec) const
Definition: grid_field_pack.hpp:311
Definition: grid_field_pack.hpp:144
Kokkos::View< Vertex *, Kokkos::LayoutRight, Device > nodes
Definition: grid.hpp:92
KOKKOS_INLINE_FUNCTION void fields_at_point(const PushControls &push_controls, const Grid< Device > &grid, const SimdVector &B, const SimdVector2D &gradpsi, const Simd< double > &fld_phi, const Simd< int > &itr, const SimdGridVec &p, SimdGyroRadius< PT > &rho, LocalFields &fld) const
Definition: grid_field_pack.hpp:398