3 #include <Cabana_AoSoA.hpp>
4 #include <Cabana_DeepCopy.hpp>
5 #include <Kokkos_Core.hpp>
73 template<
class Device>
95 Cabana::AoSoA<ParticleDataTypes,HostType,VEC_LEN>
particles;
111 Cabana::AoSoA<PhaseDataTypes,HostType,VEC_LEN>
phase0;
112 Cabana::AoSoA<PhaseDataTypes,Device,VEC_LEN>
phase0_d;
132 Species(
int idx_in,
int nonadiabatic_idx_in,
bool is_electron_in,
bool is_adiabatic_in,
KinType kintype_in,
double mass_in,
double charge_in,
double charge_eu_in,
bool is_deltaf_in,
182 auto ph = Cabana::slice<PtlSlice::Ph>(
particles);
183 auto ct = Cabana::slice<PtlSlice::Ct>(
particles);
184 auto gid = Cabana::slice<PtlSlice::Gid>(
particles);
186 auto flag = Cabana::slice<PtlSlice::Flag>(
particles);
193 long long int gid_offset = 0;
197 for (
int i=0;i<
n_ptl;i++){
199 gid(i) = gid_offset + i+1;
204 for (
int j=0;j<6;j++) ph(i, j) = gid(i) + (j)*0.1;
205 for (
int j=0;j<3;j++) ct(i, j) = gid(i) + (j+6)*0.1;
212 for (
int j=0;j<6;j++) ph(i, j) = gid(i) + (j)*0.1;
213 for (
int j=0;j<3;j++) ct(i, j) = gid(i) + (j+6)*0.1;
228 particles_d = Cabana::AoSoA<ParticleDataTypes,Device,VEC_LEN>();
260 particles = Cabana::AoSoA<ParticleDataTypes,HostType,VEC_LEN>();
297 #ifdef PRINT_COPA_ARTIFACTS
298 if(
is_rank_zero()) printf(
"\nCopying Cabana AoSoA particle data to GPU with Cabana::deep_copy\n");
317 #ifdef PRINT_COPA_ARTIFACTS
318 if(
is_rank_zero()) printf(
"\nCopying Cabana AoSoA particle data from GPU with Cabana::deep_copy\n");
362 int last_ptl_index =
n_ptl - 1;
363 auto ph = Cabana::slice<PtlSlice::Ph>(
particles_d);
364 auto ct = Cabana::slice<PtlSlice::Ct>(
particles_d);
365 auto gid = Cabana::slice<PtlSlice::Gid>(
particles_d);
367 auto flag = Cabana::slice<PtlSlice::Flag>(
particles_d);
372 for (
int j=0;j<6;j++) ph(i, j) = ph(last_ptl_index, j);
373 for (
int j=0;j<3;j++) ct(i, j) = ct(last_ptl_index, j);
376 flag(i) = flag(last_ptl_index);
391 int last_ptl_index =
n_ptl - 1;
392 auto ph = Cabana::slice<PtlSlice::Ph>(
phase0_d);
396 for (
int j=0;j<6;j++) ph(i, j) = ph(last_ptl_index, j);
414 #ifdef PRINT_COPA_ARTIFACTS
415 if(
is_rank_zero()) printf(
"\nLaunching GPU kernel '%s' on Cabana AoSoA of particles\n", label.c_str());
421 inline void back_up_SoA(Cabana::AoSoA<ParticleDataTypes,Device,VEC_LEN>& backup_SoA,
int offset,
int n)
const{
422 auto ph_b = Cabana::slice<PtlSlice::Ph>(backup_SoA);
423 auto ct_b = Cabana::slice<PtlSlice::Ct>(backup_SoA);
424 auto gid_b = Cabana::slice<PtlSlice::Gid>(backup_SoA);
426 auto flag_b = Cabana::slice<PtlSlice::Flag>(backup_SoA);
429 auto ph = Cabana::slice<PtlSlice::Ph>(
particles_d);
430 auto ct = Cabana::slice<PtlSlice::Ct>(
particles_d);
431 auto gid = Cabana::slice<PtlSlice::Gid>(
particles_d);
433 auto flag = Cabana::slice<PtlSlice::Flag>(
particles_d);
436 Kokkos::parallel_for(
"backup_first_soa", Kokkos::RangePolicy<ExSpace>( 0, n ), KOKKOS_LAMBDA(
const int i ){
437 int i_offset = i + offset;
439 for (
int j=0;j<6;j++) ph_b(i, j) = ph(i_offset, j);
440 for (
int j=0;j<3;j++) ct_b(i, j) = ct(i_offset, j);
441 gid_b(i) = gid(i_offset);
443 flag_b(i) = flag(i_offset);
450 inline void restore_backup_SoA(Cabana::AoSoA<ParticleDataTypes,Device,VEC_LEN>& backup_SoA,
int offset,
int n)
const{
451 auto ph_b = Cabana::slice<PtlSlice::Ph>(backup_SoA);
452 auto ct_b = Cabana::slice<PtlSlice::Ct>(backup_SoA);
453 auto gid_b = Cabana::slice<PtlSlice::Gid>(backup_SoA);
455 auto flag_b = Cabana::slice<PtlSlice::Flag>(backup_SoA);
458 auto ph = Cabana::slice<PtlSlice::Ph>(
particles_d);
459 auto ct = Cabana::slice<PtlSlice::Ct>(
particles_d);
460 auto gid = Cabana::slice<PtlSlice::Gid>(
particles_d);
462 auto flag = Cabana::slice<PtlSlice::Flag>(
particles_d);
465 Kokkos::parallel_for(
"backup_first_soa", Kokkos::RangePolicy<ExSpace>( 0, n ), KOKKOS_LAMBDA(
const int i ){
466 int i_offset = i + offset;
468 for (
int j=0;j<6;j++) ph(i_offset, j) = ph_b(i, j);
469 for (
int j=0;j<3;j++) ct(i_offset, j) = ct_b(i, j);
470 gid(i_offset) = gid_b(i);
472 flag(i_offset) = flag_b(i);
483 inline void for_particle_range(
int begin_idx,
int end_idx,
const std::string label, F lambda_func)
const {
484 if(end_idx <= begin_idx)
return;
487 int first_soa = begin_idx/VEC_LEN;
488 int n_other_ptl_in_first_soa = begin_idx - first_soa*VEC_LEN;
489 bool first_soa_is_partial = (n_other_ptl_in_first_soa>0);
490 int last_soa = (end_idx-1)/VEC_LEN;
491 int n_other_ptl_in_last_soa = (last_soa+1)*VEC_LEN - end_idx;
492 bool last_soa_is_partial = (n_other_ptl_in_last_soa>0);
495 int first_item_in_shifted_range = first_soa*VEC_LEN;
497 int first_item_in_shifted_range = first_soa;
500 Cabana::AoSoA<ParticleDataTypes,Device,VEC_LEN> ptl_first_soa;
501 Cabana::AoSoA<ParticleDataTypes,Device,VEC_LEN> ptl_last_soa;
502 if(first_soa_is_partial){
504 ptl_first_soa = Cabana::AoSoA<ParticleDataTypes,Device,VEC_LEN>(
"ptl_first_soa", n_other_ptl_in_first_soa);
505 back_up_SoA(ptl_first_soa, first_soa*VEC_LEN, n_other_ptl_in_first_soa);
507 if(last_soa_is_partial){
509 ptl_last_soa = Cabana::AoSoA<ParticleDataTypes,Device,VEC_LEN>(
"ptl_last_soa", n_other_ptl_in_last_soa);
510 back_up_SoA(ptl_last_soa, end_idx, n_other_ptl_in_last_soa);
514 Kokkos::RangePolicy<ExSpace> particle_range_policy( first_item_in_shifted_range,
p_range<DeviceType>(end_idx) );
518 if(first_soa_is_partial){
521 if(last_soa_is_partial){
536 if(!
owns_particles_d)
exit_XGC(
"\nSpecies tried to loop over particles on device, but doesn't own the device array.");
538 #ifdef PRINT_COPA_ARTIFACTS
539 if(
is_rank_zero()) printf(
"\nLaunching GPU kernel '%s' on Cabana AoSoA of particles\n", label.c_str());
544 use_streaming =
false;
553 if((!send_ptl) && (!return_ptl)) use_streaming =
false;
568 #ifdef USE_EPUSH_LAUNCH_BOUNDS
569 # if !defined(PUSH_MAX_THREADS_PER_BLOCK) || !defined(PUSH_MIN_WARPS_PER_EU)
570 # error "USE_EPUSH_LAUNCH_BOUNDS requires PUSH_MAX_THREADS_PER_BLOCK and PUSH_MIN_WARPS_PER_EU to be defined"
572 Kokkos::RangePolicy<ExSpace, Kokkos::LaunchBounds<PUSH_MAX_THREADS_PER_BLOCK, PUSH_MIN_WARPS_PER_EU>>
576 exit_XGC(
"\nERROR: LaunchBounds::Custom specified, but USE_EPUSH_LAUNCH_BOUNDS is not defined\n");
579 Kokkos::RangePolicy<ExSpace>
601 for (
int i_simd = 0; i_simd<SIMD_SIZE; i_simd++){
602 int p_vec = inds.
a + i_simd;
603 ph0_loc[inds.
s].
r[p_vec] = ptl_loc[inds.
s].
ph.
r[p_vec];
604 ph0_loc[inds.
s].
z[p_vec] = ptl_loc[inds.
s].
ph.
z[p_vec];
605 ph0_loc[inds.
s].
phi[p_vec] = ptl_loc[inds.
s].
ph.
phi[p_vec];
606 ph0_loc[inds.
s].
rho[p_vec] = ptl_loc[inds.
s].
ph.
rho[p_vec];
607 ph0_loc[inds.
s].
w1[p_vec] = ptl_loc[inds.
s].
ph.
w1[p_vec];
608 ph0_loc[inds.
s].
w2[p_vec] = ptl_loc[inds.
s].
ph.
w2[p_vec];
656 backup_particles = Cabana::AoSoA<ParticleDataTypes,HostType,VEC_LEN>(
"backup_particles", 0);
674 for (
int i_simd = 0; i_simd<SIMD_SIZE; i_simd++){
675 int p_vec = inds.
a + i_simd;
676 part_one.
ph.
r[i_simd] = ph0_loc[inds.
s].
r[p_vec];
677 part_one.
ph.
z[i_simd] = ph0_loc[inds.
s].
z[p_vec];
678 part_one.
ph.
phi[i_simd] = ph0_loc[inds.
s].
phi[p_vec];
679 part_one.
ph.
rho[i_simd] = ph0_loc[inds.
s].
rho[p_vec];
680 part_one.
ph.
w1[i_simd] = ph0_loc[inds.
s].
w1[p_vec];
681 part_one.
ph.
w2[i_simd] = ph0_loc[inds.
s].
w2[p_vec];
687 long long int tmp_n_ptl =
n_ptl;
688 long long int out_n_ptl = 0;
689 MPI_Allreduce(&tmp_n_ptl, &out_n_ptl, 1, MPI_LONG_LONG_INT, MPI_SUM,
SML_COMM_WORLD);
692 return (
long long int)(
n_ptl);
698 int tmp_n_ptl =
n_ptl;
700 MPI_Allreduce(&tmp_n_ptl, &out_n_ptl, 1, MPI_INT, MPI_MAX,
SML_COMM_WORLD);
760 for (
int i_simd = 0; i_simd<SIMD_SIZE; i_simd++){
761 if(not_in_triangle[i_simd])
continue;
764 not_in_poloidal_domain[i_simd] = (nearest_node[i_simd]<0 || nearest_node[i_simd]>=pol_decomp.
nnodes);
766 double temp_ev_norm = not_in_poloidal_domain[i_simd] ?
f0.temp_ev(0) :
f0.temp_ev(nearest_node[i_simd]);
769 const double& B = bmag[i_simd];
Cabana::AoSoA< PhaseDataTypes, HostType, VEC_LEN > phase0
Definition: species.hpp:111
bool stream_particles
Whether to stream particles between host and device if possible.
Definition: species.hpp:103
Definition: globals.hpp:78
KOKKOS_INLINE_FUNCTION VecPhase * ph0() const
Definition: species.hpp:592
KOKKOS_INLINE_FUNCTION int divide_and_round_up(int a, int b)
Definition: globals.hpp:173
bool owns_particles_d
Whether the species owns the device particle allocation right now.
Definition: species.hpp:100
void back_up_SoA(Cabana::AoSoA< ParticleDataTypes, Device, VEC_LEN > &backup_SoA, int offset, int n) const
Definition: species.hpp:421
KOKKOS_INLINE_FUNCTION double normalized_v_para(double c_m, double mass, double B, double temp_ev, double rho)
Definition: basic_physics.hpp:51
KOKKOS_INLINE_FUNCTION double get_f0_eq_gyro_radius(int inode, double smu_n, double bfield) const
Definition: species.hpp:711
subroutine adjust_n_ptl_for_core_ptl(n_ptl)
Definition: load.F90:453
void set_spall_num_and_ptr(int idx, int n_ptl, int n_vecs, VecParticles *ptl)
bool is_rank_zero()
Definition: globals.hpp:27
void for_particle_range(int begin_idx, int end_idx, const std::string label, F lambda_func) const
Definition: species.hpp:483
constexpr double PROTON_MASS
Definition: constants.hpp:7
Distribution< Device > f0
Species distribution in velocity space on local mesh nodes.
Definition: species.hpp:119
MPI_Comm SML_COMM_WORLD
Definition: my_mpi.cpp:4
Cabana::AoSoA< ParticleDataTypes, HostType, VEC_LEN > backup_particles
Copy of particles to be restored for RK2.
Definition: species.hpp:115
bool is_electron
Whether this species is the electrons.
Definition: species.hpp:78
void for_all_particles(const std::string label, F lambda_func, const PtlMvmt mvmt, LaunchBounds launch_bounds=LaunchBounds::Default)
Definition: species.hpp:534
void save_backup_particles()
Definition: species.hpp:613
double c2_2m
c2/2m
Definition: species.hpp:86
double rho[VEC_LEN]
Definition: particles.hpp:72
void copy_to_phase0(Species< Device > &species)
Definition: species.hpp:596
Definition: species.hpp:57
Simd< double > w1
Definition: particles.hpp:22
double c_m
c/m
Definition: species.hpp:85
Definition: species.hpp:56
constexpr double EV_2_J
Conversion rate ev to J.
Definition: constants.hpp:5
bool default_streaming_option()
Definition: species.hpp:27
Eq::Profile< Device > eq_den
Definition: species.hpp:124
Definition: globals.hpp:83
KOKKOS_INLINE_FUNCTION VecParticles * ptl() const
Definition: species.hpp:588
KOKKOS_INLINE_FUNCTION double thermal_velocity(double mass, double temp_ev)
Definition: basic_physics.hpp:29
Definition: NamelistReader.hpp:193
KinType kintype
Whether the species is gyrokinetic or drift kinetic.
Definition: species.hpp:81
Definition: magnetic_field.hpp:12
int add_vec_buffer(int n_ptl)
Definition: particles.hpp:164
int idx
Index in all_species.
Definition: species.hpp:77
Definition: particles.hpp:68
int a
The index in the inner array of the AoSoA.
Definition: particles.hpp:120
Definition: particles.hpp:85
KOKKOS_INLINE_FUNCTION void bmag_interpol(const SimdVector &v, Simd< double > &bmag) const
Definition: magnetic_field.tpp:257
bool particles_are_backed_up
Whether particles are currently backed up.
Definition: species.hpp:108
int nonadiabatic_idx
Index of species skipping adiabatic species (for compatibility with fortran arrays) ...
Definition: species.hpp:80
bool default_residence_option()
Definition: species.hpp:34
int n_ptl
Number of particles.
Definition: species.hpp:94
KOKKOS_INLINE_FUNCTION void get_particle_velocity_and_nearest_node(const Grid< DeviceType > &grid, const MagneticField< DeviceType > &magnetic_field, const DomainDecomposition< DeviceType > &pol_decomp, SimdParticles &part, Simd< double > &smu, Simd< double > &vp, Simd< int > &nearest_node, Simd< bool > ¬_in_triangle, Simd< bool > ¬_in_poloidal_domain) const
Definition: species.hpp:735
Definition: streamed_parallel_for.hpp:16
KOKKOS_INLINE_FUNCTION void maxloc(Simd< int > &ml) const
Definition: grid_structs.hpp:16
int node_offset
Offset of first mesh node belonging to this MPI rank.
Definition: domain_decomposition.hpp:41
Definition: streamed_parallel_for.hpp:14
void set_buffer_phase0_d()
Definition: species.hpp:389
long long int get_total_n_ptl()
Definition: species.hpp:685
void set_buffer_particles_d()
Definition: species.hpp:360
Simd< double > rho
Definition: particles.hpp:21
Definition: species.hpp:58
int p_range< DeviceType >(int num_particle)
Definition: particles.hpp:157
int eq_flow_type
Definition: species.hpp:126
double charge_eu
Particle charge in eu.
Definition: species.hpp:84
KOKKOS_INLINE_FUNCTION void get_psi(const SimdVector2D &x, Simd< double > &psi_out) const
Definition: magnetic_field.tpp:82
Definition: species.hpp:50
int nnodes
Number of nodes belonging to this MPI rank.
Definition: domain_decomposition.hpp:42
void resize_particles(int new_n_ptl)
Definition: species.hpp:222
double mass
Particle mass.
Definition: species.hpp:82
KOKKOS_INLINE_FUNCTION double get_f0_eq_thermal_velocity(int inode) const
Definition: species.hpp:718
Species(int idx_in, int nonadiabatic_idx_in, bool is_electron_in, bool is_adiabatic_in, KinType kintype_in, double mass_in, double charge_in, double charge_eu_in, bool is_deltaf_in, int ncycles_in)
Definition: species.tpp:23
void for_all_particles(const std::string label, F lambda_func) const
Definition: species.hpp:413
Cabana::AoSoA< ParticleDataTypes, Device, VEC_LEN > particles_d
Particles on device.
Definition: species.hpp:98
Definition: species.hpp:47
double w2[VEC_LEN]
Definition: particles.hpp:74
#define TIMER(N, F)
Definition: timer_macro.hpp:24
RKRestorationMethod
Definition: species.hpp:67
idx
Definition: diag_f0_df_port1.hpp:32
void copy_particles_to_device_if_not_resident()
Definition: species.hpp:343
RKRestorationMethod RK_restoration_method
Currently, electrons must use first method and ions must use second.
Definition: species.hpp:106
Simd< double > r
Definition: particles.hpp:18
KOKKOS_INLINE_FUNCTION SimdVector2D & x()
Definition: particles.hpp:27
void resize_host_particles_to_match_device()
Definition: species.hpp:242
Definition: species.hpp:68
KOKKOS_INLINE_FUNCTION SimdVector & v()
Definition: particles.hpp:39
ReturnOpt return_opt
Definition: species.hpp:62
KOKKOS_INLINE_FUNCTION int get_node_index(int triangle_index, int tri_vertex_index) const
Definition: grid.tpp:792
ReturnOpt
Definition: species.hpp:55
Option
Definition: streamed_parallel_for.hpp:13
void restore_particles_from_backup()
Definition: species.hpp:634
Definition: globals.hpp:84
static std::vector< MemoryPrediction > estimate_memory_usage(NLReader::NamelistReader &nlr, const Grid< DeviceType > &grid, const DomainDecomposition< DeviceType > &pol_decomp, int species_idx)
Definition: species.tpp:58
SendOpt send_opt
Definition: species.hpp:61
double charge
Particle charge.
Definition: species.hpp:83
SimdPhase ph
Definition: particles.hpp:59
void copy_particles_from_device()
Definition: species.hpp:314
KOKKOS_INLINE_FUNCTION double get_f0_eq_thermal_velocity_lnode(int inode) const
Definition: species.hpp:724
void copy_particles_from_device_if_not_resident()
Definition: species.hpp:349
KOKKOS_INLINE_FUNCTION void wedge_modulo_phi(Simd< double > &phi_mod) const
Definition: grid.tpp:673
void unassign_host_particles()
Definition: species.hpp:259
int ncycles_between_sorts
Number of subcycles between sorts.
Definition: species.hpp:91
Definition: particles.hpp:58
Cabana::AoSoA< PhaseDataTypes, Device, VEC_LEN > phase0_d
Definition: species.hpp:112
int SML_COMM_RANK
Definition: my_mpi.cpp:5
KinType
Definition: globals.hpp:82
Species(SpeciesType sp_type, int n_ptl)
Definition: species.hpp:138
bool is_deltaf
Whether this species is deltaf.
Definition: species.hpp:88
VecPhase ph
Definition: particles.hpp:86
Definition: grid_structs.hpp:7
Definition: species.hpp:43
Definition: species.hpp:69
void set_min_max_num(int isp, int n_ptl)
int minimum_ptl_reservation
The minimum reservation size for particles.
Definition: species.hpp:93
int s
The index in the outer array of the AoSoA.
Definition: particles.hpp:119
KOKKOS_INLINE_FUNCTION double normalized_sqrt_mu(double B, double temp_ev, double mu)
Definition: basic_physics.hpp:61
Simd< double > z
Definition: particles.hpp:19
void copy_particles_to_device_if_resident()
Definition: species.hpp:331
Definition: species.hpp:51
void resize_device_particles(int new_n_ptl)
Definition: species.hpp:280
Definition: species.hpp:52
constexpr double UNIT_CHARGE
Charge of an electron (C)
Definition: constants.hpp:4
KOKKOS_INLINE_FUNCTION double get_f0_eq_thermal_velocity_lnode_h(int inode) const
Definition: species.hpp:730
void exit_XGC(std::string msg)
Definition: globals.hpp:37
void copy_particles_from_device_if_resident()
Definition: species.hpp:337
bool is_adiabatic
Whether this species is adiabatic.
Definition: species.hpp:79
Simd< double > phi
Definition: particles.hpp:20
Definition: magnetic_field.F90:1
static constexpr const Kokkos::Experimental::WorkItemProperty::HintLightWeight_t Async
Definition: space_settings.hpp:82
int n_backup_particles
Definition: species.hpp:116
Eq::Profile< Device > eq_flow
Definition: species.hpp:125
Definition: streamed_parallel_for.hpp:15
SendOpt
Definition: species.hpp:49
SimdConstants ct
Definition: particles.hpp:60
void copy_particles_to_device()
Definition: species.hpp:294
KOKKOS_INLINE_FUNCTION void restore_phase_from_phase0(const AoSoAIndices< Device > &inds, SimdParticles &part_one) const
Definition: species.hpp:672
Species(int n_ptl_in)
Definition: species.hpp:168
double phi[VEC_LEN]
Definition: particles.hpp:71
Simd< double > w2
Definition: particles.hpp:23
double r[VEC_LEN]
Definition: particles.hpp:69
GyroAverageMatrices< HostType > gyro_avg_matrices
Definition: species.hpp:128
Definition: species.hpp:74
void parallel_for(const std::string name, int n_ptl, Function func, Option option, HostAoSoA aosoa_h, DeviceAoSoA aosoa_d)
Definition: streamed_parallel_for.hpp:252
Definition: species.hpp:43
Eq::Profile< Device > eq_temp
Definition: species.hpp:123
bool particles_resident_on_device
Whether the particles can reside on device.
Definition: species.hpp:102
Simd< double > mu
Definition: particles.hpp:52
PtlMvmt(SendOpt send_opt, ReturnOpt return_opt)
Definition: species.hpp:64
static int get_initial_n_ptl(NLReader::NamelistReader &nlr, const Grid< DeviceType > &grid, const DomainDecomposition< DeviceType > &pol_decomp, int sml_special, int species_idx, bool verbose)
Definition: species.tpp:115
int ncycles
Number of subcycles.
Definition: species.hpp:90
Definition: profile.hpp:171
int collision_grid_index
Which collision grid to use.
Definition: species.hpp:121
Definition: particles.hpp:118
SpeciesType
Definition: globals.hpp:77
double z[VEC_LEN]
Definition: particles.hpp:70
void restore_backup_SoA(Cabana::AoSoA< ParticleDataTypes, Device, VEC_LEN > &backup_SoA, int offset, int n) const
Definition: species.hpp:450
Definition: distribution.hpp:10
KOKKOS_INLINE_FUNCTION void check_triangle(const Simd< int > &itr, Simd< bool > &wasnt_in_triangle) const
Definition: grid.tpp:692
void resize_device_particles()
Definition: species.hpp:265
int get_max_n_ptl()
Definition: species.hpp:696
LaunchBounds
Definition: species.hpp:402
KOKKOS_INLINE_FUNCTION void charge_search_index(const MagneticField< Device > &magnetic_field, const SimdVector2D &x, const Simd< double > &phi, const Simd< double > &psi, SimdVector2D &xff, Simd< int > &itr, SimdGridVec &p) const
Definition: grid.tpp:591
Cabana::AoSoA< ParticleDataTypes, HostType, VEC_LEN > particles
Particles.
Definition: species.hpp:95
double w1[VEC_LEN]
Definition: particles.hpp:73
Definition: species.hpp:43