domain_decomposition.hpp

Go to the documentation of this file.

#ifndef DOMAIN_DECOMPOSITION_HPP
#define DOMAIN_DECOMPOSITION_HPP
#include "space_settings.hpp"
#include "NamelistReader.hpp"
#ifdef USE_MPI
#  include "my_mpi.hpp"
#endif
#include "field_decomposition.hpp"
#include "grid_weights.hpp"
#include "vertex_list.hpp"
 
#include "vertex_list.hpp"
#include "host_array.hpp"
#include "xgc_io.hpp"
 
struct DistributionPlan{
    View<int*,CLayout,HostType> cnts;
    View<int*,CLayout,HostType> displs;
    int my_rank;
 
    DistributionPlan(){}
 
    DistributionPlan(int my_rank_in, int nranks)
      : cnts(NoInit("cnts"), nranks),
        displs(NoInit("displs"), nranks),
        my_rank(my_rank_in)
    {}
 
    int my_count() const{
        return cnts(my_rank);
    }
 
    int my_displ() const{
        return displs(my_rank);
    }
};
 
// Decomposition class
template<class Device>
class DomainDecomposition {
    public:
 
    // Constructor
    DomainDecomposition(NLReader::NamelistReader& nlr);
 
    DomainDecomposition(NLReader::NamelistReader& nlr, int nnodes_in);
 
    DomainDecomposition(int nranks_in);
 
    void setup_gvid0_pid(NLReader::NamelistReader& nlr, int nnodes_in);
 
    int nplanes;
 
#ifdef USE_MPI
    // MPI communicators etc
    const MyMPI mpi;
#endif
 
    // Updater
    void update(int new_f0_inode1);
 
    void update_pol_decomp();
 
    void update_flux_surf(const HostArray<VertexList>& surfaces);
 
    KOKKOS_INLINE_FUNCTION int node_to_plane_rank(int node) const;
 
    KOKKOS_INLINE_FUNCTION void node_to_rank(const SimdGridWeights<Order::Zero, PIT_GLOBAL>& grid_wts0, const Simd<int>& iphi, bool plane_sort_only, Simd<int>& pid) const;
 
    int node_to_plane_rank_h(int node, int initial_guess) const;
 
    DistributionPlan mpi_distribution_plan(int nv) const;
 
    VertexList vertex_list() const;
 
    VertexList vertex_list(int ipid) const;
 
    void write_checkpoint_files(const XGC_IO_Stream& stream) const;
    void read_checkpoint_files(const XGC_IO_Stream& stream);
 
    // Constants
    bool pol_decomp;     
    int pe_per_plane;          
    int plane_totalpe;         
    int n_pids;                
    int my_pid;                
    int my_plane_pid;          
    int plane_index;     
    int nnodes_on_plane;  
 
    bool decompose_fields; 
    FieldDecomposition<Device> field_decomp;
 
    // Variables
    int node_offset;  
    int nnodes;      
 
    // Views
    Kokkos::View<int*,Kokkos::LayoutRight,HostType> gvid0_pid_h; 
    Kokkos::View<int*,Kokkos::LayoutRight,Device> gvid0_pid; 
 
    View<int*,CLayout,HostType> surfs; 
    View<int*,CLayout,HostType> ranks_sharing_flux_surf; 
    
};
 
#include "domain_decomposition.tpp"
#endif