domain_decomposition.hpp

Go to the documentation of this file.

#ifndef DOMAIN_DECOMPOSITION_HPP
#define DOMAIN_DECOMPOSITION_HPP
#include "space_settings.hpp"
#include "NamelistReader.hpp"
#ifdef USE_MPI
#  include "my_mpi.hpp"
#endif
#include "field_decomposition.hpp"
#include "grid_weights.hpp"

#include "vertex_list.hpp"
#include "host_array.hpp"

struct DistributionPlan{
    View<int*,CLayout,HostType> cnts;
    View<int*,CLayout,HostType> displs;
    int my_rank;

    DistributionPlan(int my_rank_in, int nranks)
      : cnts(NoInit("cnts"), nranks),
        displs(NoInit("displs"), nranks),
        my_rank(my_rank_in)
    {}

    int my_count() const{
        return cnts(my_rank);
    }

    int my_displ() const{
        return displs(my_rank);
    }
};

// Decomposition class
template<class Device>
class DomainDecomposition {
    public:

    // Constructor
    DomainDecomposition(NLReader::NamelistReader& nlr);

    DomainDecomposition(NLReader::NamelistReader& nlr, int nnodes_in);

    DomainDecomposition(int nranks_in);

    void setup_gvid0_pid(NLReader::NamelistReader& nlr, int nnodes_in);

    int nplanes;

#ifdef USE_MPI
    // MPI communicators etc
    const MyMPI mpi;
#endif

    // Updater
    void update(int new_f0_inode1);

    void update_pol_decomp();

    void update_flux_surf(const HostArray<VertexList>& surfaces);

    KOKKOS_INLINE_FUNCTION int node_to_plane_rank(int node) const;

    KOKKOS_INLINE_FUNCTION void node_to_rank(const SimdGridWeights<Order::Zero, PIT_GLOBAL>& grid_wts0, const Simd<int>& iphi, bool plane_sort_only, Simd<int>& pid) const;

    int node_to_plane_rank_h(int node, int initial_guess) const;

    DistributionPlan mpi_distribution_plan(int nv) const;

    VertexList vertex_list() const;

    VertexList vertex_list(int ipid) const;

    // Constants
    bool pol_decomp;     
    int pe_per_plane;          
    int plane_totalpe;         
    int n_pids;                
    int my_pid;                
    int my_plane_pid;          
    int plane_index;     
    int nnodes_on_plane;  

    bool decompose_fields; 
    FieldDecomposition<Device> field_decomp;

    // Variables
    int node_offset;  
    int nnodes;      

    // Views
    Kokkos::View<int*,Kokkos::LayoutRight,HostType> gvid0_pid_h; 
    Kokkos::View<int*,Kokkos::LayoutRight,Device> gvid0_pid; 

    View<int*,CLayout,HostType> surfs; 
    View<int*,CLayout,HostType> ranks_sharing_flux_surf; 
    
};

#include "domain_decomposition.tpp"
#endif