54 scatter_to_seq_(nullptr),
59 if (
dynamic_cast<Mesh*
>(
mesh_) !=
nullptr) {
105 unsigned int n_node_dofs = 0;
108 node_dof_starts.push_back(n_node_dofs);
109 n_node_dofs +=
ds_->n_node_dofs(nid);
111 node_dof_starts.push_back(n_node_dofs);
113 unsigned int n_edge_dofs = 0;
116 edge_dof_starts.push_back(n_edge_dofs);
117 n_edge_dofs +=
ds_->n_edge_dofs(
edge);
119 edge_dof_starts.push_back(n_edge_dofs);
130 for (
unsigned int n=0; n<cell.dim()+1; n++)
142 for (
unsigned int n=0; n<cell.dim()+1; n++)
161 for (
unsigned int n=0; n<cell.dim()+1; n++)
163 unsigned int eid = cell.elm().side(n)->edge_idx();
192 unsigned int n_dofs_sum = 0;
193 for (
auto nd : n_dofs) n_dofs_sum += nd;
194 dofs.resize(n_dofs_sum);
202 unsigned int n_elems;
239 unsigned int dof_offset=0;
240 for (
unsigned int gid=0; gid<
ghost_proc_el[proc].size(); gid++)
246 for (
unsigned int idof = 0; idof<dh_cell.
n_dofs(); ++idof)
252 unsigned int node_dof_idx = node_dof_starts[nid]+loc_node_dof_count[dof_nface_idx];
261 loc_node_dof_count[dof_nface_idx]++;
269 unsigned int edge_dof_idx = edge_dof_starts[eid]+loc_edge_dof_count[dof_nface_idx];
278 loc_edge_dof_count[dof_nface_idx]++;
286 dof_offset += dh_cell.
n_dofs();
292 if (!update_cells[cell.local_idx()])
continue;
297 for (
unsigned int idof = 0; idof<cell.n_dofs(); ++idof)
299 unsigned int dof_nface_idx = cell.cell_dof(idof).n_face_idx;
300 if (cell.cell_dof(idof).dim == 0)
305 dof_indices[
cell_starts[cell.local_idx()]+idof] = node_dofs[node_dof_starts[nid]+loc_node_dof_count[dof_nface_idx]];
307 loc_node_dof_count[dof_nface_idx]++;
308 }
else if (cell.cell_dof(idof).dim == cell.dim()-1)
314 unsigned int eid = cell.elm().side(dof_nface_idx)->edge_idx();
315 dof_indices[
cell_starts[cell.local_idx()]+idof] = edge_dofs[edge_dof_starts[eid]+loc_edge_dof_count[dof_nface_idx]];
317 loc_edge_dof_count[dof_nface_idx]++;
335 unsigned int next_free_dof = 0;
352 for (
unsigned int idof = 0; idof<cell.n_dofs(); ++idof)
354 unsigned int dof_dim = cell.cell_dof(idof).dim;
355 unsigned int dof_nface_idx = cell.cell_dof(idof).n_face_idx;
360 unsigned int node_dof_idx = node_dof_starts[nid]+loc_node_dof_count[dof_nface_idx];
362 switch (node_status[nid])
365 for (
int i=0; i<node_dof_starts[nid+1] - node_dof_starts[nid]; i++)
368 node_dofs[node_dof_starts[nid]+i] = next_free_dof++;
374 update_cells[cell.local_idx()] =
true;
378 loc_node_dof_count[dof_nface_idx]++;
380 else if (dof_dim == cell.dim()-1)
384 unsigned int eid = cell.elm().side(dof_nface_idx)->edge_idx();
385 unsigned int edge_dof_idx = edge_dof_starts[eid]+loc_edge_dof_count[dof_nface_idx];
386 switch (edge_status[eid])
389 for (
int i=0; i<edge_dof_starts[eid+1] - edge_dof_starts[eid]; i++)
392 edge_dofs[edge_dof_starts[eid]+i] = next_free_dof++;
398 update_cells[cell.local_idx()] =
true;
402 loc_edge_dof_count[dof_nface_idx]++;
404 else if (dof_dim == cell.dim())
419 dof_ds_ = std::make_shared<Distribution>(
lsize_, PETSC_COMM_WORLD);
432 for (
unsigned int from_higher = 0; from_higher < 2; from_higher++)
436 if ((proc >
el_ds_->
myp()) == from_higher)
456 update_cells.clear();
458 node_dof_starts.clear();
460 edge_dof_starts.clear();
466 if (
dh_seq_ !=
nullptr)
return;
470 dh_seq_ = std::make_shared<DOFHandlerMultiDim>(*
this);
507 cell_starts_loc[i+1] += cell_starts_loc[i];
513 for (
unsigned int idof=0; idof<cell.n_dofs(); idof++)
523 cell_starts_loc.size(),
529 dof_indices_loc.size(),
532 (
const int *)
distr.get_lsizes_array(),
533 (
const int *)
distr.get_starts_array(),
539 VecCreateMPI(PETSC_COMM_WORLD,
lsize_, PETSC_DETERMINE, &v_from);
547 dh_seq_->scatter_to_seq_ = std::make_shared<VecScatter>();
548 VecScatterCreateToAll(v_seq,
dh_seq_->scatter_to_seq_.get(), NULL);
569 unsigned int ndofs = 0;
571 indices.resize(ndofs);
572 for (
unsigned int k=0; k<ndofs; k++)
592 bool is_edge_local =
false;
593 for (
uint sid=0; sid<
edge.n_sides(); sid++)
596 is_edge_local =
true;
613 for (
unsigned int iel = 0; iel <
el_ds_->
lsize(); iel++ )
621 for (
unsigned int nid=0; nid<cell.elm()->n_nodes(); nid++)
630 bool has_local_node =
false;
632 for (
unsigned int nid=0; nid<cell->n_nodes(); nid++)
635 has_local_node =
true;
681 auto bgn_it = make_iter<DHCellAccessor>(
DHCellAccessor(
this, 0) );
688 auto bgn_it = make_iter<DHCellAccessor>(
DHCellAccessor(
this, 0) );
712 for (
unsigned int idof=0; idof<ndofs; idof++)
714 s << dofs[idof] <<
" ";
724 #ifdef FLOW123D_DEBUG_MESSAGES
728 s <<
"DOFHandlerMultiDim structure:" << endl;
729 s <<
"- is parallel: " << (
is_parallel_?
"true":
"false") << endl;
730 s <<
"- proc id: " <<
el_ds_->
myp() << endl;
732 s <<
"- number of locally owned cells: " <<
el_ds_->
lsize() << endl;
733 s <<
"- number of ghost cells: " <<
ghost_4_loc.size() << endl;
735 s <<
"- dofs on locally owned cells:" << endl;
738 s <<
"- dofs on ghost cells:" << endl;
741 s <<
"- locally owned dofs (" <<
lsize_ <<
"): ";
745 s <<
"- global-to-local-cell map:" << endl;
749 printf(
"%s", s.str().c_str());
766 fe_idx_(component_idx)
770 .error(
"sub_handler can be used only with dof handler using EqualOrderDiscreteSpace!");
775 ASSERT( fe_sys0 !=
nullptr ).error(
"sub_handler assumes that dof handler uses FESystem<0>!");
776 ASSERT( fe_sys1 !=
nullptr ).error(
"sub_handler assumes that dof handler uses FESystem<1>!");
777 ASSERT( fe_sys2 !=
nullptr ).error(
"sub_handler assumes that dof handler uses FESystem<2>!");
778 ASSERT( fe_sys3 !=
nullptr ).error(
"sub_handler assumes that dof handler uses FESystem<3>!");
780 fe_sys1->
fe()[component_idx],
781 fe_sys2->
fe()[component_idx],
782 fe_sys3->
fe()[component_idx]);
783 ds_ = std::make_shared<EqualOrderDiscreteSpace>(
mesh_, sub_mixed);
793 for (
unsigned int d=0; d<=3; d++)
794 ASSERT( fs[d] !=
nullptr ).error(
"Function space must be of type FESystemFunctionSpace!" );
796 for (
unsigned int i=0; i<fe_sys0->
n_dofs(); i++)
797 if (fs[0]->
dof_indices()[i].fe_index == component_idx) sub_fe_dofs[0].push_back(i);
798 for (
unsigned int i=0; i<fe_sys1->
n_dofs(); i++)
799 if (fs[1]->
dof_indices()[i].fe_index == component_idx) sub_fe_dofs[1].push_back(i);
800 for (
unsigned int i=0; i<fe_sys2->
n_dofs(); i++)
801 if (fs[2]->
dof_indices()[i].fe_index == component_idx) sub_fe_dofs[2].push_back(i);
802 for (
unsigned int i=0; i<fe_sys3->
n_dofs(); i++)
803 if (fs[3]->
dof_indices()[i].fe_index == component_idx) sub_fe_dofs[3].push_back(i);
812 for (
auto cell : dh->local_range())
814 LocDofVec cell_dof_indices = cell.get_loc_dof_indices();
815 for (
auto sub_dof : sub_fe_dofs[cell.dim()])
817 if (cell_dof_indices[sub_dof] <
static_cast<int>(dh->lsize_) &&
818 sub_local_indices[cell_dof_indices[sub_dof]] ==
INVALID_DOF)
829 for (
auto cell : dh->ghost_range())
831 LocDofVec cell_dof_indices = cell.get_loc_dof_indices();
832 for (
auto sub_dof : sub_fe_dofs[cell.dim()])
834 if (sub_local_indices[cell_dof_indices[sub_dof]] ==
INVALID_DOF)
841 ghost_dof_proc.push_back(cell.elm().proc());
846 for (
auto cell : dh->local_range())
848 LocDofVec cell_dof_indices = cell.get_loc_dof_indices();
849 unsigned int idof = 0;
850 for (
auto sub_dof : sub_fe_dofs[cell.dim()])
854 dof_ds_ = std::make_shared<Distribution>(
lsize_, PETSC_COMM_WORLD);
860 for (
unsigned int i=0; i<
lsize_; i++)
865 for (
unsigned int from_higher = 0; from_higher < 2; from_higher++)
869 if ((proc >
el_ds_->
myp()) == from_higher)
886 if (ghost_dof_proc[i-
lsize_] == proc)
895 dofs.resize(n_ghosts);
899 unsigned int idof = 0;
901 if (ghost_dof_proc[i-
lsize_] == proc)
913 unsigned int n_ghosts;
923 dofs.push_back(global_to_local_dof_idx.at(global_dof) +
dof_ds_->begin());
930 ASSERT(
vec.size() ==
parent_->local_to_global_dof_idx_.size() ).error(
"Incompatible parent vector in update_subvector()!");
940 ASSERT(
vec.size() ==
parent_->local_to_global_dof_idx_.size() ).error(
"Incompatible parent vector in update_subvector()!");
#define ASSERT_PERMANENT(expr)
Allow use shorter versions of macro names if these names is not used with external library.
Cell accessor allow iterate over DOF handler cells.
const ElementAccessor< 3 > elm() const
Return ElementAccessor to element of loc_ele_idx_.
unsigned int n_dofs() const
Return number of dofs on given cell.
const Dof & cell_dof(unsigned int idof) const
Return dof on a given cell.
unsigned int get_dof_indices(std::vector< LongIdx > &indices) const
Fill vector of the global indices of dofs associated to the cell.
unsigned int dim() const
Return dimension of element appropriate to cell.
unsigned int elm_idx() const
Return serial idx to element of loc_ele_idx_.
unsigned int local_idx() const
Return local index to element (index of DOF handler).
unsigned int max_elem_dofs_
Max. number of dofs per element.
MeshBase * mesh_
Pointer to the mesh to which the dof handler is associated.
unsigned int lsize_
Number of dofs associated to local process.
unsigned int n_global_dofs_
Number of global dofs assigned by the handler.
unsigned int loffset_
Index of the first dof on the local process.
std::shared_ptr< Distribution > dof_ds_
Distribution of dofs associated to local process.
virtual ~DOFHandlerBase()
Destructor.
std::shared_ptr< Distribution > distr() const
Provides the numbering of the finite element degrees of freedom on the computational mesh.
static const int ASSIGNED_NFACE
static const int INVALID_NFACE
vector< LongIdx > nb_4_loc
Local neighbour index -> global neighbour index.
Distribution * el_ds_
Distribution of elements.
std::shared_ptr< DiscreteSpace > ds_
Pointer to the discrete space for which the handler distributes dofs.
std::shared_ptr< VecScatter > scatter_to_seq_
Scatter context for parallel to sequential vectors.
unsigned int get_dof_indices(const DHCellAccessor &cell, std::vector< LongIdx > &indices) const override
Returns the global indices of dofs associated to the cell.
Range< DHCellAccessor > own_range() const
Returns range of DOF handler cells (only range of own without ghost cells)
void make_elem_partitioning()
Prepare parallel distribution of elements, edges and neighbours.
void update_local_dofs(unsigned int proc, const std::vector< bool > &update_cells, const std::vector< LongIdx > &dofs, const std::vector< LongIdx > &node_dof_starts, std::vector< LongIdx > &node_dofs, const std::vector< LongIdx > &edge_dof_starts, std::vector< LongIdx > &edge_dofs)
Update dofs on local elements from ghost element dofs.
vector< LongIdx > ghost_4_loc
Indices of ghost cells (neighbouring with local elements).
bool distribute_edge_dofs
Temporary flag which prevents using dof handler on meshes where edges are not allocated (currently BC...
std::vector< LongIdx > cell_starts
Starting indices for local (owned+ghost) element dofs.
friend class SubDOFHandlerMultiDim
std::size_t hash() const override
bool el_is_local(int index) const
friend class DHCellAccessor
vector< LongIdx > edg_4_loc
Local edge index -> global edge index.
bool is_parallel_
Indicator for parallel/sequential dof handler.
static const int INVALID_DOF
set< unsigned int > ghost_proc
Processors of ghost elements.
void distribute_dofs(std::shared_ptr< DiscreteSpace > ds)
Distributes degrees of freedom on the mesh needed for the given discrete space.
std::vector< IntIdx > dof_indices
Dof numbers on local and ghost elements.
std::shared_ptr< DOFHandlerMultiDim > sequential()
Returns sequential version of the current dof handler.
static const int VALID_NFACE
std::shared_ptr< DOFHandlerMultiDim > dh_seq_
Sequential dof handler associated to the current (parallel) one.
void print_cell_dofs(stringstream &s, DHCellAccessor cell) const
Auxiliary function for output of information about dofs in a cell.
virtual VectorMPI create_vector()
Allocates PETSc vector according to the dof distribution.
void init_cell_starts()
Initialize vector of starting indices for elements.
DOFHandlerMultiDim(MeshBase &_mesh, bool make_elem_part=true)
Constructor.
~DOFHandlerMultiDim() override
Destructor.
void create_sequential()
Communicate local dof indices to all processors and create new sequential dof handler.
const DHCellAccessor cell_accessor_from_element(unsigned int elm_idx) const
Return DHCellAccessor appropriate to ElementAccessor of given idx.
void print() const
Output structure of dof handler.
Range< DHCellAccessor > local_range() const
Returns range over own and ghost cells of DOF handler.
Range< DHCellAccessor > ghost_range() const
Returns range over ghosts DOF handler cells.
void init_status(std::vector< short int > &node_status, std::vector< short int > &edge_status)
Initialize node_status and edge_status.
void init_dof_starts(std::vector< LongIdx > &node_dof_starts, std::vector< LongIdx > &edge_dof_starts)
Initialize auxiliary vector of starting indices of nodal/edge dofs.
std::vector< LongIdx > local_to_global_dof_idx_
Maps local and ghost dof indices to global ones.
void send_ghost_dofs(unsigned int proc)
Send dof numbers to other processor.
std::shared_ptr< DiscreteSpace > ds() const
Return pointer to discrete space for which the handler distributes dofs.
std::unordered_map< LongIdx, LongIdx > global_to_local_el_idx_
Maps global element index into local/ghost index (obsolete).
std::shared_ptr< VecScatter > sequential_scatter()
Returns scatter context from parallel to sequential vectors.
void receive_ghost_dofs(unsigned int proc, std::vector< LongIdx > &dofs)
Obtain dof numbers on ghost elements from other processor.
map< unsigned int, vector< LongIdx > > ghost_proc_el
Arrays of ghost cells for each neighbouring processor.
bool is_local(unsigned int idx) const
identify local index
unsigned int myp() const
get my processor
unsigned int lsize(int proc) const
get local size
unsigned int np() const
get num of processors
unsigned int n_face_idx
Index of n-face to which the dof is associated.
unsigned int dim
Association to n-face of given dimension (point, line, triangle, tetrahedron.
unsigned int n_nodes() const
const std::vector< MeshObject > & objects(unsigned int dim) const
const std::vector< unsigned int > & obj_4_el() const
unsigned int input_id() const
Return the element ID in the input mesh. Should be only used for special output.
NodeAccessor< 3 > node(unsigned int ni) const
SideIter side(const unsigned int loc_index)
unsigned int idx() const
We need this method after replacing Region by RegionIdx, and movinf RegionDB instance into particular...
unsigned int n_sides() const
unsigned int n_nodes() const
Compound finite element on dim dimensional simplex.
const std::vector< std::shared_ptr< FiniteElement< dim > > > & fe() const
std::shared_ptr< FunctionSpace > function_space_
Function space defining the FE.
unsigned int n_dofs() const
Returns the number of degrees of freedom needed by the finite element.
Base class for Mesh and BCMesh.
LongIdx * get_el_4_loc() const
Range< Edge > edge_range() const
Return range of edges.
unsigned int n_edges() const
LongIdx * get_row_4_el() const
const Neighbour & vb_neighbour(unsigned int nb) const
Return neighbour with given index.
Distribution * get_el_ds() const
unsigned int n_elements() const
Range< ElementAccessor< 3 > > elements_range() const
Returns range of mesh elements.
unsigned int n_vb_neighbours() const
const DuplicateNodes * duplicate_nodes() const
ElementAccessor< 3 > element() const
ElementAccessor< 3 > element() const
Returns iterator to the element of the side.
unsigned int edge_idx() const
Returns global index of the edge connected to the side.
std::vector< LongIdx > parent_dof_idx_
Local indices in the parent handler.
void update_parent_vector(VectorMPI &vec, const VectorMPI &subvec)
Update values in parent vector from values of subvector.
void receive_sub_ghost_dofs(unsigned int proc, vector< unsigned int > &ghost_dof_proc, map< LongIdx, LongIdx > &global_to_local_dof_idx)
Get global dof indices of ghost dofs for sub-handler.
void update_subvector(const VectorMPI &vec, VectorMPI &subvec)
Update values in subvector from parent vector.
void send_sub_ghost_dofs(unsigned int proc, const map< LongIdx, LongIdx > &global_to_local_dof_idx)
Send global indices of dofs that are ghost on other processors.
std::shared_ptr< DOFHandlerMultiDim > parent_
Parent dof handler.
double get(unsigned int pos) const
Return value on given position.
void set(unsigned int pos, double val)
Set value on given position.
unsigned int size() const
Return size of output data.
Support classes for parallel programing.
Declaration of class which handles the ordering of degrees of freedom (dof) and mappings between loca...
Class FESystem for compound finite elements.
Abstract class for description of finite elements.
arma::Col< IntIdx > LocDofVec
int LongIdx
Define type that represents indices of large arrays (elements, nodes, dofs etc.)
#define MPI_Allreduce(sendbuf, recvbuf, count, datatype, op, comm)
#define MPI_Send(buf, count, datatype, dest, tag, comm)
#define MPI_STATUS_IGNORE
#define MPI_Allgatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm)
#define MPI_Recv(buf, count, datatype, source, tag, comm, status)
void printf(BasicWriter< Char > &w, BasicCStringRef< Char > format, ArgList args)
Implementation of range helper class.