OGS
NodeWiseMeshPartitioner.cpp
Go to the documentation of this file.
1 
16 
17 #include <limits>
18 #include <numeric>
19 #include <unordered_map>
20 
21 #include "BaseLib/Error.h"
22 #include "BaseLib/FileTools.h"
23 #include "BaseLib/Logging.h"
24 #include "BaseLib/Stream.h"
26 
27 namespace ApplicationUtils
28 {
29 struct NodeStruct
30 {
32  double const x_,
33  double const y_,
34  double const z_)
35  : id(id_), x(x_), y(y_), z(z_)
36  {
37  }
38 
40  double x;
41  double y;
42  double z;
43 };
44 
46  MeshLib::MeshItemType const item_type) const
47 {
48  if (item_type == MeshLib::MeshItemType::Node)
49  {
50  return nodes.size();
51  }
52 
53  if (item_type == MeshLib::MeshItemType::Cell)
54  {
55  return regular_elements.size() + ghost_elements.size();
56  }
57  OGS_FATAL("Mesh items other than nodes and cells are not supported.");
58 }
59 
60 std::ostream& Partition::writeNodes(
61  std::ostream& os, std::vector<std::size_t> const& global_node_ids) const
62 {
63  std::vector<NodeStruct> nodes_buffer;
64  nodes_buffer.reserve(nodes.size());
65 
66  for (const auto* node : nodes)
67  {
68  double const* coords = node->getCoords();
69  nodes_buffer.emplace_back(global_node_ids[node->getID()], coords[0],
70  coords[1], coords[2]);
71  }
72  return os.write(reinterpret_cast<const char*>(nodes_buffer.data()),
73  sizeof(NodeStruct) * nodes_buffer.size());
74 }
75 
81  std::vector<const MeshLib::Element*> const& elements)
82 {
83  return 3 * elements.size() +
84  std::accumulate(begin(elements), end(elements), 0,
85  [](auto const nnodes, auto const* e)
86  { return nnodes + e->getNumberOfNodes(); });
87 }
88 
89 std::ostream& Partition::writeConfig(std::ostream& os) const
90 {
91  long const data[] = {
92  static_cast<long>(nodes.size()),
93  static_cast<long>(number_of_base_nodes),
94  static_cast<long>(regular_elements.size()),
95  static_cast<long>(ghost_elements.size()),
96  static_cast<long>(number_of_regular_base_nodes),
97  static_cast<long>(number_of_regular_nodes),
98  static_cast<long>(number_of_mesh_base_nodes),
99  static_cast<long>(number_of_mesh_all_nodes),
100  static_cast<long>(
102  static_cast<long>(
104  };
105 
106  return os.write(reinterpret_cast<const char*>(data), sizeof(data));
107 }
108 
109 std::size_t nodeIdBulkMesh(
110  MeshLib::Node const& node,
111  std::vector<std::size_t> const* node_id_mapping = nullptr)
112 {
113  return node_id_mapping ? (*node_id_mapping)[node.getID()] : node.getID();
114 }
115 
116 std::size_t partitionLookup(
117  MeshLib::Node const& node,
118  std::vector<std::size_t> const& partition_ids,
119  std::vector<std::size_t> const* node_id_mapping = nullptr)
120 {
121  auto node_id = [&node_id_mapping](MeshLib::Node const& n)
122  { return nodeIdBulkMesh(n, node_id_mapping); };
123 
124  return partition_ids[node_id(node)];
125 }
126 
134 std::pair<std::vector<MeshLib::Node*>, std::vector<MeshLib::Node*>>
136  std::size_t const part_id,
137  std::vector<MeshLib::Node*> const& nodes,
138  std::vector<std::size_t> const& partition_ids,
139  MeshLib::Mesh const& mesh,
140  std::vector<std::size_t> const* node_id_mapping = nullptr)
141 {
142  // Find nodes belonging to a given partition id.
143  std::vector<MeshLib::Node*> partition_nodes;
144  copy_if(begin(nodes), end(nodes), std::back_inserter(partition_nodes),
145  [&](auto const& n) {
146  return partitionLookup(*n, partition_ids, node_id_mapping) ==
147  part_id;
148  });
149 
150  // Space for resulting vectors.
151  std::vector<MeshLib::Node*> base_nodes;
152  base_nodes.reserve(partition_nodes.size() /
153  2); // if linear mesh, then one reallocation, no realloc
154  // for higher order elements meshes.
155  std::vector<MeshLib::Node*> higher_order_nodes;
156  higher_order_nodes.reserve(
157  partition_nodes.size() /
158  2); // if linear mesh, then wasted space, good estimate for quadratic
159  // order mesh, and realloc needed for higher order element meshes.
160 
161  // Split the nodes into base nodes and extra nodes.
162  std::partition_copy(
163  begin(partition_nodes), end(partition_nodes),
164  std::back_inserter(base_nodes), std::back_inserter(higher_order_nodes),
165  [&](MeshLib::Node* const n)
166  { return isBaseNode(*n, mesh.getElementsConnectedToNode(*n)); });
167 
168  return {base_nodes, higher_order_nodes};
169 }
170 
171 std::ptrdiff_t numberOfRegularNodes(
172  MeshLib::Element const& e, std::size_t const part_id,
173  std::vector<std::size_t> const& partition_ids,
174  std::vector<std::size_t> const* node_id_mapping = nullptr)
175 {
176  return std::count_if(e.getNodes(), e.getNodes() + e.getNumberOfNodes(),
177  [&](MeshLib::Node* const n) {
178  return partitionLookup(*n, partition_ids,
179  node_id_mapping) == part_id;
180  });
181 }
182 
187 std::tuple<std::vector<MeshLib::Element const*>,
188  std::vector<MeshLib::Element const*>>
190  std::size_t const part_id,
191  std::vector<MeshLib::Element*> const& elements,
192  std::vector<std::size_t> const& partition_ids,
193  std::vector<std::size_t> const* node_id_mapping = nullptr)
194 {
195  std::vector<MeshLib::Element const*> regular_elements;
196  std::vector<MeshLib::Element const*> ghost_elements;
197 
198  for (auto elem : elements)
199  {
200  auto const regular_nodes = numberOfRegularNodes(
201  *elem, part_id, partition_ids, node_id_mapping);
202 
203  if (regular_nodes == 0)
204  {
205  continue;
206  }
207 
208  if (regular_nodes ==
209  static_cast<std::ptrdiff_t>(elem->getNumberOfNodes()))
210  {
211  regular_elements.push_back(elem);
212  }
213  else
214  {
215  ghost_elements.push_back(elem);
216  }
217  }
218  return std::tuple<std::vector<MeshLib::Element const*>,
219  std::vector<MeshLib::Element const*>>{regular_elements,
220  ghost_elements};
221 }
222 
227 std::tuple<std::vector<MeshLib::Node*>, std::vector<MeshLib::Node*>>
229  std::size_t const part_id,
230  std::vector<MeshLib::Node*> const& nodes,
231  std::vector<MeshLib::Element const*> const& ghost_elements,
232  std::vector<std::size_t> const& partition_ids,
233  MeshLib::Mesh const& mesh,
234  std::vector<std::size_t> const* node_id_mapping = nullptr)
235 {
236  std::vector<MeshLib::Node*> base_nodes;
237  std::vector<MeshLib::Node*> ghost_nodes;
238 
239  std::vector<bool> is_ghost_node(nodes.size(), false);
240  for (const auto* ghost_elem : ghost_elements)
241  {
242  for (unsigned i = 0; i < ghost_elem->getNumberOfNodes(); i++)
243  {
244  auto const& n = ghost_elem->getNode(i);
245  if (is_ghost_node[n->getID()])
246  {
247  continue;
248  }
249 
250  if (partitionLookup(*n, partition_ids, node_id_mapping) != part_id)
251  {
252  if (isBaseNode(*n, mesh.getElementsConnectedToNode(*n)))
253  {
254  base_nodes.push_back(nodes[n->getID()]);
255  }
256  else
257  {
258  ghost_nodes.push_back(nodes[n->getID()]);
259  }
260  is_ghost_node[n->getID()] = true;
261  }
262  }
263  }
264  return std::tuple<std::vector<MeshLib::Node*>, std::vector<MeshLib::Node*>>{
265  base_nodes, ghost_nodes};
266 }
267 
268 void NodeWiseMeshPartitioner::processPartition(std::size_t const part_id)
269 {
270  auto& partition = _partitions[part_id];
271  std::vector<MeshLib::Node*> higher_order_regular_nodes;
272  std::tie(partition.nodes, higher_order_regular_nodes) =
273  findRegularNodesInPartition(part_id, _mesh->getNodes(),
275 
276  partition.number_of_regular_base_nodes = partition.nodes.size();
277  std::copy(begin(higher_order_regular_nodes),
278  end(higher_order_regular_nodes),
279  std::back_inserter(partition.nodes));
280 
281  partition.number_of_regular_nodes = partition.nodes.size();
282 
283  std::tie(partition.regular_elements, partition.ghost_elements) =
284  findElementsInPartition(part_id, _mesh->getElements(),
286  std::vector<MeshLib::Node*> base_ghost_nodes;
287  std::vector<MeshLib::Node*> higher_order_ghost_nodes;
288  std::tie(base_ghost_nodes, higher_order_ghost_nodes) =
289  findGhostNodesInPartition(part_id, _mesh->getNodes(),
290  partition.ghost_elements,
292 
293  std::copy(begin(base_ghost_nodes), end(base_ghost_nodes),
294  std::back_inserter(partition.nodes));
295 
296  partition.number_of_base_nodes =
297  partition.number_of_regular_base_nodes + base_ghost_nodes.size();
298 
299  std::copy(begin(higher_order_ghost_nodes),
300  end(higher_order_ghost_nodes),
301  std::back_inserter(partition.nodes));
302 
303  // Set the node numbers of base and all mesh nodes.
304  partition.number_of_mesh_base_nodes = _mesh->getNumberOfBaseNodes();
305  partition.number_of_mesh_all_nodes = _mesh->getNumberOfNodes();
306 }
307 
310 template <typename T>
312  Partition const& p,
313  std::size_t const offset,
314  MeshLib::PropertyVector<T> const& pv,
315  MeshLib::PropertyVector<T>& partitioned_pv)
316 {
317  auto const& nodes = p.nodes;
318  auto const nnodes = nodes.size();
319  auto const n_components = pv.getNumberOfGlobalComponents();
320  for (std::size_t i = 0; i < nnodes; ++i)
321  {
322  const auto global_id = nodes[i]->getID();
323  std::copy_n(&pv[n_components * global_id], n_components,
324  &partitioned_pv[offset + n_components * i]);
325  }
326  return n_components * nnodes;
327 }
328 
332 template <typename T>
334  Partition const& p,
335  std::size_t const offset,
336  MeshLib::PropertyVector<T> const& pv,
337  MeshLib::PropertyVector<T>& partitioned_pv)
338 {
339  std::size_t const n_regular(p.regular_elements.size());
340  auto const n_components = pv.getNumberOfGlobalComponents();
341  for (std::size_t i = 0; i < n_regular; ++i)
342  {
343  const auto id = p.regular_elements[i]->getID();
344  std::copy_n(&pv[n_components * id], n_components,
345  &partitioned_pv[offset + n_components * i]);
346  }
347 
348  std::size_t const n_ghost(p.ghost_elements.size());
349  for (std::size_t i = 0; i < n_ghost; ++i)
350  {
351  const auto id = p.ghost_elements[i]->getID();
352  std::copy_n(&pv[n_components * id], n_components,
353  &partitioned_pv[offset + n_components * (n_regular + i)]);
354  }
355  return n_components * (n_regular + n_ghost);
356 }
357 
358 template <typename T>
360  MeshLib::Properties& partitioned_properties,
361  std::vector<Partition> const& partitions,
362  MeshLib::PropertyVector<T> const* const pv,
363  std::map<MeshLib::MeshItemType, std::size_t> const& total_number_of_tuples)
364 {
365  if (pv == nullptr)
366  {
367  return false;
368  }
369  auto const item_type = pv->getMeshItemType();
370 
372  {
373  return true; // Skip integration point data. Requires parsing of json
374  // for the integration point data. Return true, because
375  // the property was "successfully" parsed.
376  }
377 
378  auto partitioned_pv = partitioned_properties.createNewPropertyVector<T>(
379  pv->getPropertyName(), pv->getMeshItemType(),
381  partitioned_pv->resize(total_number_of_tuples.at(item_type) *
383 
384  auto copy_property_vector_values =
385  [&](Partition const& p, std::size_t offset)
386  {
387  if (item_type == MeshLib::MeshItemType::Node)
388  {
389  return copyNodePropertyVectorValues(p, offset, *pv,
390  *partitioned_pv);
391  }
392  if (item_type == MeshLib::MeshItemType::Cell)
393  {
394  return copyCellPropertyVectorValues(p, offset, *pv,
395  *partitioned_pv);
396  }
397  OGS_FATAL(
398  "Copying of property vector values for mesh item type {:s} is not "
399  "implemented.",
400  item_type);
401  };
402 
403  std::size_t position_offset(0);
404  for (auto p : partitions)
405  {
406  position_offset += copy_property_vector_values(p, position_offset);
407  }
408  return true;
409 }
410 
416 template <typename Function>
418 {
419  for (auto [name, property] : properties)
420  {
421  // Open question, why is the 'unsigned long' case not compiling giving
422  // an error "expected '(' for function-style cast or type construction"
423  // with clang-7, and "error C4576: a parenthesized type followed by an
424  // initializer list is a non-standard explicit type conversion syntax"
425  // with MSVC-15.
426  bool success = f(double{}, property) || f(float{}, property) ||
427  f(int{}, property) || f(long{}, property) ||
428  f(unsigned{}, property) || f(long{}, property) ||
429  f(static_cast<unsigned long>(0), property) ||
430  f(std::size_t{}, property) || f(char{}, property) ||
431  f(static_cast<unsigned char>(0), property);
432  if (!success)
433  {
434  OGS_FATAL("Could not apply function to PropertyVector '{:s}'.",
435  property->getPropertyName());
436  }
437  }
438 }
439 
440 void addVtkGhostTypeProperty(MeshLib::Properties& partitioned_properties,
441  std::vector<Partition> const& partitions,
442  std::size_t const total_number_of_cells)
443 {
444  auto* vtk_ghost_type =
445  partitioned_properties.createNewPropertyVector<unsigned char>(
446  "vtkGhostType", MeshLib::MeshItemType::Cell);
447  if (vtk_ghost_type == nullptr)
448  {
449  OGS_FATAL("Could not create vtkGhostType cell data array.");
450  }
451 
452  vtk_ghost_type->resize(total_number_of_cells);
453  std::size_t offset = 0;
454  for (auto const& partition : partitions)
455  {
456  offset += partition.regular_elements.size();
457  for (std::size_t i = 0; i < partition.ghost_elements.size(); ++i)
458  {
459  if (partition.duplicate_ghost_cell[i])
460  {
461  (*vtk_ghost_type)[offset + i] |=
462  vtkDataSetAttributes::DUPLICATECELL;
463  }
464  }
465  offset += partition.ghost_elements.size();
466  }
467 }
468 
471  MeshLib::Properties const& properties,
472  std::vector<Partition> const& partitions)
473 {
474  using namespace MeshLib;
475 
476  Properties partitioned_properties;
477 
478  auto count_tuples = [&](MeshItemType const mesh_item_type)
479  {
480  return std::accumulate(
481  begin(partitions), end(partitions), 0,
482  [&](std::size_t const sum, Partition const& p)
483  { return sum + p.numberOfMeshItems(mesh_item_type); });
484  };
485  std::map<MeshItemType, std::size_t> const total_number_of_tuples = {
486  {MeshItemType::Cell, count_tuples(MeshItemType::Cell)},
487  {MeshItemType::Node, count_tuples(MeshItemType::Node)}};
488 
489  DBUG(
490  "total number of tuples after partitioning defined for cells is {:d} "
491  "and for nodes {:d}.",
492  total_number_of_tuples.at(MeshItemType::Cell),
493  total_number_of_tuples.at(MeshItemType::Node));
494 
495  // 1 create new PV
496  // 2 resize the PV with total_number_of_tuples
497  // 3 copy the values according to the partition info
499  properties,
500  [&](auto type, auto const property)
501  {
502  return copyPropertyVector<decltype(type)>(
503  partitioned_properties, partitions,
504  dynamic_cast<PropertyVector<decltype(type)> const*>(property),
505  total_number_of_tuples);
506  });
507 
508  addVtkGhostTypeProperty(partitioned_properties,
509  partitions,
510  total_number_of_tuples.at(MeshItemType::Cell));
511 
512  return partitioned_properties;
513 }
514 
516  std::vector<Partition>& partitions)
517 {
518  std::vector<bool> cell_visited(mesh.getElements().size(), false);
519 
520  for (auto& partition : partitions)
521  {
522  partition.duplicate_ghost_cell.resize(partition.ghost_elements.size(),
523  true);
524 
525  for (std::size_t i = 0; i < partition.ghost_elements.size(); i++)
526  {
527  const auto& ghost_element = *partition.ghost_elements[i];
528  if (!cell_visited[ghost_element.getID()])
529  {
530  cell_visited[ghost_element.getID()] = true;
531  partition.duplicate_ghost_cell[i] = false;
532  }
533  }
534  }
535 }
536 
538 {
539  for (std::size_t part_id = 0; part_id < _partitions.size(); part_id++)
540  {
541  INFO("Processing partition: {:d}", part_id);
542  processPartition(part_id);
543  }
544 
546 
548 
550  partitionProperties(_mesh->getProperties(), _partitions);
551 }
552 
554  MeshLib::PropertyVector<std::size_t>* const bulk_node_ids_pv,
555  std::vector<Partition> const& local_partitions) const
556 {
557  if (bulk_node_ids_pv == nullptr)
558  {
559  return;
560  }
561 
562  auto& bulk_node_ids = *bulk_node_ids_pv;
563 
564  std::size_t offset = 0; // offset in property vector for current partition
565 
566  assert(_partitions.size() == local_partitions.size());
567  int const n_partitions = static_cast<int>(_partitions.size());
568  for (int partition_id = 0; partition_id < n_partitions; ++partition_id)
569  {
570  auto const& bulk_partition = _partitions[partition_id];
571  auto const& local_partition = local_partitions[partition_id];
572 
573  // Create global-to-local node id mapping for the bulk partition.
574  auto const& bulk_nodes = bulk_partition.nodes;
575  auto const n_bulk_nodes = bulk_nodes.size();
576  std::map<std::size_t, std::size_t> global_to_local;
577  for (std::size_t local_node_id = 0; local_node_id < n_bulk_nodes;
578  ++local_node_id)
579  {
580  global_to_local[bulk_nodes[local_node_id]->getID()] = local_node_id;
581  }
582 
583  auto const& local_nodes = local_partition.nodes;
584  auto const n_local_nodes = local_nodes.size();
585  for (std::size_t local_node_id = 0; local_node_id < n_local_nodes;
586  ++local_node_id)
587  {
588  bulk_node_ids[offset + local_node_id] =
589  global_to_local[bulk_node_ids[offset + local_node_id]];
590  }
591  offset += n_local_nodes;
592  }
593 }
594 
596  MeshLib::PropertyVector<std::size_t>* const bulk_element_ids_pv,
597  std::vector<Partition> const& local_partitions) const
598 {
599  if (bulk_element_ids_pv == nullptr)
600  {
601  return;
602  }
603 
604  auto& bulk_element_ids = *bulk_element_ids_pv;
605 
606  std::size_t offset = 0; // offset in property vector for current partition
607 
608  assert(_partitions.size() == local_partitions.size());
609  int const n_partitions = static_cast<int>(_partitions.size());
610  for (int partition_id = 0; partition_id < n_partitions; ++partition_id)
611  {
612  auto const& bulk_partition = _partitions[partition_id];
613  auto const& local_partition = local_partitions[partition_id];
614 
615  // Create global-to-local element id mapping for the bulk partition.
616  std::map<std::size_t, std::size_t> global_to_local;
617  auto map_elements =
618  [&global_to_local](
619  std::vector<MeshLib::Element const*> const& elements,
620  std::size_t const offset)
621  {
622  auto const n_elements = elements.size();
623  for (std::size_t e = 0; e < n_elements; ++e)
624  {
625  global_to_local[elements[e]->getID()] = offset + e;
626  }
627  };
628 
629  map_elements(bulk_partition.regular_elements, 0);
630  map_elements(bulk_partition.ghost_elements,
631  bulk_partition.regular_elements.size());
632 
633  // Renumber the local bulk_element_ids map.
634  auto renumber_elements =
635  [&bulk_element_ids, &global_to_local](
636  std::vector<MeshLib::Element const*> const& elements,
637  std::size_t const offset)
638  {
639  auto const n_elements = elements.size();
640  for (std::size_t e = 0; e < n_elements; ++e)
641  {
642  bulk_element_ids[offset + e] =
643  global_to_local[bulk_element_ids[offset + e]];
644  }
645  return n_elements;
646  };
647 
648  offset += renumber_elements(local_partition.regular_elements, offset);
649  offset += renumber_elements(local_partition.ghost_elements, offset);
650  }
651 }
652 
654  MeshLib::Mesh const& mesh) const
655 {
656  auto const& bulk_node_ids =
657  mesh.getProperties().getPropertyVector<std::size_t>(
658  "bulk_node_ids", MeshLib::MeshItemType::Node, 1);
659 
660  std::vector<Partition> partitions(_partitions.size());
661  for (std::size_t part_id = 0; part_id < _partitions.size(); part_id++)
662  {
663  auto& partition = partitions[part_id];
664  INFO("Processing partition: {:d}", part_id);
665  // Set the node numbers of base and all mesh nodes.
666  partition.number_of_mesh_base_nodes = mesh.getNumberOfBaseNodes();
667  partition.number_of_mesh_all_nodes = mesh.getNumberOfNodes();
668 
669  std::vector<MeshLib::Node*> higher_order_regular_nodes;
670  std::tie(partition.nodes, higher_order_regular_nodes) =
673  bulk_node_ids);
674 
675  partition.number_of_regular_base_nodes = partition.nodes.size();
676  std::copy(begin(higher_order_regular_nodes),
677  end(higher_order_regular_nodes),
678  std::back_inserter(partition.nodes));
679 
680  partition.number_of_regular_nodes = partition.nodes.size();
681 
682  std::tie(partition.regular_elements, partition.ghost_elements) =
684  _nodes_partition_ids, bulk_node_ids);
685 
686  std::vector<MeshLib::Node*> base_ghost_nodes;
687  std::vector<MeshLib::Node*> higher_order_ghost_nodes;
688  std::tie(base_ghost_nodes, higher_order_ghost_nodes) =
690  part_id, mesh.getNodes(), partition.ghost_elements,
691  _nodes_partition_ids, mesh, bulk_node_ids);
692 
693  std::copy(begin(base_ghost_nodes), end(base_ghost_nodes),
694  std::back_inserter(partition.nodes));
695 
696  partition.number_of_base_nodes =
697  partition.number_of_regular_base_nodes + base_ghost_nodes.size();
698 
699  std::copy(begin(higher_order_ghost_nodes),
700  end(higher_order_ghost_nodes),
701  std::back_inserter(partition.nodes));
702  }
703 
704  markDuplicateGhostCells(mesh, partitions);
705  return partitions;
706 }
707 
709 {
710  std::size_t node_global_id_offset = 0;
711  // Renumber the global indices.
712  for (auto& partition : _partitions)
713  {
714  for (std::size_t i = 0; i < partition.number_of_regular_nodes; i++)
715  {
716  _nodes_global_ids[partition.nodes[i]->getID()] =
717  node_global_id_offset++;
718  }
719  }
720 }
721 
722 template <typename T>
723 void writePropertyVectorValues(std::ostream& os,
724  MeshLib::PropertyVector<T> const& pv)
725 {
726  os.write(reinterpret_cast<const char*>(pv.data()), pv.size() * sizeof(T));
727 }
728 
729 template <typename T>
731  MeshLib::MeshItemType const mesh_item_type,
732  std::ostream& out_val, std::ostream& out_meta)
733 {
734  if (pv == nullptr)
735  {
736  return false;
737  }
738  // skip property of different mesh item type. Return true, because this
739  // operation was successful.
740  if (pv->getMeshItemType() != mesh_item_type)
741  {
742  return true;
743  }
744 
746  pvmd.property_name = pv->getPropertyName();
749  pvmd.number_of_tuples = pv->getNumberOfTuples();
750  writePropertyVectorValues(out_val, *pv);
752  return true;
753 }
754 
755 void writeProperties(const std::string& file_name_base,
756  MeshLib::Properties const& partitioned_properties,
757  std::vector<Partition> const& partitions,
758  MeshLib::MeshItemType const mesh_item_type)
759 {
760  auto const number_of_properties =
761  partitioned_properties.size(mesh_item_type);
762  if (number_of_properties == 0)
763  {
764  return;
765  }
766 
767  auto const file_name_infix = toString(mesh_item_type);
768 
769  auto const file_name_cfg = file_name_base + "_partitioned_" +
770  file_name_infix + "_properties_cfg" +
771  std::to_string(partitions.size()) + ".bin";
772  std::ofstream out(file_name_cfg, std::ios::binary);
773  if (!out)
774  {
775  OGS_FATAL("Could not open file '{:s}' for output.", file_name_cfg);
776  }
777 
778  auto const file_name_val = file_name_base + "_partitioned_" +
779  file_name_infix + "_properties_val" +
780  std::to_string(partitions.size()) + ".bin";
781  std::ofstream out_val(file_name_val, std::ios::binary);
782  if (!out_val)
783  {
784  OGS_FATAL("Could not open file '{:s}' for output.", file_name_val);
785  }
786 
788 
790  partitioned_properties,
791  [&](auto type, auto const& property)
792  {
793  return writePropertyVector<decltype(type)>(
794  dynamic_cast<MeshLib::PropertyVector<decltype(type)> const*>(
795  property),
796  mesh_item_type, out_val, out);
797  });
798 
799  unsigned long offset = 0;
800  for (const auto& partition : partitions)
801  {
803  offset, static_cast<unsigned long>(
804  partition.numberOfMeshItems(mesh_item_type))};
805  DBUG(
806  "Write meta data for node-based PropertyVector: global offset "
807  "{:d}, number of tuples {:d}",
808  pvpmd.offset, pvpmd.number_of_tuples);
810  offset += pvpmd.number_of_tuples;
811  }
812 }
813 
815 {
819 
820  std::ostream& writeConfig(std::ostream& os) const;
821 };
822 
823 std::ostream& ConfigOffsets::writeConfig(std::ostream& os) const
824 {
825  os.write(reinterpret_cast<const char*>(this), sizeof(ConfigOffsets));
826 
827  static long reserved = 0; // Value reserved in the binary format, not used
828  // in the partitioning process.
829  return os.write(reinterpret_cast<const char*>(&reserved), sizeof(long));
830 }
831 
833 {
834  long node;
837 };
838 
840 {
841  return {static_cast<long>(partition.nodes.size()),
842  static_cast<long>(partition.regular_elements.size() +
844  partition.regular_elements)),
845  static_cast<long>(partition.ghost_elements.size() +
847  partition.ghost_elements))};
848 }
849 
851  PartitionOffsets const& offsets)
852 {
853  return {
854  static_cast<long>(oldConfig.node_rank_offset +
855  offsets.node * sizeof(NodeStruct)),
856  // Offset the ending entry of the element integer variables of
857  // the non-ghost elements of this partition in the vector of elem_info.
858  static_cast<long>(oldConfig.element_rank_offset +
859  offsets.regular_elements * sizeof(long)),
860 
861  // Offset the ending entry of the element integer variables of
862  // the ghost elements of this partition in the vector of elem_info.
863  static_cast<long>(oldConfig.ghost_element_rank_offset +
864  offsets.ghost_elements * sizeof(long))};
865 }
866 
872 std::tuple<std::vector<long>, std::vector<long>> writeConfigData(
873  const std::string& file_name_base, std::vector<Partition> const& partitions)
874 {
875  auto const file_name_cfg = file_name_base + "_partitioned_msh_cfg" +
876  std::to_string(partitions.size()) + ".bin";
877  std::ofstream of_bin_cfg(file_name_cfg, std::ios::binary);
878  if (!of_bin_cfg)
879  {
880  OGS_FATAL("Could not open file '{:s}' for output.", file_name_cfg);
881  }
882 
883  std::vector<long> partitions_element_offsets;
884  partitions_element_offsets.reserve(partitions.size());
885  std::vector<long> partitions_ghost_element_offsets;
886  partitions_ghost_element_offsets.reserve(partitions.size());
887 
888  ConfigOffsets config_offsets = {0, 0, 0}; // 0 for first partition.
889  for (const auto& partition : partitions)
890  {
891  partition.writeConfig(of_bin_cfg);
892 
893  config_offsets.writeConfig(of_bin_cfg);
894  auto const& partition_offsets = computePartitionOffsets(partition);
895  config_offsets =
896  incrementConfigOffsets(config_offsets, partition_offsets);
897 
898  partitions_element_offsets.push_back(
899  partition_offsets.regular_elements);
900  partitions_ghost_element_offsets.push_back(
901  partition_offsets.ghost_elements);
902  }
903 
904  return std::make_tuple(partitions_element_offsets,
905  partitions_ghost_element_offsets);
906 }
907 
916  const MeshLib::Element& elem,
917  const std::unordered_map<std::size_t, long>& local_node_ids,
918  std::vector<long>& elem_info,
919  long& counter)
920 {
921  constexpr unsigned mat_id =
922  0; // TODO: Material ID to be set from the mesh data
923  const long nn = elem.getNumberOfNodes();
924  elem_info[counter++] = mat_id;
925  elem_info[counter++] = static_cast<long>(elem.getCellType());
926  elem_info[counter++] = nn;
927 
928  for (long i = 0; i < nn; i++)
929  {
930  auto const& n = *elem.getNode(i);
931  elem_info[counter++] = local_node_ids.at(n.getID());
932  }
933 }
934 
936 std::unordered_map<std::size_t, long> enumerateLocalNodeIds(
937  std::vector<MeshLib::Node*> const& nodes)
938 {
939  std::unordered_map<std::size_t, long> local_ids;
940  local_ids.reserve(nodes.size());
941 
942  long local_node_id = 0;
943  for (const auto* node : nodes)
944  {
945  local_ids[node->getID()] = local_node_id++;
946  }
947  return local_ids;
948 }
949 
956 void writeElements(std::string const& file_name_base,
957  std::vector<Partition> const& partitions,
958  std::vector<long> const& regular_element_offsets,
959  std::vector<long> const& ghost_element_offsets)
960 {
961  const std::string npartitions_str = std::to_string(partitions.size());
962 
963  auto const file_name_ele =
964  file_name_base + "_partitioned_msh_ele" + npartitions_str + ".bin";
965  std::ofstream element_info_os(file_name_ele, std::ios::binary);
966  if (!element_info_os)
967  {
968  OGS_FATAL("Could not open file '{:s}' for output.", file_name_ele);
969  }
970 
971  auto const file_name_ele_g =
972  file_name_base + "_partitioned_msh_ele_g" + npartitions_str + ".bin";
973  std::ofstream ghost_element_info_os(file_name_ele_g, std::ios::binary);
974  if (!ghost_element_info_os)
975  {
976  OGS_FATAL("Could not open file '{:s}' for output.", file_name_ele_g);
977  }
978 
979  for (std::size_t i = 0; i < partitions.size(); i++)
980  {
981  const auto& partition = partitions[i];
982  auto const local_node_ids = enumerateLocalNodeIds(partition.nodes);
983 
984  // Vector containing the offsets of the regular elements of this
985  // partition
986  std::vector<long> ele_info(regular_element_offsets[i]);
987 
988  auto writeElementData =
989  [&local_node_ids](
990  std::vector<MeshLib::Element const*> const& elements,
991  long const element_offsets,
992  std::ofstream& output_stream)
993  {
994  long counter = elements.size();
995  std::vector<long> ele_info(element_offsets);
996 
997  for (std::size_t j = 0; j < elements.size(); j++)
998  {
999  const auto* elem = elements[j];
1000  ele_info[j] = counter;
1001  getElementIntegerVariables(*elem, local_node_ids, ele_info,
1002  counter);
1003  }
1004  // Write vector data of regular elements
1005  output_stream.write(reinterpret_cast<const char*>(ele_info.data()),
1006  ele_info.size() * sizeof(long));
1007  };
1008 
1009  // regular elements.
1010  writeElementData(partition.regular_elements, regular_element_offsets[i],
1011  element_info_os);
1012  // Ghost elements
1013  writeElementData(partition.ghost_elements, ghost_element_offsets[i],
1014  ghost_element_info_os);
1015  }
1016 }
1017 
1022 void writeNodes(const std::string& file_name_base,
1023  std::vector<Partition> const& partitions,
1024  std::vector<std::size_t> const& global_node_ids)
1025 {
1026  auto const file_name = file_name_base + "_partitioned_msh_nod" +
1027  std::to_string(partitions.size()) + ".bin";
1028  std::ofstream os(file_name, std::ios::binary);
1029  if (!os)
1030  {
1031  OGS_FATAL("Could not open file '{:s}' for output.", file_name);
1032  }
1033 
1034  for (const auto& partition : partitions)
1035  {
1036  partition.writeNodes(os, global_node_ids);
1037  }
1038 }
1039 
1040 void NodeWiseMeshPartitioner::write(const std::string& file_name_base)
1041 {
1046 
1047  const auto elements_offsets = writeConfigData(file_name_base, _partitions);
1048 
1049  const std::vector<IntegerType>& regular_element_offsets =
1050  std::get<0>(elements_offsets);
1051  const std::vector<IntegerType>& ghost_element_offsets =
1052  std::get<1>(elements_offsets);
1053  writeElements(file_name_base, _partitions, regular_element_offsets,
1054  ghost_element_offsets);
1055 
1056  writeNodes(file_name_base, _partitions, _nodes_global_ids);
1057 }
1058 
1060  std::string const& output_filename_base,
1061  std::vector<Partition> const& partitions,
1062  MeshLib::Properties const& partitioned_properties) const
1063 {
1064  writeNodes(output_filename_base, partitions, _nodes_global_ids);
1065 
1066  const auto elem_integers =
1067  writeConfigData(output_filename_base, partitions);
1068 
1069  const std::vector<IntegerType>& num_elem_integers =
1070  std::get<0>(elem_integers);
1071  const std::vector<IntegerType>& num_g_elem_integers =
1072  std::get<1>(elem_integers);
1073  writeElements(output_filename_base, partitions, num_elem_integers,
1074  num_g_elem_integers);
1075 
1076  writeProperties(output_filename_base, partitioned_properties, partitions,
1078  writeProperties(output_filename_base, partitioned_properties, partitions,
1080 }
1081 } // namespace ApplicationUtils
#define OGS_FATAL(...)
Definition: Error.h:26
Filename manipulation routines.
void INFO(char const *fmt, Args const &... args)
Definition: Logging.h:32
void DBUG(char const *fmt, Args const &... args)
Definition: Logging.h:27
Declare a class to perform node wise mesh partitioning.
Helper tools for debugging.
Implementation of the VtuInterface class.
void renumberBulkElementIdsProperty(MeshLib::PropertyVector< std::size_t > *const bulk_element_ids_pv, std::vector< Partition > const &local_partitions) const
std::vector< Partition > partitionOtherMesh(MeshLib::Mesh const &mesh) const
void write(const std::string &file_name_base)
void renumberBulkNodeIdsProperty(MeshLib::PropertyVector< std::size_t > *const bulk_node_ids, std::vector< Partition > const &local_partitions) const
std::unique_ptr< MeshLib::Mesh > _mesh
Pointer to a mesh object.
MeshLib::Properties _partitioned_properties
Properties where values at ghost nodes and extra nodes are inserted.
void processPartition(std::size_t const part_id)
std::vector< std::size_t > _nodes_global_ids
Global IDs of all nodes after partitioning.
std::vector< Partition > _partitions
Data for all partitions.
std::vector< std::size_t > _nodes_partition_ids
Partition IDs of each nodes.
void writeOtherMesh(std::string const &output_filename_base, std::vector< Partition > const &partitions, MeshLib::Properties const &partitioned_properties) const
std::size_t getID() const
Definition: Point3dWithID.h:62
virtual Node *const * getNodes() const =0
Get array of element nodes.
virtual CellType getCellType() const =0
virtual const Node * getNode(unsigned idx) const =0
virtual unsigned getNumberOfNodes() const =0
std::size_t getNumberOfBaseNodes() const
Get the number of base nodes.
Definition: Mesh.cpp:214
std::vector< Node * > const & getNodes() const
Get the nodes-vector for the mesh.
Definition: Mesh.h:95
std::vector< Element * > const & getElements() const
Get the element-vector for the mesh.
Definition: Mesh.h:98
Properties & getProperties()
Definition: Mesh.h:123
std::size_t getNumberOfNodes() const
Get the number of nodes.
Definition: Mesh.h:89
std::vector< Element const * > const & getElementsConnectedToNode(std::size_t node_id) const
Definition: Mesh.cpp:232
Property manager on mesh items. Class Properties manages scalar, vector or matrix properties....
Definition: Properties.h:36
std::map< std::string, PropertyVectorBase * >::size_type size() const
Definition: Properties.cpp:165
PropertyVector< T > const * getPropertyVector(std::string const &name) const
PropertyVector< T > * createNewPropertyVector(std::string const &name, MeshItemType mesh_item_type, std::size_t n_components=1)
std::string const & getPropertyName() const
MeshItemType getMeshItemType() const
int getNumberOfGlobalComponents() const
std::size_t getNumberOfTuples() const
std::size_t size() const
std::function< bool(const double t, MappedConstVector< N > const &y, MappedVector< N > &ydot)> Function
std::size_t copyCellPropertyVectorValues(Partition const &p, std::size_t const offset, MeshLib::PropertyVector< T > const &pv, MeshLib::PropertyVector< T > &partitioned_pv)
void addVtkGhostTypeProperty(MeshLib::Properties &partitioned_properties, std::vector< Partition > const &partitions, std::size_t const total_number_of_cells)
bool writePropertyVector(MeshLib::PropertyVector< T > const *const pv, MeshLib::MeshItemType const mesh_item_type, std::ostream &out_val, std::ostream &out_meta)
std::ptrdiff_t numberOfRegularNodes(MeshLib::Element const &e, std::size_t const part_id, std::vector< std::size_t > const &partition_ids, std::vector< std::size_t > const *node_id_mapping=nullptr)
std::tuple< std::vector< MeshLib::Node * >, std::vector< MeshLib::Node * > > findGhostNodesInPartition(std::size_t const part_id, std::vector< MeshLib::Node * > const &nodes, std::vector< MeshLib::Element const * > const &ghost_elements, std::vector< std::size_t > const &partition_ids, MeshLib::Mesh const &mesh, std::vector< std::size_t > const *node_id_mapping=nullptr)
MeshLib::Properties partitionProperties(MeshLib::Properties const &properties, std::vector< Partition > const &partitions)
Partition existing properties and add vtkGhostType cell data array property.
void writeNodes(const std::string &file_name_base, std::vector< Partition > const &partitions, std::vector< std::size_t > const &global_node_ids)
std::size_t nodeIdBulkMesh(MeshLib::Node const &node, std::vector< std::size_t > const *node_id_mapping=nullptr)
ConfigOffsets incrementConfigOffsets(ConfigOffsets const &oldConfig, PartitionOffsets const &offsets)
std::tuple< std::vector< MeshLib::Element const * >, std::vector< MeshLib::Element const * > > findElementsInPartition(std::size_t const part_id, std::vector< MeshLib::Element * > const &elements, std::vector< std::size_t > const &partition_ids, std::vector< std::size_t > const *node_id_mapping=nullptr)
bool copyPropertyVector(MeshLib::Properties &partitioned_properties, std::vector< Partition > const &partitions, MeshLib::PropertyVector< T > const *const pv, std::map< MeshLib::MeshItemType, std::size_t > const &total_number_of_tuples)
void applyToPropertyVectors(MeshLib::Properties const &properties, Function f)
PartitionOffsets computePartitionOffsets(Partition const &partition)
NodeWiseMeshPartitioner::IntegerType getNumberOfIntegerVariablesOfElements(std::vector< const MeshLib::Element * > const &elements)
std::size_t partitionLookup(MeshLib::Node const &node, std::vector< std::size_t > const &partition_ids, std::vector< std::size_t > const *node_id_mapping=nullptr)
std::tuple< std::vector< long >, std::vector< long > > writeConfigData(const std::string &file_name_base, std::vector< Partition > const &partitions)
std::pair< std::vector< MeshLib::Node * >, std::vector< MeshLib::Node * > > findRegularNodesInPartition(std::size_t const part_id, std::vector< MeshLib::Node * > const &nodes, std::vector< std::size_t > const &partition_ids, MeshLib::Mesh const &mesh, std::vector< std::size_t > const *node_id_mapping=nullptr)
void writePropertyVectorValues(std::ostream &os, MeshLib::PropertyVector< T > const &pv)
std::unordered_map< std::size_t, long > enumerateLocalNodeIds(std::vector< MeshLib::Node * > const &nodes)
Generates a mapping of given node ids to a new local (renumbered) node ids.
void getElementIntegerVariables(const MeshLib::Element &elem, const std::unordered_map< std::size_t, long > &local_node_ids, std::vector< long > &elem_info, long &counter)
void markDuplicateGhostCells(MeshLib::Mesh const &mesh, std::vector< Partition > &partitions)
std::size_t copyNodePropertyVectorValues(Partition const &p, std::size_t const offset, MeshLib::PropertyVector< T > const &pv, MeshLib::PropertyVector< T > &partitioned_pv)
void writeProperties(const std::string &file_name_base, MeshLib::Properties const &partitioned_properties, std::vector< Partition > const &partitions, MeshLib::MeshItemType const mesh_item_type)
void writeElements(std::string const &file_name_base, std::vector< Partition > const &partitions, std::vector< long > const &regular_element_offsets, std::vector< long > const &ghost_element_offsets)
void writeValueBinary(std::ostream &out, T const &val)
write value as binary into the given output stream
Definition: FileTools.h:63
const char * toString(mgis::behaviour::Behaviour::Kinematic kin)
Converts MGIS kinematic to a string representation.
Definition: MFront.cpp:103
void copy(PETScVector const &x, PETScVector &y)
Definition: LinAlg.cpp:37
void writePropertyVectorPartitionMetaData(std::ostream &os, PropertyVectorPartitionMetaData const &pvpmd)
void writePropertyVectorMetaData(std::ostream &os, PropertyVectorMetaData const &pvmd)
MeshItemType
Definition: Location.h:21
bool isBaseNode(Node const &node, std::vector< Element const * > const &elements_connected_to_node)
Definition: Mesh.cpp:370
std::ostream & writeConfig(std::ostream &os) const
NodeWiseMeshPartitioner::IntegerType id
NodeStruct(NodeWiseMeshPartitioner::IntegerType const id_, double const x_, double const y_, double const z_)
std::ostream & writeConfig(std::ostream &os) const
std::vector< MeshLib::Node * > nodes
nodes.
std::size_t numberOfMeshItems(MeshLib::MeshItemType const item_type) const
std::vector< const MeshLib::Element * > regular_elements
Non ghost elements.
std::vector< const MeshLib::Element * > ghost_elements
std::ostream & writeNodes(std::ostream &os, std::vector< std::size_t > const &global_node_ids) const