From 8c2756ffb2af707771653b88ececa4de9b021551 Mon Sep 17 00:00:00 2001 From: ampdes Date: Wed, 19 Jun 2024 02:52:28 +0200 Subject: [PATCH 01/60] Add basic write function --- cpp/dolfinx/io/checkpointing.h | 37 ++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 cpp/dolfinx/io/checkpointing.h diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h new file mode 100644 index 00000000000..5bfa7322f03 --- /dev/null +++ b/cpp/dolfinx/io/checkpointing.h @@ -0,0 +1,37 @@ +// Copyright (C) year authors +// +// This file is part of DOLFINX (https://www.fenicsproject.org) +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include +#include +#include + +/// @file checkpointing.h +/// @brief ADIOS2 based checkpointing + +namespace dolfinx::io::checkpointing +{ + +void write(MPI_Comm comm, const std::filesystem::path& filename, + std::string tag, std::shared_ptr> mesh) + { + adios2::ADIOS adios(comm); + adios2::IO io = adios.DeclareIO(tag); + adios2::Engine writer = io.Open(filename, adios2::Mode::Write); + + const std::string mesh_name = mesh->name; + const std::int16_t mesh_dim = mesh->geometry().dim(); + adios2::Variable name = io.DefineVariable("name"); + adios2::Variable dim = io.DefineVariable("dim"); + writer.BeginStep(); + writer.Put(name, mesh_name); + writer.Put(dim, mesh_dim); + writer.EndStep(); + writer.Close(); + +} + + +} \ No newline at end of file From 42487b3a48772aa6bf47e72bd6ab86eecbb23fb4 Mon Sep 17 00:00:00 2001 From: ampdes Date: Wed, 19 Jun 2024 02:53:22 +0200 Subject: [PATCH 02/60] Update CMakeLists and dolfinx_io --- cpp/dolfinx/io/dolfinx_io.h | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/dolfinx/io/dolfinx_io.h b/cpp/dolfinx/io/dolfinx_io.h index 551fecffdcf..f8f499f049d 100644 --- a/cpp/dolfinx/io/dolfinx_io.h +++ b/cpp/dolfinx/io/dolfinx_io.h @@ -11,3 +11,4 @@ namespace dolfinx::io #include #include +#include From bf4080287727e371b4282f4cebf07807469e262d Mon Sep 17 00:00:00 2001 From: ampdes Date: Wed, 19 Jun 2024 02:59:55 +0200 Subject: [PATCH 03/60] Add checkpointing demo --- cpp/demo/checkpointing/CMakeLists.txt | 40 +++++++++++++++++++++++++++ cpp/demo/checkpointing/main.cpp | 25 +++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 cpp/demo/checkpointing/CMakeLists.txt create mode 100644 cpp/demo/checkpointing/main.cpp diff --git a/cpp/demo/checkpointing/CMakeLists.txt b/cpp/demo/checkpointing/CMakeLists.txt new file mode 100644 index 00000000000..bd43a542bfd --- /dev/null +++ b/cpp/demo/checkpointing/CMakeLists.txt @@ -0,0 +1,40 @@ +# This file was generated by running +# +# python cmake/scripts/generate-cmakefiles.py from dolfinx/cpp +# +cmake_minimum_required(VERSION 3.19) + +set(PROJECT_NAME demo_checkpointing) +project(${PROJECT_NAME} LANGUAGES C CXX) + +# Set C++20 standard +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +if(NOT TARGET dolfinx) + find_package(DOLFINX REQUIRED) +endif() + +set(CMAKE_INCLUDE_CURRENT_DIR ON) + +add_executable(${PROJECT_NAME} main.cpp) +target_link_libraries(${PROJECT_NAME} dolfinx) + +# Do not throw error for 'multi-line comments' (these are typical in rst which +# includes LaTeX) +include(CheckCXXCompilerFlag) +check_cxx_compiler_flag("-Wno-comment" HAVE_NO_MULTLINE) +set_source_files_properties( + main.cpp + PROPERTIES + COMPILE_FLAGS + "$<$:-Wno-comment -Wall -Wextra -pedantic -Werror>" +) + +# Test targets (used by DOLFINx testing system) +set(TEST_PARAMETERS2 -np 2 ${MPIEXEC_PARAMS} "./${PROJECT_NAME}") +set(TEST_PARAMETERS3 -np 3 ${MPIEXEC_PARAMS} "./${PROJECT_NAME}") +add_test(NAME ${PROJECT_NAME}_mpi_2 COMMAND "mpirun" ${TEST_PARAMETERS2}) +add_test(NAME ${PROJECT_NAME}_mpi_3 COMMAND "mpirun" ${TEST_PARAMETERS3}) +add_test(NAME ${PROJECT_NAME}_serial COMMAND ${PROJECT_NAME}) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp new file mode 100644 index 00000000000..0c99db69dda --- /dev/null +++ b/cpp/demo/checkpointing/main.cpp @@ -0,0 +1,25 @@ +// # Checkpointing +// + +#include +#include + +using namespace dolfinx; + +int main(int argc, char* argv[]) +{ + dolfinx::init_logging(argc, argv); + MPI_Init(&argc, &argv); + + // Create mesh and function space + auto part = mesh::create_cell_partitioner(mesh::GhostMode::shared_facet); + auto mesh = std::make_shared>( + mesh::create_rectangle(MPI_COMM_WORLD, {{{0.0, 0.0}, {1.0, 1.0}}}, + {4, 4}, mesh::CellType::quadrilateral, part)); + + io::checkpointing::write(MPI_COMM_WORLD, "mesh.bp", "mesh-write", mesh); + + MPI_Finalize(); + return 0; + +} \ No newline at end of file From e66b319583dc83a297c8cc6ef005193135946cb2 Mon Sep 17 00:00:00 2001 From: ampdes Date: Fri, 21 Jun 2024 01:11:56 +0200 Subject: [PATCH 04/60] Update dolfinx/io CMakeLists --- cpp/dolfinx/io/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/dolfinx/io/CMakeLists.txt b/cpp/dolfinx/io/CMakeLists.txt index a3060dd6852..340c34f8d36 100644 --- a/cpp/dolfinx/io/CMakeLists.txt +++ b/cpp/dolfinx/io/CMakeLists.txt @@ -2,6 +2,7 @@ set(HEADERS_io ${CMAKE_CURRENT_SOURCE_DIR}/dolfinx_io.h ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.h ${CMAKE_CURRENT_SOURCE_DIR}/cells.h + ${CMAKE_CURRENT_SOURCE_DIR}/checkpointing.h ${CMAKE_CURRENT_SOURCE_DIR}/HDF5Interface.h ${CMAKE_CURRENT_SOURCE_DIR}/vtk_utils.h ${CMAKE_CURRENT_SOURCE_DIR}/VTKFile.h From bf786378f6da10574ee429653a354e40d2bafe3c Mon Sep 17 00:00:00 2001 From: ampdes Date: Thu, 4 Jul 2024 06:15:14 +0200 Subject: [PATCH 05/60] Add implementation of mesh write Update cmakelists --- cpp/dolfinx/io/CMakeLists.txt | 1 + cpp/dolfinx/io/checkpointing.cpp | 129 +++++++++++++++++++++++++++++++ cpp/dolfinx/io/checkpointing.h | 28 +++---- 3 files changed, 140 insertions(+), 18 deletions(-) create mode 100644 cpp/dolfinx/io/checkpointing.cpp diff --git a/cpp/dolfinx/io/CMakeLists.txt b/cpp/dolfinx/io/CMakeLists.txt index 340c34f8d36..9374d06c067 100644 --- a/cpp/dolfinx/io/CMakeLists.txt +++ b/cpp/dolfinx/io/CMakeLists.txt @@ -17,6 +17,7 @@ target_sources( dolfinx PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/cells.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/checkpointing.cpp ${CMAKE_CURRENT_SOURCE_DIR}/HDF5Interface.cpp ${CMAKE_CURRENT_SOURCE_DIR}/VTKFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/vtk_utils.cpp diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp new file mode 100644 index 00000000000..dbe3b82473c --- /dev/null +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -0,0 +1,129 @@ +// Copyright (C) year authors +// +// This file is part of DOLFINX (https://www.fenicsproject.org) +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#ifdef HAS_ADIOS2 + +#include "checkpointing.h" +#include +#include +#include +#include + +/// @file checkpointing.h +/// @brief ADIOS2 based checkpointing + +std::map lagrange_variants { + {basix::element::lagrange_variant::unset, "unset"}, + {basix::element::lagrange_variant::equispaced, "equispaced"}, + {basix::element::lagrange_variant::gll_warped, "gll_warped"}, + }; + +using namespace dolfinx; +using namespace dolfinx::io; + +//----------------------------------------------------------------------------- +template +void checkpointing::write(MPI_Comm comm, std::string filename, + std::string tag, std::shared_ptr> mesh) +{ + adios2::ADIOS adios(comm); + adios2::IO io = adios.DeclareIO(tag); + adios2::Engine writer = io.Open(filename, adios2::Mode::Write); + + const mesh::Geometry& geometry = mesh->geometry(); + auto topology = mesh->topology(); + + const std::int16_t mesh_dim = geometry.dim(); + const std::vector mesh_input_global_indices = geometry.input_global_indices(); + const std::span mesh_input_global_indices_span(mesh_input_global_indices.begin(), + mesh_input_global_indices.end()); + const std::span mesh_x = geometry.x(); + + auto imap = mesh->geometry().index_map(); + const std::int64_t num_nodes_global = imap->size_global(); + const std::int32_t num_nodes_local = imap->size_local(); + const std::int64_t offset = imap->local_range()[0]; + + auto dmap = mesh->geometry().dofmap(); + + const std::shared_ptr topo_imap = topology->index_map(mesh_dim); + const std::int64_t num_cells_global = topo_imap->size_global(); + const std::int32_t num_cells_local = topo_imap->size_local(); + const std::int64_t cell_offset = topo_imap->local_range()[0]; + + auto cmap = mesh->geometry().cmap(); + auto edegree = cmap.degree(); + auto ecelltype = cmap.cell_shape(); + auto elagrange_variant = cmap.variant(); + auto geom_layout = cmap.create_dof_layout(); + int num_dofs_per_cell = geom_layout.num_entity_closure_dofs(mesh_dim); + + io.DefineAttribute("name", mesh->name); + io.DefineAttribute("dim", geometry.dim()); + io.DefineAttribute("CellType", mesh::to_string(cmap.cell_shape())); + io.DefineAttribute("Degree", cmap.degree()); + io.DefineAttribute("LagrangeVariant", lagrange_variants[elagrange_variant]); + + adios2::Variable n_nodes = io.DefineVariable("n_nodes"); + adios2::Variable n_cells = io.DefineVariable("n_cells"); + adios2::Variable n_dofs_per_cell = io.DefineVariable("n_dofs_per_cell"); + + adios2::Variable input_global_indices = io.DefineVariable("input_global_indices", + {num_nodes_global}, + {offset}, + {num_nodes_local}, + adios2::ConstantDims); + + adios2::Variable x = io.DefineVariable("Points", + {num_nodes_global, 3}, + {offset, 0}, + {num_nodes_local, 3}, + adios2::ConstantDims); + + adios2::Variable cell_indices = io.DefineVariable("cell_indices", + {num_cells_global*num_dofs_per_cell}, + {cell_offset*num_dofs_per_cell}, + {num_cells_local*num_dofs_per_cell}, + adios2::ConstantDims); + + adios2::Variable cell_indices_offsets = io.DefineVariable("cell_indices_offsets", + {num_cells_global+1}, + {cell_offset}, + {num_cells_local+1}, + adios2::ConstantDims); + + auto connectivity = topology->connectivity(mesh_dim, 0); + auto indices = connectivity->array(); + const std::span indices_span(indices.begin(), + indices.end()); + + auto indices_offsets = connectivity->offsets(); + for (std::size_t i = 0; i < indices_offsets.size(); ++i) + { + indices_offsets[i] += cell_offset*num_dofs_per_cell; + } + + const std::span indices_offsets_span(indices_offsets.begin(), + indices_offsets.end()); + + std::vector connectivity_nodes_global(indices_offsets[num_cells_local]); + + imap->local_to_global(indices_span.subspan(0, indices_offsets[num_cells_local]), connectivity_nodes_global); + + writer.BeginStep(); + writer.Put(n_nodes, num_nodes_global); + writer.Put(n_cells, num_cells_global); + writer.Put(n_dofs_per_cell, num_dofs_per_cell); + writer.Put(input_global_indices, mesh_input_global_indices_span.subspan(0, num_nodes_local).data()); + writer.Put(x, mesh_x.subspan(0, num_nodes_local*3).data()); + writer.Put(cell_indices, connectivity_nodes_global.data()); + writer.Put(cell_indices_offsets, indices_offsets_span.subspan(0, num_cells_local+1).data()); + writer.EndStep(); + writer.Close(); + +} + +#endif \ No newline at end of file diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 5bfa7322f03..555e1403dc9 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -4,9 +4,14 @@ // // SPDX-License-Identifier: LGPL-3.0-or-later +#pragma once + +#ifdef HAS_ADIOS2 + #include #include #include +#include /// @file checkpointing.h /// @brief ADIOS2 based checkpointing @@ -14,24 +19,11 @@ namespace dolfinx::io::checkpointing { -void write(MPI_Comm comm, const std::filesystem::path& filename, - std::string tag, std::shared_ptr> mesh) - { - adios2::ADIOS adios(comm); - adios2::IO io = adios.DeclareIO(tag); - adios2::Engine writer = io.Open(filename, adios2::Mode::Write); - - const std::string mesh_name = mesh->name; - const std::int16_t mesh_dim = mesh->geometry().dim(); - adios2::Variable name = io.DefineVariable("name"); - adios2::Variable dim = io.DefineVariable("dim"); - writer.BeginStep(); - writer.Put(name, mesh_name); - writer.Put(dim, mesh_dim); - writer.EndStep(); - writer.Close(); +template +void write(MPI_Comm comm, std::string filename, + std::string tag, std::shared_ptr> mesh); -} +} //namespace dolfinx::io::checkpointing -} \ No newline at end of file +#endif \ No newline at end of file From 339c1f0db71f0b2bb063e394580780fdc557d975 Mon Sep 17 00:00:00 2001 From: ampdes Date: Thu, 4 Jul 2024 11:08:38 +0200 Subject: [PATCH 06/60] Remove const and clean --- cpp/dolfinx/io/checkpointing.cpp | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index dbe3b82473c..d2381a88571 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -36,28 +36,23 @@ void checkpointing::write(MPI_Comm comm, std::string filename, const mesh::Geometry& geometry = mesh->geometry(); auto topology = mesh->topology(); - const std::int16_t mesh_dim = geometry.dim(); + std::int16_t mesh_dim = geometry.dim(); const std::vector mesh_input_global_indices = geometry.input_global_indices(); const std::span mesh_input_global_indices_span(mesh_input_global_indices.begin(), mesh_input_global_indices.end()); const std::span mesh_x = geometry.x(); auto imap = mesh->geometry().index_map(); - const std::int64_t num_nodes_global = imap->size_global(); - const std::int32_t num_nodes_local = imap->size_local(); - const std::int64_t offset = imap->local_range()[0]; - - auto dmap = mesh->geometry().dofmap(); + std::int64_t num_nodes_global = imap->size_global(); + std::int32_t num_nodes_local = imap->size_local(); + std::int64_t offset = imap->local_range()[0]; const std::shared_ptr topo_imap = topology->index_map(mesh_dim); - const std::int64_t num_cells_global = topo_imap->size_global(); - const std::int32_t num_cells_local = topo_imap->size_local(); - const std::int64_t cell_offset = topo_imap->local_range()[0]; + std::int64_t num_cells_global = topo_imap->size_global(); + std::int32_t num_cells_local = topo_imap->size_local(); + std::int64_t cell_offset = topo_imap->local_range()[0]; auto cmap = mesh->geometry().cmap(); - auto edegree = cmap.degree(); - auto ecelltype = cmap.cell_shape(); - auto elagrange_variant = cmap.variant(); auto geom_layout = cmap.create_dof_layout(); int num_dofs_per_cell = geom_layout.num_entity_closure_dofs(mesh_dim); @@ -65,7 +60,7 @@ void checkpointing::write(MPI_Comm comm, std::string filename, io.DefineAttribute("dim", geometry.dim()); io.DefineAttribute("CellType", mesh::to_string(cmap.cell_shape())); io.DefineAttribute("Degree", cmap.degree()); - io.DefineAttribute("LagrangeVariant", lagrange_variants[elagrange_variant]); + io.DefineAttribute("LagrangeVariant", lagrange_variants[cmap.variant()]); adios2::Variable n_nodes = io.DefineVariable("n_nodes"); adios2::Variable n_cells = io.DefineVariable("n_cells"); From 6ebac6e0467fefb80193fb77d66a80d62d96d9b6 Mon Sep 17 00:00:00 2001 From: ampdes Date: Thu, 4 Jul 2024 11:09:12 +0200 Subject: [PATCH 07/60] Add checkpointing demo to regression tests --- cpp/demo/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/demo/CMakeLists.txt b/cpp/demo/CMakeLists.txt index be4d0a305c0..b54af00ee0f 100644 --- a/cpp/demo/CMakeLists.txt +++ b/cpp/demo/CMakeLists.txt @@ -23,3 +23,4 @@ add_demo_subdirectory(hyperelasticity) add_demo_subdirectory(interpolation-io) add_demo_subdirectory(interpolation_different_meshes) add_demo_subdirectory(biharmonic) +add_demo_subdirectory(checkpointing) From 5a4d1900de550ea586447e8e9989295ce617ad65 Mon Sep 17 00:00:00 2001 From: ampdes Date: Thu, 4 Jul 2024 11:13:58 +0200 Subject: [PATCH 08/60] clang formatting --- cpp/dolfinx/io/checkpointing.cpp | 205 ++++++++++++++++--------------- cpp/dolfinx/io/checkpointing.h | 13 +- 2 files changed, 111 insertions(+), 107 deletions(-) diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index d2381a88571..1c5c8efd375 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -7,118 +7,123 @@ #ifdef HAS_ADIOS2 #include "checkpointing.h" -#include #include -#include #include +#include +#include /// @file checkpointing.h /// @brief ADIOS2 based checkpointing -std::map lagrange_variants { - {basix::element::lagrange_variant::unset, "unset"}, - {basix::element::lagrange_variant::equispaced, "equispaced"}, - {basix::element::lagrange_variant::gll_warped, "gll_warped"}, - }; +std::map lagrange_variants{ + {basix::element::lagrange_variant::unset, "unset"}, + {basix::element::lagrange_variant::equispaced, "equispaced"}, + {basix::element::lagrange_variant::gll_warped, "gll_warped"}, +}; using namespace dolfinx; using namespace dolfinx::io; //----------------------------------------------------------------------------- template -void checkpointing::write(MPI_Comm comm, std::string filename, - std::string tag, std::shared_ptr> mesh) +void checkpointing::write(MPI_Comm comm, std::string filename, std::string tag, + std::shared_ptr> mesh) { - adios2::ADIOS adios(comm); - adios2::IO io = adios.DeclareIO(tag); - adios2::Engine writer = io.Open(filename, adios2::Mode::Write); - - const mesh::Geometry& geometry = mesh->geometry(); - auto topology = mesh->topology(); - - std::int16_t mesh_dim = geometry.dim(); - const std::vector mesh_input_global_indices = geometry.input_global_indices(); - const std::span mesh_input_global_indices_span(mesh_input_global_indices.begin(), - mesh_input_global_indices.end()); - const std::span mesh_x = geometry.x(); - - auto imap = mesh->geometry().index_map(); - std::int64_t num_nodes_global = imap->size_global(); - std::int32_t num_nodes_local = imap->size_local(); - std::int64_t offset = imap->local_range()[0]; - - const std::shared_ptr topo_imap = topology->index_map(mesh_dim); - std::int64_t num_cells_global = topo_imap->size_global(); - std::int32_t num_cells_local = topo_imap->size_local(); - std::int64_t cell_offset = topo_imap->local_range()[0]; - - auto cmap = mesh->geometry().cmap(); - auto geom_layout = cmap.create_dof_layout(); - int num_dofs_per_cell = geom_layout.num_entity_closure_dofs(mesh_dim); - - io.DefineAttribute("name", mesh->name); - io.DefineAttribute("dim", geometry.dim()); - io.DefineAttribute("CellType", mesh::to_string(cmap.cell_shape())); - io.DefineAttribute("Degree", cmap.degree()); - io.DefineAttribute("LagrangeVariant", lagrange_variants[cmap.variant()]); - - adios2::Variable n_nodes = io.DefineVariable("n_nodes"); - adios2::Variable n_cells = io.DefineVariable("n_cells"); - adios2::Variable n_dofs_per_cell = io.DefineVariable("n_dofs_per_cell"); - - adios2::Variable input_global_indices = io.DefineVariable("input_global_indices", - {num_nodes_global}, - {offset}, - {num_nodes_local}, - adios2::ConstantDims); - - adios2::Variable x = io.DefineVariable("Points", - {num_nodes_global, 3}, - {offset, 0}, - {num_nodes_local, 3}, - adios2::ConstantDims); - - adios2::Variable cell_indices = io.DefineVariable("cell_indices", - {num_cells_global*num_dofs_per_cell}, - {cell_offset*num_dofs_per_cell}, - {num_cells_local*num_dofs_per_cell}, - adios2::ConstantDims); - - adios2::Variable cell_indices_offsets = io.DefineVariable("cell_indices_offsets", - {num_cells_global+1}, - {cell_offset}, - {num_cells_local+1}, - adios2::ConstantDims); - - auto connectivity = topology->connectivity(mesh_dim, 0); - auto indices = connectivity->array(); - const std::span indices_span(indices.begin(), - indices.end()); - - auto indices_offsets = connectivity->offsets(); - for (std::size_t i = 0; i < indices_offsets.size(); ++i) - { - indices_offsets[i] += cell_offset*num_dofs_per_cell; - } - - const std::span indices_offsets_span(indices_offsets.begin(), - indices_offsets.end()); - - std::vector connectivity_nodes_global(indices_offsets[num_cells_local]); - - imap->local_to_global(indices_span.subspan(0, indices_offsets[num_cells_local]), connectivity_nodes_global); - - writer.BeginStep(); - writer.Put(n_nodes, num_nodes_global); - writer.Put(n_cells, num_cells_global); - writer.Put(n_dofs_per_cell, num_dofs_per_cell); - writer.Put(input_global_indices, mesh_input_global_indices_span.subspan(0, num_nodes_local).data()); - writer.Put(x, mesh_x.subspan(0, num_nodes_local*3).data()); - writer.Put(cell_indices, connectivity_nodes_global.data()); - writer.Put(cell_indices_offsets, indices_offsets_span.subspan(0, num_cells_local+1).data()); - writer.EndStep(); - writer.Close(); - + adios2::ADIOS adios(comm); + adios2::IO io = adios.DeclareIO(tag); + adios2::Engine writer = io.Open(filename, adios2::Mode::Write); + + const mesh::Geometry& geometry = mesh->geometry(); + auto topology = mesh->topology(); + + std::int16_t mesh_dim = geometry.dim(); + const std::vector mesh_input_global_indices + = geometry.input_global_indices(); + const std::span mesh_input_global_indices_span( + mesh_input_global_indices.begin(), mesh_input_global_indices.end()); + const std::span mesh_x = geometry.x(); + + auto imap = mesh->geometry().index_map(); + std::int64_t num_nodes_global = imap->size_global(); + std::int32_t num_nodes_local = imap->size_local(); + std::int64_t offset = imap->local_range()[0]; + + const std::shared_ptr topo_imap + = topology->index_map(mesh_dim); + std::int64_t num_cells_global = topo_imap->size_global(); + std::int32_t num_cells_local = topo_imap->size_local(); + std::int64_t cell_offset = topo_imap->local_range()[0]; + + auto cmap = mesh->geometry().cmap(); + auto geom_layout = cmap.create_dof_layout(); + int num_dofs_per_cell = geom_layout.num_entity_closure_dofs(mesh_dim); + + io.DefineAttribute("name", mesh->name); + io.DefineAttribute("dim", geometry.dim()); + io.DefineAttribute("CellType", + mesh::to_string(cmap.cell_shape())); + io.DefineAttribute("Degree", cmap.degree()); + io.DefineAttribute("LagrangeVariant", + lagrange_variants[cmap.variant()]); + + adios2::Variable n_nodes + = io.DefineVariable("n_nodes"); + adios2::Variable n_cells + = io.DefineVariable("n_cells"); + adios2::Variable n_dofs_per_cell + = io.DefineVariable("n_dofs_per_cell"); + + adios2::Variable input_global_indices + = io.DefineVariable( + "input_global_indices", {num_nodes_global}, {offset}, + {num_nodes_local}, adios2::ConstantDims); + + adios2::Variable x + = io.DefineVariable("Points", {num_nodes_global, 3}, {offset, 0}, + {num_nodes_local, 3}, adios2::ConstantDims); + + adios2::Variable cell_indices = io.DefineVariable( + "cell_indices", {num_cells_global * num_dofs_per_cell}, + {cell_offset * num_dofs_per_cell}, {num_cells_local * num_dofs_per_cell}, + adios2::ConstantDims); + + adios2::Variable cell_indices_offsets + = io.DefineVariable( + "cell_indices_offsets", {num_cells_global + 1}, {cell_offset}, + {num_cells_local + 1}, adios2::ConstantDims); + + auto connectivity = topology->connectivity(mesh_dim, 0); + auto indices = connectivity->array(); + const std::span indices_span(indices.begin(), indices.end()); + + auto indices_offsets = connectivity->offsets(); + for (std::size_t i = 0; i < indices_offsets.size(); ++i) + { + indices_offsets[i] += cell_offset * num_dofs_per_cell; + } + + const std::span indices_offsets_span(indices_offsets.begin(), + indices_offsets.end()); + + std::vector connectivity_nodes_global( + indices_offsets[num_cells_local]); + + imap->local_to_global( + indices_span.subspan(0, indices_offsets[num_cells_local]), + connectivity_nodes_global); + + writer.BeginStep(); + writer.Put(n_nodes, num_nodes_global); + writer.Put(n_cells, num_cells_global); + writer.Put(n_dofs_per_cell, num_dofs_per_cell); + writer.Put(input_global_indices, + mesh_input_global_indices_span.subspan(0, num_nodes_local).data()); + writer.Put(x, mesh_x.subspan(0, num_nodes_local * 3).data()); + writer.Put(cell_indices, connectivity_nodes_global.data()); + writer.Put(cell_indices_offsets, + indices_offsets_span.subspan(0, num_cells_local + 1).data()); + writer.EndStep(); + writer.Close(); } -#endif \ No newline at end of file +#endif diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 555e1403dc9..231922058d0 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -8,10 +8,10 @@ #ifdef HAS_ADIOS2 -#include #include -#include #include +#include +#include /// @file checkpointing.h /// @brief ADIOS2 based checkpointing @@ -20,10 +20,9 @@ namespace dolfinx::io::checkpointing { template -void write(MPI_Comm comm, std::string filename, - std::string tag, std::shared_ptr> mesh); - +void write(MPI_Comm comm, std::string filename, std::string tag, + std::shared_ptr> mesh); -} //namespace dolfinx::io::checkpointing +} // namespace dolfinx::io::checkpointing -#endif \ No newline at end of file +#endif From 1d97e78c91b87f28aaed548e84ed53cde3928727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20S=2E=20Dokken?= Date: Fri, 5 Jul 2024 15:03:50 +0000 Subject: [PATCH 09/60] Minor cahnges in layout --- cpp/demo/checkpointing/main.cpp | 12 +++++++----- cpp/dolfinx/io/checkpointing.cpp | 32 +++++++++++++++++++++++--------- cpp/dolfinx/io/checkpointing.h | 5 +++-- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 0c99db69dda..cec594d715a 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -2,6 +2,7 @@ // #include +#include #include using namespace dolfinx; @@ -13,13 +14,14 @@ int main(int argc, char* argv[]) // Create mesh and function space auto part = mesh::create_cell_partitioner(mesh::GhostMode::shared_facet); - auto mesh = std::make_shared>( + auto mesh = std::make_shared>( mesh::create_rectangle(MPI_COMM_WORLD, {{{0.0, 0.0}, {1.0, 1.0}}}, - {4, 4}, mesh::CellType::quadrilateral, part)); - - io::checkpointing::write(MPI_COMM_WORLD, "mesh.bp", "mesh-write", mesh); + {4, 4}, mesh::CellType::quadrilateral, + part)); + dolfinx::io::checkpointing::write(mesh->comm(), "mesh.bp", "mesh-write", + mesh); + dolfinx::io::checkpointing::test(); MPI_Finalize(); return 0; - } \ No newline at end of file diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 1c5c8efd375..5e7e4c3c93e 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -12,28 +12,28 @@ #include #include +using namespace dolfinx; +using namespace dolfinx::io; + /// @file checkpointing.h /// @brief ADIOS2 based checkpointing - +namespace +{ std::map lagrange_variants{ {basix::element::lagrange_variant::unset, "unset"}, {basix::element::lagrange_variant::equispaced, "equispaced"}, {basix::element::lagrange_variant::gll_warped, "gll_warped"}, }; -using namespace dolfinx; -using namespace dolfinx::io; - -//----------------------------------------------------------------------------- template -void checkpointing::write(MPI_Comm comm, std::string filename, std::string tag, - std::shared_ptr> mesh) +void _write(MPI_Comm comm, std::string filename, std::string tag, + std::shared_ptr> mesh) { adios2::ADIOS adios(comm); adios2::IO io = adios.DeclareIO(tag); adios2::Engine writer = io.Open(filename, adios2::Mode::Write); - const mesh::Geometry& geometry = mesh->geometry(); + const dolfinx::mesh::Geometry& geometry = mesh->geometry(); auto topology = mesh->topology(); std::int16_t mesh_dim = geometry.dim(); @@ -61,7 +61,7 @@ void checkpointing::write(MPI_Comm comm, std::string filename, std::string tag, io.DefineAttribute("name", mesh->name); io.DefineAttribute("dim", geometry.dim()); io.DefineAttribute("CellType", - mesh::to_string(cmap.cell_shape())); + dolfinx::mesh::to_string(cmap.cell_shape())); io.DefineAttribute("Degree", cmap.degree()); io.DefineAttribute("LagrangeVariant", lagrange_variants[cmap.variant()]); @@ -126,4 +126,18 @@ void checkpointing::write(MPI_Comm comm, std::string filename, std::string tag, writer.Close(); } +} // namespace + +using namespace dolfinx::io::checkpointing; + +void dolfinx::io::checkpointing::write( + MPI_Comm comm, std::string filename, std::string tag, + std::shared_ptr> mesh) +{ + + _write(comm, filename, tag, mesh); +} + +//----------------------------------------------------------------------------- + #endif diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 231922058d0..2211d231142 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -19,9 +19,10 @@ namespace dolfinx::io::checkpointing { -template +void test() { std::cout << "HELLO" << std::endl; }; + void write(MPI_Comm comm, std::string filename, std::string tag, - std::shared_ptr> mesh); + std::shared_ptr> mesh); } // namespace dolfinx::io::checkpointing From 1b2a243614a41fc87e4b24f63e8e2a27fc61ec57 Mon Sep 17 00:00:00 2001 From: ampdes Date: Fri, 5 Jul 2024 21:09:55 +0200 Subject: [PATCH 10/60] Cast adios2 variable creation args to unsigned long int --- cpp/demo/checkpointing/main.cpp | 16 +++++++-------- cpp/dolfinx/io/checkpointing.cpp | 35 ++++++++++++++++++++++---------- cpp/dolfinx/io/checkpointing.h | 5 +++-- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index cec594d715a..ebe8409d34e 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -1,8 +1,8 @@ // # Checkpointing // +#include #include -#include #include using namespace dolfinx; @@ -14,14 +14,12 @@ int main(int argc, char* argv[]) // Create mesh and function space auto part = mesh::create_cell_partitioner(mesh::GhostMode::shared_facet); - auto mesh = std::make_shared>( - mesh::create_rectangle(MPI_COMM_WORLD, {{{0.0, 0.0}, {1.0, 1.0}}}, - {4, 4}, mesh::CellType::quadrilateral, - part)); + auto mesh = std::make_shared>(mesh::create_rectangle( + MPI_COMM_WORLD, {{{0.0, 0.0}, {1.0, 1.0}}}, {4, 4}, + mesh::CellType::quadrilateral, part)); + + io::checkpointing::write(mesh->comm(), "mesh.bp", "mesh-write", mesh); - dolfinx::io::checkpointing::write(mesh->comm(), "mesh.bp", "mesh-write", - mesh); - dolfinx::io::checkpointing::test(); MPI_Finalize(); return 0; -} \ No newline at end of file +} diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 5e7e4c3c93e..6c0ef7e2812 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -56,7 +56,7 @@ void _write(MPI_Comm comm, std::string filename, std::string tag, auto cmap = mesh->geometry().cmap(); auto geom_layout = cmap.create_dof_layout(); - int num_dofs_per_cell = geom_layout.num_entity_closure_dofs(mesh_dim); + auto num_dofs_per_cell = geom_layout.num_entity_closure_dofs(mesh_dim); io.DefineAttribute("name", mesh->name); io.DefineAttribute("dim", geometry.dim()); @@ -75,22 +75,27 @@ void _write(MPI_Comm comm, std::string filename, std::string tag, adios2::Variable input_global_indices = io.DefineVariable( - "input_global_indices", {num_nodes_global}, {offset}, - {num_nodes_local}, adios2::ConstantDims); + "input_global_indices", {(long unsigned int)num_nodes_global}, + {(long unsigned int)offset}, {(long unsigned int)num_nodes_local}, + adios2::ConstantDims); - adios2::Variable x - = io.DefineVariable("Points", {num_nodes_global, 3}, {offset, 0}, - {num_nodes_local, 3}, adios2::ConstantDims); + adios2::Variable x = io.DefineVariable( + "Points", {(long unsigned int)num_nodes_global, 3}, + {(long unsigned int)offset, 0}, {(long unsigned int)num_nodes_local, 3}, + adios2::ConstantDims); adios2::Variable cell_indices = io.DefineVariable( - "cell_indices", {num_cells_global * num_dofs_per_cell}, - {cell_offset * num_dofs_per_cell}, {num_cells_local * num_dofs_per_cell}, + "cell_indices", + {(long unsigned int)(num_cells_global * num_dofs_per_cell)}, + {(long unsigned int)(cell_offset * num_dofs_per_cell)}, + {(long unsigned int)(num_cells_local * num_dofs_per_cell)}, adios2::ConstantDims); adios2::Variable cell_indices_offsets = io.DefineVariable( - "cell_indices_offsets", {num_cells_global + 1}, {cell_offset}, - {num_cells_local + 1}, adios2::ConstantDims); + "cell_indices_offsets", {(long unsigned int)(num_cells_global + 1)}, + {(long unsigned int)(cell_offset)}, + {(long unsigned int)(num_cells_local + 1)}, adios2::ConstantDims); auto connectivity = topology->connectivity(mesh_dim, 0); auto indices = connectivity->array(); @@ -130,6 +135,7 @@ void _write(MPI_Comm comm, std::string filename, std::string tag, using namespace dolfinx::io::checkpointing; +//----------------------------------------------------------------------------- void dolfinx::io::checkpointing::write( MPI_Comm comm, std::string filename, std::string tag, std::shared_ptr> mesh) @@ -139,5 +145,12 @@ void dolfinx::io::checkpointing::write( } //----------------------------------------------------------------------------- +void dolfinx::io::checkpointing::write( + MPI_Comm comm, std::string filename, std::string tag, + std::shared_ptr> mesh) +{ + + _write(comm, filename, tag, mesh); +} -#endif +#endif \ No newline at end of file diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 2211d231142..ea011961f2d 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -19,11 +19,12 @@ namespace dolfinx::io::checkpointing { -void test() { std::cout << "HELLO" << std::endl; }; - void write(MPI_Comm comm, std::string filename, std::string tag, std::shared_ptr> mesh); +void write(MPI_Comm comm, std::string filename, std::string tag, + std::shared_ptr> mesh); + } // namespace dolfinx::io::checkpointing #endif From f2d3115d4ed45f1722e70dbbd6495b8f695c1dbd Mon Sep 17 00:00:00 2001 From: ampdes Date: Fri, 5 Jul 2024 21:51:35 +0200 Subject: [PATCH 11/60] Declare with uint instead of cast later --- cpp/dolfinx/io/checkpointing.cpp | 49 ++++++++++++++------------------ 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 6c0ef7e2812..589a4b2ecef 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -44,15 +44,15 @@ void _write(MPI_Comm comm, std::string filename, std::string tag, const std::span mesh_x = geometry.x(); auto imap = mesh->geometry().index_map(); - std::int64_t num_nodes_global = imap->size_global(); - std::int32_t num_nodes_local = imap->size_local(); - std::int64_t offset = imap->local_range()[0]; + std::uint64_t num_nodes_global = imap->size_global(); + std::uint32_t num_nodes_local = imap->size_local(); + std::uint64_t offset = imap->local_range()[0]; const std::shared_ptr topo_imap = topology->index_map(mesh_dim); - std::int64_t num_cells_global = topo_imap->size_global(); - std::int32_t num_cells_local = topo_imap->size_local(); - std::int64_t cell_offset = topo_imap->local_range()[0]; + std::uint64_t num_cells_global = topo_imap->size_global(); + std::uint32_t num_cells_local = topo_imap->size_local(); + std::uint64_t cell_offset = topo_imap->local_range()[0]; auto cmap = mesh->geometry().cmap(); auto geom_layout = cmap.create_dof_layout(); @@ -66,36 +66,31 @@ void _write(MPI_Comm comm, std::string filename, std::string tag, io.DefineAttribute("LagrangeVariant", lagrange_variants[cmap.variant()]); - adios2::Variable n_nodes - = io.DefineVariable("n_nodes"); - adios2::Variable n_cells - = io.DefineVariable("n_cells"); - adios2::Variable n_dofs_per_cell - = io.DefineVariable("n_dofs_per_cell"); + adios2::Variable n_nodes + = io.DefineVariable("n_nodes"); + adios2::Variable n_cells + = io.DefineVariable("n_cells"); + adios2::Variable n_dofs_per_cell + = io.DefineVariable("n_dofs_per_cell"); adios2::Variable input_global_indices = io.DefineVariable( - "input_global_indices", {(long unsigned int)num_nodes_global}, - {(long unsigned int)offset}, {(long unsigned int)num_nodes_local}, - adios2::ConstantDims); + "input_global_indices", {num_nodes_global}, {offset}, + {num_nodes_local}, adios2::ConstantDims); - adios2::Variable x = io.DefineVariable( - "Points", {(long unsigned int)num_nodes_global, 3}, - {(long unsigned int)offset, 0}, {(long unsigned int)num_nodes_local, 3}, - adios2::ConstantDims); + adios2::Variable x + = io.DefineVariable("Points", {num_nodes_global, 3}, {offset, 0}, + {num_nodes_local, 3}, adios2::ConstantDims); adios2::Variable cell_indices = io.DefineVariable( - "cell_indices", - {(long unsigned int)(num_cells_global * num_dofs_per_cell)}, - {(long unsigned int)(cell_offset * num_dofs_per_cell)}, - {(long unsigned int)(num_cells_local * num_dofs_per_cell)}, + "cell_indices", {num_cells_global * num_dofs_per_cell}, + {cell_offset * num_dofs_per_cell}, {num_cells_local * num_dofs_per_cell}, adios2::ConstantDims); adios2::Variable cell_indices_offsets = io.DefineVariable( - "cell_indices_offsets", {(long unsigned int)(num_cells_global + 1)}, - {(long unsigned int)(cell_offset)}, - {(long unsigned int)(num_cells_local + 1)}, adios2::ConstantDims); + "cell_indices_offsets", {num_cells_global + 1}, {cell_offset}, + {num_cells_local + 1}, adios2::ConstantDims); auto connectivity = topology->connectivity(mesh_dim, 0); auto indices = connectivity->array(); @@ -153,4 +148,4 @@ void dolfinx::io::checkpointing::write( _write(comm, filename, tag, mesh); } -#endif \ No newline at end of file +#endif From 5dd47c6eda96ae2ac732ce00cda5438cddd75226 Mon Sep 17 00:00:00 2001 From: ampdes Date: Tue, 9 Jul 2024 13:36:26 +0200 Subject: [PATCH 12/60] Fix offset computations Debug local-to-global mapping --- cpp/demo/checkpointing/main.cpp | 7 ++- cpp/dolfinx/io/checkpointing.cpp | 78 +++++++++++++++++++++----------- 2 files changed, 58 insertions(+), 27 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index ebe8409d34e..84abe1dd4a7 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -1,6 +1,5 @@ // # Checkpointing // -#include #include #include @@ -12,6 +11,12 @@ int main(int argc, char* argv[]) dolfinx::init_logging(argc, argv); MPI_Init(&argc, &argv); + // { + // int i=0; + // while (i == 0) + // sleep(5); + // } + // Create mesh and function space auto part = mesh::create_cell_partitioner(mesh::GhostMode::shared_facet); auto mesh = std::make_shared>(mesh::create_rectangle( diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 589a4b2ecef..4ce1cf81f84 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -37,11 +37,6 @@ void _write(MPI_Comm comm, std::string filename, std::string tag, auto topology = mesh->topology(); std::int16_t mesh_dim = geometry.dim(); - const std::vector mesh_input_global_indices - = geometry.input_global_indices(); - const std::span mesh_input_global_indices_span( - mesh_input_global_indices.begin(), mesh_input_global_indices.end()); - const std::span mesh_x = geometry.x(); auto imap = mesh->geometry().index_map(); std::uint64_t num_nodes_global = imap->size_global(); @@ -56,7 +51,58 @@ void _write(MPI_Comm comm, std::string filename, std::string tag, auto cmap = mesh->geometry().cmap(); auto geom_layout = cmap.create_dof_layout(); - auto num_dofs_per_cell = geom_layout.num_entity_closure_dofs(mesh_dim); + std::uint32_t num_dofs_per_cell = geom_layout.num_entity_closure_dofs(mesh_dim); + + const std::vector mesh_input_global_indices + = geometry.input_global_indices(); + const std::span mesh_input_global_indices_span( + mesh_input_global_indices.begin(), mesh_input_global_indices.end()); + const std::span mesh_x = geometry.x(); + + auto connectivity = topology->connectivity(mesh_dim, 0); + auto indices = connectivity->array(); + const std::span indices_span(indices.begin(), indices.end()); + + auto indices_offsets = connectivity->offsets(); + + std::vector connectivity_nodes_global( + indices_offsets[num_cells_local]); + + std::iota(connectivity_nodes_global.begin(), connectivity_nodes_global.end(), 0); + + std::cout << indices.size() << "\n"; + std::cout << indices_offsets[num_cells_local] << "\n"; + std::cout << indices_offsets[num_cells_local-1] << "\n"; + +// for (std::size_t i = 0; i < connectivity_nodes_global.size(); ++i) +// { +// std::cout << i << " "; +// std::cout << indices[i] << " "; +// std::cout << mesh_input_global_indices[indices[i]] << "\n"; +// connectivity_nodes_global[i] = mesh_input_global_indices[indices[i]]; +// } + + std::cout << indices_span.subspan(0, indices_offsets[num_cells_local]).size() << std::endl; + std::cout << indices_offsets[num_cells_local] << std::endl; + imap->local_to_global( + indices_span.subspan(0, indices_offsets[num_cells_local]), + connectivity_nodes_global); + + for (std::size_t i = 0; i < connectivity_nodes_global.size(); ++i) + { + std::cout << i << " "; + std::cout << indices[i] << " "; + std::cout << connectivity_nodes_global[i] << " "; + std::cout << mesh_input_global_indices[indices[i]] << "\n"; + } + + for (std::size_t i = 0; i < indices_offsets.size(); ++i) + { + indices_offsets[i] += cell_offset * num_dofs_per_cell; + } + + const std::span indices_offsets_span(indices_offsets.begin(), + indices_offsets.end()); io.DefineAttribute("name", mesh->name); io.DefineAttribute("dim", geometry.dim()); @@ -92,26 +138,6 @@ void _write(MPI_Comm comm, std::string filename, std::string tag, "cell_indices_offsets", {num_cells_global + 1}, {cell_offset}, {num_cells_local + 1}, adios2::ConstantDims); - auto connectivity = topology->connectivity(mesh_dim, 0); - auto indices = connectivity->array(); - const std::span indices_span(indices.begin(), indices.end()); - - auto indices_offsets = connectivity->offsets(); - for (std::size_t i = 0; i < indices_offsets.size(); ++i) - { - indices_offsets[i] += cell_offset * num_dofs_per_cell; - } - - const std::span indices_offsets_span(indices_offsets.begin(), - indices_offsets.end()); - - std::vector connectivity_nodes_global( - indices_offsets[num_cells_local]); - - imap->local_to_global( - indices_span.subspan(0, indices_offsets[num_cells_local]), - connectivity_nodes_global); - writer.BeginStep(); writer.Put(n_nodes, num_nodes_global); writer.Put(n_cells, num_cells_global); From 0e1b770d8c45db436fa52bdccee41b0effa1a610 Mon Sep 17 00:00:00 2001 From: ampdes Date: Tue, 9 Jul 2024 15:18:37 +0200 Subject: [PATCH 13/60] Add ADIOS2Engine class adapted from ADIOS2Writer --- cpp/demo/checkpointing/main.cpp | 7 ++++ cpp/dolfinx/io/ADIOS2_utils.cpp | 50 ++++++++++++++++++++++++ cpp/dolfinx/io/ADIOS2_utils.h | 66 ++++++++++++++++++++++++++++++++ cpp/dolfinx/io/CMakeLists.txt | 4 +- cpp/dolfinx/io/checkpointing.cpp | 47 ++++++++++++++--------- cpp/dolfinx/io/checkpointing.h | 3 ++ cpp/dolfinx/io/dolfinx_io.h | 1 + 7 files changed, 158 insertions(+), 20 deletions(-) create mode 100644 cpp/dolfinx/io/ADIOS2_utils.cpp create mode 100644 cpp/dolfinx/io/ADIOS2_utils.h diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 84abe1dd4a7..175730b34f6 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -23,6 +23,13 @@ int main(int argc, char* argv[]) MPI_COMM_WORLD, {{{0.0, 0.0}, {1.0, 1.0}}}, {4, 4}, mesh::CellType::quadrilateral, part)); + auto writer = io::ADIOS2Engine(mesh->comm(), "mesh.bp", "mesh-write", "BP5", + adios2::Mode::Write); + + // auto io = writer.io(); + // auto engine = writer.engine(); + + // io::checkpointing::write(io, engine, mesh); io::checkpointing::write(mesh->comm(), "mesh.bp", "mesh-write", mesh); MPI_Finalize(); diff --git a/cpp/dolfinx/io/ADIOS2_utils.cpp b/cpp/dolfinx/io/ADIOS2_utils.cpp new file mode 100644 index 00000000000..9458870c965 --- /dev/null +++ b/cpp/dolfinx/io/ADIOS2_utils.cpp @@ -0,0 +1,50 @@ +// Copyright (C) 2021-2023 Jørgen S. Dokken and Garth N. Wells +// +// This file is part of DOLFINX (https://www.fenicsproject.org) +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#ifdef HAS_ADIOS2 + +#include "ADIOS2_utils.h" +#include +#include + +using namespace dolfinx::io; + +//----------------------------------------------------------------------------- +ADIOS2Engine::ADIOS2Engine(MPI_Comm comm, const std::filesystem::path& filename, + std::string tag, std::string engine, + const adios2::Mode mode) + + : _adios(std::make_unique(comm)), + _io(std::make_unique(_adios->DeclareIO(tag))) +{ + _io->SetEngine(engine); + _engine = std::make_unique(_io->Open(filename, mode)); +} +//----------------------------------------------------------------------------- +ADIOS2Engine::~ADIOS2Engine() { close(); } +//----------------------------------------------------------------------------- +void ADIOS2Engine::close() +{ + assert(_engine); + if (*_engine) + _engine->Close(); +} +// //----------------------------------------------------------------------------- +// std::unique_ptr ADIOS2Engine::io() +// { +// assert(_io); +// if (*_io) +// return _io; +// } +// //----------------------------------------------------------------------------- +// std::unique_ptr ADIOS2Engine::engine() +// { +// assert(_engine); +// if (*_engine) +// return _engine; +// } + +#endif diff --git a/cpp/dolfinx/io/ADIOS2_utils.h b/cpp/dolfinx/io/ADIOS2_utils.h new file mode 100644 index 00000000000..88069425007 --- /dev/null +++ b/cpp/dolfinx/io/ADIOS2_utils.h @@ -0,0 +1,66 @@ +// Copyright (C) 2021-2023 Jørgen S. Dokken and Garth N. Wells +// +// This file is part of DOLFINX (https://www.fenicsproject.org) +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#pragma once + +#ifdef HAS_ADIOS2 + +#include +#include +#include +#include + +/// @file ADIOS2_utils.h +/// @brief Utils for ADIOS2 + +namespace dolfinx::io +{ +class ADIOS2Engine +{ +public: + /// @brief Create an ADIOS2-based engine writer/reader + /// @param[in] comm The MPI communicator + /// @param[in] filename Name of output file + /// @param[in] tag The ADIOS2 object name + /// @param[in] engine ADIOS2 engine type. See + /// https://adios2.readthedocs.io/en/latest/engines/engines.html. + ADIOS2Engine(MPI_Comm comm, const std::filesystem::path& filename, + std::string tag, std::string engine = "BP5", + const adios2::Mode mode = adios2::Mode::Write); + + /// @brief Move constructor + ADIOS2Engine(ADIOS2Engine&& engine) = default; + + /// @brief Copy constructor + ADIOS2Engine(const ADIOS2Engine&) = delete; + + /// @brief Destructor + ~ADIOS2Engine(); + + /// @brief Move assignment + ADIOS2Engine& operator=(ADIOS2Engine&& engine) = default; + + // Copy assignment + ADIOS2Engine& operator=(const ADIOS2Engine&) = delete; + + /// @brief Close the file + void close(); + + // /// @brief Get the IO object + // std::unique_ptr io(); + + // /// @brief Close the Engine object + // std::unique_ptr engine(); + +protected: + std::unique_ptr _adios; + std::unique_ptr _io; + std::unique_ptr _engine; +}; + +} // namespace dolfinx::io + +#endif diff --git a/cpp/dolfinx/io/CMakeLists.txt b/cpp/dolfinx/io/CMakeLists.txt index 9374d06c067..6e8c6fe59d0 100644 --- a/cpp/dolfinx/io/CMakeLists.txt +++ b/cpp/dolfinx/io/CMakeLists.txt @@ -1,5 +1,6 @@ set(HEADERS_io ${CMAKE_CURRENT_SOURCE_DIR}/dolfinx_io.h + ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2_utils.h ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.h ${CMAKE_CURRENT_SOURCE_DIR}/cells.h ${CMAKE_CURRENT_SOURCE_DIR}/checkpointing.h @@ -15,7 +16,8 @@ set(HEADERS_io target_sources( dolfinx - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.cpp + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2_utils.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/cells.cpp ${CMAKE_CURRENT_SOURCE_DIR}/checkpointing.cpp ${CMAKE_CURRENT_SOURCE_DIR}/HDF5Interface.cpp diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 4ce1cf81f84..bd2fcafc6fd 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -27,6 +27,7 @@ std::map lagrange_variants{ template void _write(MPI_Comm comm, std::string filename, std::string tag, + // adios2::IO io, adios2::Engine engine, std::shared_ptr> mesh) { adios2::ADIOS adios(comm); @@ -51,7 +52,8 @@ void _write(MPI_Comm comm, std::string filename, std::string tag, auto cmap = mesh->geometry().cmap(); auto geom_layout = cmap.create_dof_layout(); - std::uint32_t num_dofs_per_cell = geom_layout.num_entity_closure_dofs(mesh_dim); + std::uint32_t num_dofs_per_cell + = geom_layout.num_entity_closure_dofs(mesh_dim); const std::vector mesh_input_global_indices = geometry.input_global_indices(); @@ -68,33 +70,36 @@ void _write(MPI_Comm comm, std::string filename, std::string tag, std::vector connectivity_nodes_global( indices_offsets[num_cells_local]); - std::iota(connectivity_nodes_global.begin(), connectivity_nodes_global.end(), 0); + std::iota(connectivity_nodes_global.begin(), connectivity_nodes_global.end(), + 0); std::cout << indices.size() << "\n"; std::cout << indices_offsets[num_cells_local] << "\n"; - std::cout << indices_offsets[num_cells_local-1] << "\n"; - -// for (std::size_t i = 0; i < connectivity_nodes_global.size(); ++i) -// { -// std::cout << i << " "; -// std::cout << indices[i] << " "; -// std::cout << mesh_input_global_indices[indices[i]] << "\n"; -// connectivity_nodes_global[i] = mesh_input_global_indices[indices[i]]; -// } - - std::cout << indices_span.subspan(0, indices_offsets[num_cells_local]).size() << std::endl; + std::cout << indices_offsets[num_cells_local - 1] << "\n"; + + // for (std::size_t i = 0; i < connectivity_nodes_global.size(); ++i) + // { + // std::cout << i << " "; + // std::cout << indices[i] << " "; + // std::cout << mesh_input_global_indices[indices[i]] << "\n"; + // connectivity_nodes_global[i] = + // mesh_input_global_indices[indices[i]]; + // } + + std::cout << indices_span.subspan(0, indices_offsets[num_cells_local]).size() + << std::endl; std::cout << indices_offsets[num_cells_local] << std::endl; imap->local_to_global( indices_span.subspan(0, indices_offsets[num_cells_local]), connectivity_nodes_global); for (std::size_t i = 0; i < connectivity_nodes_global.size(); ++i) - { - std::cout << i << " "; - std::cout << indices[i] << " "; - std::cout << connectivity_nodes_global[i] << " "; - std::cout << mesh_input_global_indices[indices[i]] << "\n"; - } + { + std::cout << i << " "; + std::cout << indices[i] << " "; + std::cout << connectivity_nodes_global[i] << " "; + std::cout << mesh_input_global_indices[indices[i]] << "\n"; + } for (std::size_t i = 0; i < indices_offsets.size(); ++i) { @@ -159,18 +164,22 @@ using namespace dolfinx::io::checkpointing; //----------------------------------------------------------------------------- void dolfinx::io::checkpointing::write( MPI_Comm comm, std::string filename, std::string tag, + // adios2::IO io, adios2::Engine engine, std::shared_ptr> mesh) { + // _write(io, engine, mesh); _write(comm, filename, tag, mesh); } //----------------------------------------------------------------------------- void dolfinx::io::checkpointing::write( MPI_Comm comm, std::string filename, std::string tag, + // adios2::IO io, adios2::Engine engine, std::shared_ptr> mesh) { + // _write(io, engine, mesh); _write(comm, filename, tag, mesh); } diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index ea011961f2d..48437740031 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -8,6 +8,7 @@ #ifdef HAS_ADIOS2 +#include "ADIOS2_utils.h" #include #include #include @@ -20,9 +21,11 @@ namespace dolfinx::io::checkpointing { void write(MPI_Comm comm, std::string filename, std::string tag, + // adios2::IO io, adios2::Engine engine, std::shared_ptr> mesh); void write(MPI_Comm comm, std::string filename, std::string tag, + // adios2::IO io, adios2::Engine engine, std::shared_ptr> mesh); } // namespace dolfinx::io::checkpointing diff --git a/cpp/dolfinx/io/dolfinx_io.h b/cpp/dolfinx/io/dolfinx_io.h index f8f499f049d..1bb24940950 100644 --- a/cpp/dolfinx/io/dolfinx_io.h +++ b/cpp/dolfinx/io/dolfinx_io.h @@ -9,6 +9,7 @@ namespace dolfinx::io // DOLFINx io interface +#include #include #include #include From b8ebd93d28e1ffa3bd3f7e15377d634c9e19cb0f Mon Sep 17 00:00:00 2001 From: ampdes Date: Tue, 9 Jul 2024 16:19:59 +0200 Subject: [PATCH 14/60] Update documentation --- cpp/dolfinx/io/ADIOS2_utils.h | 3 +++ cpp/dolfinx/io/checkpointing.h | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/cpp/dolfinx/io/ADIOS2_utils.h b/cpp/dolfinx/io/ADIOS2_utils.h index 88069425007..8b5ac5f9c9c 100644 --- a/cpp/dolfinx/io/ADIOS2_utils.h +++ b/cpp/dolfinx/io/ADIOS2_utils.h @@ -18,6 +18,8 @@ namespace dolfinx::io { + +/// ADIOS2-based writers/readers class ADIOS2Engine { public: @@ -27,6 +29,7 @@ class ADIOS2Engine /// @param[in] tag The ADIOS2 object name /// @param[in] engine ADIOS2 engine type. See /// https://adios2.readthedocs.io/en/latest/engines/engines.html. + /// @param[in] mode ADIOS2 mode, default is Write or Read ADIOS2Engine(MPI_Comm comm, const std::filesystem::path& filename, std::string tag, std::string engine = "BP5", const adios2::Mode mode = adios2::Mode::Write); diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 48437740031..1e363fe63f5 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -20,10 +20,26 @@ namespace dolfinx::io::checkpointing { + /// @brief Write mesh to a file. + /// + /// @param[in] comm The MPI communicator to open the file on + /// @param[in] filename Name of output file + /// @param[in] tag ADIOS2 tag for IO + /// @param[in] mesh Mesh of type float to write to the file + /// @note This is experimental version. Expected would be to + /// pass ADIOS2Engine object. void write(MPI_Comm comm, std::string filename, std::string tag, // adios2::IO io, adios2::Engine engine, std::shared_ptr> mesh); + /// @brief Write mesh to a file. + /// + /// @param[in] comm The MPI communicator to open the file on + /// @param[in] filename Name of output file + /// @param[in] tag ADIOS2 tag for IO + /// @param[in] mesh Mesh of type double to write to the file + /// @note This is experimental version. Expected would be to + /// pass ADIOS2Engine object. void write(MPI_Comm comm, std::string filename, std::string tag, // adios2::IO io, adios2::Engine engine, std::shared_ptr> mesh); From 76f018ffcd1bd0b90bd5b0af498c3f282e37d8c9 Mon Sep 17 00:00:00 2001 From: ampdes Date: Tue, 9 Jul 2024 16:47:17 +0200 Subject: [PATCH 15/60] Include headers in demo --- cpp/demo/checkpointing/main.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 175730b34f6..4711b842d82 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -1,10 +1,14 @@ // # Checkpointing // +#include #include +#include +#include #include using namespace dolfinx; +using namespace dolfinx::io::ADIOS2Engine; int main(int argc, char* argv[]) { @@ -23,7 +27,7 @@ int main(int argc, char* argv[]) MPI_COMM_WORLD, {{{0.0, 0.0}, {1.0, 1.0}}}, {4, 4}, mesh::CellType::quadrilateral, part)); - auto writer = io::ADIOS2Engine(mesh->comm(), "mesh.bp", "mesh-write", "BP5", + auto writer = ADIOS2Engine(mesh->comm(), "mesh.bp", "mesh-write", "BP5", adios2::Mode::Write); // auto io = writer.io(); From 7ae238ab094e8b834f428e2ebb196ea975be2a97 Mon Sep 17 00:00:00 2001 From: ampdes Date: Tue, 9 Jul 2024 17:54:42 +0200 Subject: [PATCH 16/60] Remove demo from tests --- cpp/demo/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/demo/CMakeLists.txt b/cpp/demo/CMakeLists.txt index b54af00ee0f..be4d0a305c0 100644 --- a/cpp/demo/CMakeLists.txt +++ b/cpp/demo/CMakeLists.txt @@ -23,4 +23,3 @@ add_demo_subdirectory(hyperelasticity) add_demo_subdirectory(interpolation-io) add_demo_subdirectory(interpolation_different_meshes) add_demo_subdirectory(biharmonic) -add_demo_subdirectory(checkpointing) From eb0e8138a15a77777dbebe66280dca6c9d5b4ad3 Mon Sep 17 00:00:00 2001 From: ampdes Date: Wed, 10 Jul 2024 17:48:17 +0200 Subject: [PATCH 17/60] Update copyright text --- cpp/demo/checkpointing/main.cpp | 6 ++++++ cpp/dolfinx/io/ADIOS2_utils.cpp | 2 +- cpp/dolfinx/io/ADIOS2_utils.h | 2 +- cpp/dolfinx/io/checkpointing.cpp | 2 +- cpp/dolfinx/io/checkpointing.h | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 4711b842d82..7caeb849291 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -1,3 +1,9 @@ +// ```text +// Copyright (C) 2024 Abdullah Mujahid +// This file is part of DOLFINx (https://www.fenicsproject.org) +// SPDX-License-Identifier: LGPL-3.0-or-later +// ``` + // # Checkpointing // diff --git a/cpp/dolfinx/io/ADIOS2_utils.cpp b/cpp/dolfinx/io/ADIOS2_utils.cpp index 9458870c965..9a6d22ca4db 100644 --- a/cpp/dolfinx/io/ADIOS2_utils.cpp +++ b/cpp/dolfinx/io/ADIOS2_utils.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2021-2023 Jørgen S. Dokken and Garth N. Wells +// Copyright (C) 2024 Abdullah Mujahid, Jørgen S. Dokken and Garth N. Wells // // This file is part of DOLFINX (https://www.fenicsproject.org) // diff --git a/cpp/dolfinx/io/ADIOS2_utils.h b/cpp/dolfinx/io/ADIOS2_utils.h index 8b5ac5f9c9c..0364558371d 100644 --- a/cpp/dolfinx/io/ADIOS2_utils.h +++ b/cpp/dolfinx/io/ADIOS2_utils.h @@ -1,4 +1,4 @@ -// Copyright (C) 2021-2023 Jørgen S. Dokken and Garth N. Wells +// Copyright (C) 2024 Abdullah Mujahid, Jørgen S. Dokken and Garth N. Wells // // This file is part of DOLFINX (https://www.fenicsproject.org) // diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index bd2fcafc6fd..ddc96736996 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -1,4 +1,4 @@ -// Copyright (C) year authors +// Copyright (C) 2024 Abdullah Mujahid // // This file is part of DOLFINX (https://www.fenicsproject.org) // diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 1e363fe63f5..7bd8a864aa6 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -1,4 +1,4 @@ -// Copyright (C) year authors +// Copyright (C) 2024 Abdullah Mujahid // // This file is part of DOLFINX (https://www.fenicsproject.org) // From d3a965be2dc70e539e09e97a85064ae91c8789e6 Mon Sep 17 00:00:00 2001 From: ampdes Date: Sun, 21 Jul 2024 07:04:51 +0200 Subject: [PATCH 18/60] Clean up --- cpp/dolfinx/io/checkpointing.cpp | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index ddc96736996..e5586bc63e2 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -73,38 +73,12 @@ void _write(MPI_Comm comm, std::string filename, std::string tag, std::iota(connectivity_nodes_global.begin(), connectivity_nodes_global.end(), 0); - std::cout << indices.size() << "\n"; - std::cout << indices_offsets[num_cells_local] << "\n"; - std::cout << indices_offsets[num_cells_local - 1] << "\n"; - - // for (std::size_t i = 0; i < connectivity_nodes_global.size(); ++i) - // { - // std::cout << i << " "; - // std::cout << indices[i] << " "; - // std::cout << mesh_input_global_indices[indices[i]] << "\n"; - // connectivity_nodes_global[i] = - // mesh_input_global_indices[indices[i]]; - // } - - std::cout << indices_span.subspan(0, indices_offsets[num_cells_local]).size() - << std::endl; - std::cout << indices_offsets[num_cells_local] << std::endl; imap->local_to_global( indices_span.subspan(0, indices_offsets[num_cells_local]), connectivity_nodes_global); - for (std::size_t i = 0; i < connectivity_nodes_global.size(); ++i) - { - std::cout << i << " "; - std::cout << indices[i] << " "; - std::cout << connectivity_nodes_global[i] << " "; - std::cout << mesh_input_global_indices[indices[i]] << "\n"; - } - for (std::size_t i = 0; i < indices_offsets.size(); ++i) - { indices_offsets[i] += cell_offset * num_dofs_per_cell; - } const std::span indices_offsets_span(indices_offsets.begin(), indices_offsets.end()); From 729846b21a0dc1919563977ff99cd5b52e2135e9 Mon Sep 17 00:00:00 2001 From: ampdes Date: Fri, 26 Jul 2024 12:41:15 +0200 Subject: [PATCH 19/60] Rename variables --- cpp/demo/checkpointing/main.cpp | 2 +- cpp/dolfinx/io/checkpointing.cpp | 86 +++++++++++++++----------------- 2 files changed, 42 insertions(+), 46 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 7caeb849291..de8c5b8de54 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -14,7 +14,7 @@ #include using namespace dolfinx; -using namespace dolfinx::io::ADIOS2Engine; +using namespace dolfinx::io; int main(int argc, char* argv[]) { diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index e5586bc63e2..59013a94012 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -55,78 +55,74 @@ void _write(MPI_Comm comm, std::string filename, std::string tag, std::uint32_t num_dofs_per_cell = geom_layout.num_entity_closure_dofs(mesh_dim); - const std::vector mesh_input_global_indices + const std::vector input_global_indices = geometry.input_global_indices(); - const std::span mesh_input_global_indices_span( - mesh_input_global_indices.begin(), mesh_input_global_indices.end()); + const std::span input_global_indices_span( + input_global_indices.begin(), num_nodes_local); const std::span mesh_x = geometry.x(); auto connectivity = topology->connectivity(mesh_dim, 0); - auto indices = connectivity->array(); - const std::span indices_span(indices.begin(), indices.end()); + auto topology_array = connectivity->array(); + auto topology_offsets = connectivity->offsets(); - auto indices_offsets = connectivity->offsets(); + const std::span topology_array_span( + topology_array.begin(), topology_offsets[num_cells_local]); + std::vector topology_array_global( + topology_offsets[num_cells_local]); - std::vector connectivity_nodes_global( - indices_offsets[num_cells_local]); + std::iota(topology_array_global.begin(), topology_array_global.end(), 0); - std::iota(connectivity_nodes_global.begin(), connectivity_nodes_global.end(), - 0); + imap->local_to_global(topology_array_span, topology_array_global); - imap->local_to_global( - indices_span.subspan(0, indices_offsets[num_cells_local]), - connectivity_nodes_global); + for (std::size_t i = 0; i < num_cells_local + 1; ++i) + topology_offsets[i] += cell_offset * num_dofs_per_cell; - for (std::size_t i = 0; i < indices_offsets.size(); ++i) - indices_offsets[i] += cell_offset * num_dofs_per_cell; - - const std::span indices_offsets_span(indices_offsets.begin(), - indices_offsets.end()); + const std::span topology_offsets_span(topology_offsets.begin(), + num_cells_local + 1); io.DefineAttribute("name", mesh->name); io.DefineAttribute("dim", geometry.dim()); - io.DefineAttribute("CellType", + io.DefineAttribute("cell_type", dolfinx::mesh::to_string(cmap.cell_shape())); - io.DefineAttribute("Degree", cmap.degree()); - io.DefineAttribute("LagrangeVariant", + io.DefineAttribute("degree", cmap.degree()); + io.DefineAttribute("lagrange_variant", lagrange_variants[cmap.variant()]); - adios2::Variable n_nodes - = io.DefineVariable("n_nodes"); - adios2::Variable n_cells - = io.DefineVariable("n_cells"); - adios2::Variable n_dofs_per_cell - = io.DefineVariable("n_dofs_per_cell"); + adios2::Variable var_num_nodes + = io.DefineVariable("num_nodes"); + adios2::Variable var_num_cells + = io.DefineVariable("num_cells"); + adios2::Variable var_num_dofs_per_cell + = io.DefineVariable("num_dofs_per_cell"); - adios2::Variable input_global_indices + adios2::Variable var_input_global_indices = io.DefineVariable( "input_global_indices", {num_nodes_global}, {offset}, {num_nodes_local}, adios2::ConstantDims); - adios2::Variable x - = io.DefineVariable("Points", {num_nodes_global, 3}, {offset, 0}, + adios2::Variable var_x + = io.DefineVariable("x", {num_nodes_global, 3}, {offset, 0}, {num_nodes_local, 3}, adios2::ConstantDims); - adios2::Variable cell_indices = io.DefineVariable( - "cell_indices", {num_cells_global * num_dofs_per_cell}, - {cell_offset * num_dofs_per_cell}, {num_cells_local * num_dofs_per_cell}, - adios2::ConstantDims); + adios2::Variable var_topology_array + = io.DefineVariable( + "topology_array", {num_cells_global * num_dofs_per_cell}, + {cell_offset * num_dofs_per_cell}, + {num_cells_local * num_dofs_per_cell}, adios2::ConstantDims); - adios2::Variable cell_indices_offsets + adios2::Variable var_topology_offsets = io.DefineVariable( - "cell_indices_offsets", {num_cells_global + 1}, {cell_offset}, + "topology_offsets", {num_cells_global + 1}, {cell_offset}, {num_cells_local + 1}, adios2::ConstantDims); writer.BeginStep(); - writer.Put(n_nodes, num_nodes_global); - writer.Put(n_cells, num_cells_global); - writer.Put(n_dofs_per_cell, num_dofs_per_cell); - writer.Put(input_global_indices, - mesh_input_global_indices_span.subspan(0, num_nodes_local).data()); - writer.Put(x, mesh_x.subspan(0, num_nodes_local * 3).data()); - writer.Put(cell_indices, connectivity_nodes_global.data()); - writer.Put(cell_indices_offsets, - indices_offsets_span.subspan(0, num_cells_local + 1).data()); + writer.Put(var_num_nodes, num_nodes_global); + writer.Put(var_num_cells, num_cells_global); + writer.Put(var_num_dofs_per_cell, num_dofs_per_cell); + writer.Put(var_input_global_indices, input_global_indices_span.data()); + writer.Put(var_x, mesh_x.subspan(0, num_nodes_local * 3).data()); + writer.Put(var_topology_array, topology_array_global.data()); + writer.Put(var_topology_offsets, topology_offsets_span.data()); writer.EndStep(); writer.Close(); } From b2f4bd087fcf75552f7e1638ac1f83ed063616ba Mon Sep 17 00:00:00 2001 From: ampdes Date: Fri, 26 Jul 2024 13:42:51 +0200 Subject: [PATCH 20/60] Pass ADIOS2Engine writer to write function --- cpp/demo/checkpointing/main.cpp | 14 +------ cpp/dolfinx/io/ADIOS2_utils.cpp | 14 ------- cpp/dolfinx/io/ADIOS2_utils.h | 14 +++---- cpp/dolfinx/io/checkpointing.cpp | 71 +++++++++++++++----------------- cpp/dolfinx/io/checkpointing.h | 38 ++++++++--------- 5 files changed, 60 insertions(+), 91 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index de8c5b8de54..95d0946394a 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -21,12 +21,6 @@ int main(int argc, char* argv[]) dolfinx::init_logging(argc, argv); MPI_Init(&argc, &argv); - // { - // int i=0; - // while (i == 0) - // sleep(5); - // } - // Create mesh and function space auto part = mesh::create_cell_partitioner(mesh::GhostMode::shared_facet); auto mesh = std::make_shared>(mesh::create_rectangle( @@ -34,13 +28,9 @@ int main(int argc, char* argv[]) mesh::CellType::quadrilateral, part)); auto writer = ADIOS2Engine(mesh->comm(), "mesh.bp", "mesh-write", "BP5", - adios2::Mode::Write); - - // auto io = writer.io(); - // auto engine = writer.engine(); + adios2::Mode::Write); - // io::checkpointing::write(io, engine, mesh); - io::checkpointing::write(mesh->comm(), "mesh.bp", "mesh-write", mesh); + io::checkpointing::write(writer, mesh); MPI_Finalize(); return 0; diff --git a/cpp/dolfinx/io/ADIOS2_utils.cpp b/cpp/dolfinx/io/ADIOS2_utils.cpp index 9a6d22ca4db..120ed2dabbd 100644 --- a/cpp/dolfinx/io/ADIOS2_utils.cpp +++ b/cpp/dolfinx/io/ADIOS2_utils.cpp @@ -32,19 +32,5 @@ void ADIOS2Engine::close() if (*_engine) _engine->Close(); } -// //----------------------------------------------------------------------------- -// std::unique_ptr ADIOS2Engine::io() -// { -// assert(_io); -// if (*_io) -// return _io; -// } -// //----------------------------------------------------------------------------- -// std::unique_ptr ADIOS2Engine::engine() -// { -// assert(_engine); -// if (*_engine) -// return _engine; -// } #endif diff --git a/cpp/dolfinx/io/ADIOS2_utils.h b/cpp/dolfinx/io/ADIOS2_utils.h index 0364558371d..e104e50f578 100644 --- a/cpp/dolfinx/io/ADIOS2_utils.h +++ b/cpp/dolfinx/io/ADIOS2_utils.h @@ -52,16 +52,16 @@ class ADIOS2Engine /// @brief Close the file void close(); - // /// @brief Get the IO object - // std::unique_ptr io(); + /// @brief Get the IO object + std::shared_ptr io() { return _io; } - // /// @brief Close the Engine object - // std::unique_ptr engine(); + /// @brief Close the Engine object + std::shared_ptr engine() { return _engine; } protected: - std::unique_ptr _adios; - std::unique_ptr _io; - std::unique_ptr _engine; + std::shared_ptr _adios; + std::shared_ptr _io; + std::shared_ptr _engine; }; } // namespace dolfinx::io diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 59013a94012..0b9bdf7e47c 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -7,6 +7,7 @@ #ifdef HAS_ADIOS2 #include "checkpointing.h" +#include "ADIOS2_utils.h" #include #include #include @@ -26,13 +27,12 @@ std::map lagrange_variants{ }; template -void _write(MPI_Comm comm, std::string filename, std::string tag, - // adios2::IO io, adios2::Engine engine, +void _write(ADIOS2Engine& adios2engine, std::shared_ptr> mesh) { - adios2::ADIOS adios(comm); - adios2::IO io = adios.DeclareIO(tag); - adios2::Engine writer = io.Open(filename, adios2::Mode::Write); + + auto io = adios2engine.io(); + auto writer = adios2engine.engine(); const dolfinx::mesh::Geometry& geometry = mesh->geometry(); auto topology = mesh->topology(); @@ -80,51 +80,50 @@ void _write(MPI_Comm comm, std::string filename, std::string tag, const std::span topology_offsets_span(topology_offsets.begin(), num_cells_local + 1); - io.DefineAttribute("name", mesh->name); - io.DefineAttribute("dim", geometry.dim()); - io.DefineAttribute("cell_type", - dolfinx::mesh::to_string(cmap.cell_shape())); - io.DefineAttribute("degree", cmap.degree()); - io.DefineAttribute("lagrange_variant", - lagrange_variants[cmap.variant()]); + io->DefineAttribute("name", mesh->name); + io->DefineAttribute("dim", geometry.dim()); + io->DefineAttribute("cell_type", + dolfinx::mesh::to_string(cmap.cell_shape())); + io->DefineAttribute("degree", cmap.degree()); + io->DefineAttribute("lagrange_variant", + lagrange_variants[cmap.variant()]); adios2::Variable var_num_nodes - = io.DefineVariable("num_nodes"); + = io->DefineVariable("num_nodes"); adios2::Variable var_num_cells - = io.DefineVariable("num_cells"); + = io->DefineVariable("num_cells"); adios2::Variable var_num_dofs_per_cell - = io.DefineVariable("num_dofs_per_cell"); + = io->DefineVariable("num_dofs_per_cell"); adios2::Variable var_input_global_indices - = io.DefineVariable( + = io->DefineVariable( "input_global_indices", {num_nodes_global}, {offset}, {num_nodes_local}, adios2::ConstantDims); adios2::Variable var_x - = io.DefineVariable("x", {num_nodes_global, 3}, {offset, 0}, - {num_nodes_local, 3}, adios2::ConstantDims); + = io->DefineVariable("x", {num_nodes_global, 3}, {offset, 0}, + {num_nodes_local, 3}, adios2::ConstantDims); adios2::Variable var_topology_array - = io.DefineVariable( + = io->DefineVariable( "topology_array", {num_cells_global * num_dofs_per_cell}, {cell_offset * num_dofs_per_cell}, {num_cells_local * num_dofs_per_cell}, adios2::ConstantDims); adios2::Variable var_topology_offsets - = io.DefineVariable( + = io->DefineVariable( "topology_offsets", {num_cells_global + 1}, {cell_offset}, {num_cells_local + 1}, adios2::ConstantDims); - writer.BeginStep(); - writer.Put(var_num_nodes, num_nodes_global); - writer.Put(var_num_cells, num_cells_global); - writer.Put(var_num_dofs_per_cell, num_dofs_per_cell); - writer.Put(var_input_global_indices, input_global_indices_span.data()); - writer.Put(var_x, mesh_x.subspan(0, num_nodes_local * 3).data()); - writer.Put(var_topology_array, topology_array_global.data()); - writer.Put(var_topology_offsets, topology_offsets_span.data()); - writer.EndStep(); - writer.Close(); + writer->BeginStep(); + writer->Put(var_num_nodes, num_nodes_global); + writer->Put(var_num_cells, num_cells_global); + writer->Put(var_num_dofs_per_cell, num_dofs_per_cell); + writer->Put(var_input_global_indices, input_global_indices_span.data()); + writer->Put(var_x, mesh_x.subspan(0, num_nodes_local * 3).data()); + writer->Put(var_topology_array, topology_array_global.data()); + writer->Put(var_topology_offsets, topology_offsets_span.data()); + writer->EndStep(); } } // namespace @@ -133,24 +132,20 @@ using namespace dolfinx::io::checkpointing; //----------------------------------------------------------------------------- void dolfinx::io::checkpointing::write( - MPI_Comm comm, std::string filename, std::string tag, - // adios2::IO io, adios2::Engine engine, + ADIOS2Engine& adios2engine, std::shared_ptr> mesh) { - // _write(io, engine, mesh); - _write(comm, filename, tag, mesh); + _write(adios2engine, mesh); } //----------------------------------------------------------------------------- void dolfinx::io::checkpointing::write( - MPI_Comm comm, std::string filename, std::string tag, - // adios2::IO io, adios2::Engine engine, + ADIOS2Engine& adios2engine, std::shared_ptr> mesh) { - // _write(io, engine, mesh); - _write(comm, filename, tag, mesh); + _write(adios2engine, mesh); } #endif diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 7bd8a864aa6..3679ccfe81d 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -20,28 +20,26 @@ namespace dolfinx::io::checkpointing { - /// @brief Write mesh to a file. - /// - /// @param[in] comm The MPI communicator to open the file on - /// @param[in] filename Name of output file - /// @param[in] tag ADIOS2 tag for IO - /// @param[in] mesh Mesh of type float to write to the file - /// @note This is experimental version. Expected would be to - /// pass ADIOS2Engine object. -void write(MPI_Comm comm, std::string filename, std::string tag, - // adios2::IO io, adios2::Engine engine, +/// @brief Write mesh to a file. +/// +/// @param[in] comm The MPI communicator to open the file on +/// @param[in] filename Name of output file +/// @param[in] tag ADIOS2 tag for IO +/// @param[in] mesh Mesh of type float to write to the file +/// @note This is experimental version. Expected would be to +/// pass ADIOS2Engine object. +void write(ADIOS2Engine& adios2engine, std::shared_ptr> mesh); - /// @brief Write mesh to a file. - /// - /// @param[in] comm The MPI communicator to open the file on - /// @param[in] filename Name of output file - /// @param[in] tag ADIOS2 tag for IO - /// @param[in] mesh Mesh of type double to write to the file - /// @note This is experimental version. Expected would be to - /// pass ADIOS2Engine object. -void write(MPI_Comm comm, std::string filename, std::string tag, - // adios2::IO io, adios2::Engine engine, +/// @brief Write mesh to a file. +/// +/// @param[in] comm The MPI communicator to open the file on +/// @param[in] filename Name of output file +/// @param[in] tag ADIOS2 tag for IO +/// @param[in] mesh Mesh of type double to write to the file +/// @note This is experimental version. Expected would be to +/// pass ADIOS2Engine object. +void write(ADIOS2Engine& adios2engine, std::shared_ptr> mesh); } // namespace dolfinx::io::checkpointing From d7e8e3a3a940f1e1e7bfbff7445f9b5bc4a20a6b Mon Sep 17 00:00:00 2001 From: ampdes Date: Fri, 26 Jul 2024 14:01:15 +0200 Subject: [PATCH 21/60] Update docs --- cpp/dolfinx/io/checkpointing.h | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 3679ccfe81d..256a6bd80b4 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -22,23 +22,15 @@ namespace dolfinx::io::checkpointing /// @brief Write mesh to a file. /// -/// @param[in] comm The MPI communicator to open the file on -/// @param[in] filename Name of output file -/// @param[in] tag ADIOS2 tag for IO +/// @param[in] adios2engine ADIOS2Engine /// @param[in] mesh Mesh of type float to write to the file -/// @note This is experimental version. Expected would be to -/// pass ADIOS2Engine object. void write(ADIOS2Engine& adios2engine, std::shared_ptr> mesh); /// @brief Write mesh to a file. /// -/// @param[in] comm The MPI communicator to open the file on -/// @param[in] filename Name of output file -/// @param[in] tag ADIOS2 tag for IO +/// @param[in] adios2engine ADIOS2Engine /// @param[in] mesh Mesh of type double to write to the file -/// @note This is experimental version. Expected would be to -/// pass ADIOS2Engine object. void write(ADIOS2Engine& adios2engine, std::shared_ptr> mesh); From ca94e17c9e2db28277a916bb65b68604434eeeff Mon Sep 17 00:00:00 2001 From: ampdes Date: Sat, 27 Jul 2024 16:31:13 +0200 Subject: [PATCH 22/60] Update from review --- cpp/dolfinx/io/ADIOS2_utils.h | 2 +- cpp/dolfinx/io/checkpointing.cpp | 89 ++++++++++++++------------------ cpp/dolfinx/io/checkpointing.h | 12 ++--- 3 files changed, 43 insertions(+), 60 deletions(-) diff --git a/cpp/dolfinx/io/ADIOS2_utils.h b/cpp/dolfinx/io/ADIOS2_utils.h index e104e50f578..e926bfd684c 100644 --- a/cpp/dolfinx/io/ADIOS2_utils.h +++ b/cpp/dolfinx/io/ADIOS2_utils.h @@ -55,7 +55,7 @@ class ADIOS2Engine /// @brief Get the IO object std::shared_ptr io() { return _io; } - /// @brief Close the Engine object + /// @brief Get the Engine object std::shared_ptr engine() { return _engine; } protected: diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 0b9bdf7e47c..590de3e84eb 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -10,6 +10,7 @@ #include "ADIOS2_utils.h" #include #include +#include #include #include @@ -25,35 +26,38 @@ std::map lagrange_variants{ {basix::element::lagrange_variant::equispaced, "equispaced"}, {basix::element::lagrange_variant::gll_warped, "gll_warped"}, }; +} // namespace +namespace dolfinx::io::checkpointing +{ template -void _write(ADIOS2Engine& adios2engine, - std::shared_ptr> mesh) +void write(ADIOS2Engine& adios2engine, + std::shared_ptr> mesh) { auto io = adios2engine.io(); auto writer = adios2engine.engine(); - const dolfinx::mesh::Geometry& geometry = mesh->geometry(); - auto topology = mesh->topology(); + const mesh::Geometry& geometry = mesh->geometry(); + std::shared_ptr topology = mesh->topology(); - std::int16_t mesh_dim = geometry.dim(); + std::int32_t dim = geometry.dim(); - auto imap = mesh->geometry().index_map(); - std::uint64_t num_nodes_global = imap->size_global(); - std::uint32_t num_nodes_local = imap->size_local(); - std::uint64_t offset = imap->local_range()[0]; + std::shared_ptr geom_imap + = mesh->geometry().index_map(); + std::uint64_t num_nodes_global = geom_imap->size_global(); + std::uint32_t num_nodes_local = geom_imap->size_local(); + std::uint64_t offset = geom_imap->local_range()[0]; - const std::shared_ptr topo_imap - = topology->index_map(mesh_dim); + const std::shared_ptr topo_imap + = topology->index_map(dim); std::uint64_t num_cells_global = topo_imap->size_global(); std::uint32_t num_cells_local = topo_imap->size_local(); std::uint64_t cell_offset = topo_imap->local_range()[0]; - auto cmap = mesh->geometry().cmap(); - auto geom_layout = cmap.create_dof_layout(); - std::uint32_t num_dofs_per_cell - = geom_layout.num_entity_closure_dofs(mesh_dim); + const fem::CoordinateElement& cmap = mesh->geometry().cmap(); + fem::ElementDofLayout geom_layout = cmap.create_dof_layout(); + std::uint32_t num_dofs_per_cell = geom_layout.num_entity_closure_dofs(dim); const std::vector input_global_indices = geometry.input_global_indices(); @@ -61,27 +65,26 @@ void _write(ADIOS2Engine& adios2engine, input_global_indices.begin(), num_nodes_local); const std::span mesh_x = geometry.x(); - auto connectivity = topology->connectivity(mesh_dim, 0); - auto topology_array = connectivity->array(); - auto topology_offsets = connectivity->offsets(); + std::shared_ptr> connectivity + = topology->connectivity(dim, 0); + const std::vector& array = connectivity->array(); + const std::vector& offsets = connectivity->offsets(); - const std::span topology_array_span( - topology_array.begin(), topology_offsets[num_cells_local]); - std::vector topology_array_global( - topology_offsets[num_cells_local]); + const std::span array_span(array.begin(), + offsets[num_cells_local]); + std::vector array_global(offsets[num_cells_local]); - std::iota(topology_array_global.begin(), topology_array_global.end(), 0); + std::vector offsets_global(num_cells_local + 1); - imap->local_to_global(topology_array_span, topology_array_global); + std::iota(array_global.begin(), array_global.end(), 0); - for (std::size_t i = 0; i < num_cells_local + 1; ++i) - topology_offsets[i] += cell_offset * num_dofs_per_cell; + geom_imap->local_to_global(array_span, array_global); - const std::span topology_offsets_span(topology_offsets.begin(), - num_cells_local + 1); + for (std::size_t i = 0; i < num_cells_local + 1; ++i) + offsets_global[i] = offsets[i] + cell_offset * num_dofs_per_cell; io->DefineAttribute("name", mesh->name); - io->DefineAttribute("dim", geometry.dim()); + io->DefineAttribute("dim", geometry.dim()); io->DefineAttribute("cell_type", dolfinx::mesh::to_string(cmap.cell_shape())); io->DefineAttribute("degree", cmap.degree()); @@ -121,31 +124,17 @@ void _write(ADIOS2Engine& adios2engine, writer->Put(var_num_dofs_per_cell, num_dofs_per_cell); writer->Put(var_input_global_indices, input_global_indices_span.data()); writer->Put(var_x, mesh_x.subspan(0, num_nodes_local * 3).data()); - writer->Put(var_topology_array, topology_array_global.data()); - writer->Put(var_topology_offsets, topology_offsets_span.data()); + writer->Put(var_topology_array, array_global.data()); + writer->Put(var_topology_offsets, offsets_global.data()); writer->EndStep(); } -} // namespace - -using namespace dolfinx::io::checkpointing; +template void write(ADIOS2Engine& adios2engine, + std::shared_ptr> mesh); -//----------------------------------------------------------------------------- -void dolfinx::io::checkpointing::write( - ADIOS2Engine& adios2engine, - std::shared_ptr> mesh) -{ - - _write(adios2engine, mesh); -} +template void write(ADIOS2Engine& adios2engine, + std::shared_ptr> mesh); -//----------------------------------------------------------------------------- -void dolfinx::io::checkpointing::write( - ADIOS2Engine& adios2engine, - std::shared_ptr> mesh) -{ - - _write(adios2engine, mesh); -} +} // namespace dolfinx::io::checkpointing #endif diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 256a6bd80b4..544ddc09762 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -23,16 +23,10 @@ namespace dolfinx::io::checkpointing /// @brief Write mesh to a file. /// /// @param[in] adios2engine ADIOS2Engine -/// @param[in] mesh Mesh of type float to write to the file +/// @param[in] mesh Mesh of type float or double to write to the file +template void write(ADIOS2Engine& adios2engine, - std::shared_ptr> mesh); - -/// @brief Write mesh to a file. -/// -/// @param[in] adios2engine ADIOS2Engine -/// @param[in] mesh Mesh of type double to write to the file -void write(ADIOS2Engine& adios2engine, - std::shared_ptr> mesh); + std::shared_ptr> mesh); } // namespace dolfinx::io::checkpointing From 55b92889f43aa869b359cd08150707d1fa727ae8 Mon Sep 17 00:00:00 2001 From: ampdes Date: Sat, 27 Jul 2024 21:20:00 +0200 Subject: [PATCH 23/60] Pass IO and Engine shared ptrs --- cpp/demo/checkpointing/main.cpp | 16 +++++++++---- cpp/dolfinx/io/checkpointing.cpp | 39 ++++++++++++++++---------------- cpp/dolfinx/io/checkpointing.h | 5 ++-- 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 95d0946394a..99326a97398 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -9,7 +9,6 @@ #include #include -#include #include #include @@ -27,10 +26,19 @@ int main(int argc, char* argv[]) MPI_COMM_WORLD, {{{0.0, 0.0}, {1.0, 1.0}}}, {4, 4}, mesh::CellType::quadrilateral, part)); - auto writer = ADIOS2Engine(mesh->comm(), "mesh.bp", "mesh-write", "BP5", - adios2::Mode::Write); + // Set up ADIOS2 IO and Engine + std::shared_ptr adios + = std::make_shared(mesh->comm()); + // adios2::ADIOS adios(mesh->comm()); + std::shared_ptr io + = std::make_shared(adios->DeclareIO("mesh-write")); + io->SetEngine("BP5"); + std::shared_ptr engine = std::make_shared( + io->Open("mesh.bp", adios2::Mode::Write)); - io::checkpointing::write(writer, mesh); + io::checkpointing::write_mesh(io, engine, mesh); + + engine->Close(); MPI_Finalize(); return 0; diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 590de3e84eb..935e014839f 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -7,7 +7,6 @@ #ifdef HAS_ADIOS2 #include "checkpointing.h" -#include "ADIOS2_utils.h" #include #include #include @@ -31,13 +30,11 @@ std::map lagrange_variants{ namespace dolfinx::io::checkpointing { template -void write(ADIOS2Engine& adios2engine, - std::shared_ptr> mesh) +void write_mesh(std::shared_ptr io, + std::shared_ptr engine, + std::shared_ptr> mesh) { - auto io = adios2engine.io(); - auto writer = adios2engine.engine(); - const mesh::Geometry& geometry = mesh->geometry(); std::shared_ptr topology = mesh->topology(); @@ -118,22 +115,26 @@ void write(ADIOS2Engine& adios2engine, "topology_offsets", {num_cells_global + 1}, {cell_offset}, {num_cells_local + 1}, adios2::ConstantDims); - writer->BeginStep(); - writer->Put(var_num_nodes, num_nodes_global); - writer->Put(var_num_cells, num_cells_global); - writer->Put(var_num_dofs_per_cell, num_dofs_per_cell); - writer->Put(var_input_global_indices, input_global_indices_span.data()); - writer->Put(var_x, mesh_x.subspan(0, num_nodes_local * 3).data()); - writer->Put(var_topology_array, array_global.data()); - writer->Put(var_topology_offsets, offsets_global.data()); - writer->EndStep(); + engine->BeginStep(); + engine->Put(var_num_nodes, num_nodes_global); + engine->Put(var_num_cells, num_cells_global); + engine->Put(var_num_dofs_per_cell, num_dofs_per_cell); + engine->Put(var_input_global_indices, input_global_indices_span.data()); + engine->Put(var_x, mesh_x.subspan(0, num_nodes_local * 3).data()); + engine->Put(var_topology_array, array_global.data()); + engine->Put(var_topology_offsets, offsets_global.data()); + engine->EndStep(); } -template void write(ADIOS2Engine& adios2engine, - std::shared_ptr> mesh); +template void +write_mesh(std::shared_ptr io, + std::shared_ptr engine, + std::shared_ptr> mesh); -template void write(ADIOS2Engine& adios2engine, - std::shared_ptr> mesh); +template void +write_mesh(std::shared_ptr io, + std::shared_ptr engine, + std::shared_ptr> mesh); } // namespace dolfinx::io::checkpointing diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 544ddc09762..b59a4eff1eb 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -25,8 +25,9 @@ namespace dolfinx::io::checkpointing /// @param[in] adios2engine ADIOS2Engine /// @param[in] mesh Mesh of type float or double to write to the file template -void write(ADIOS2Engine& adios2engine, - std::shared_ptr> mesh); +void write_mesh(std::shared_ptr io, + std::shared_ptr engine, + std::shared_ptr> mesh); } // namespace dolfinx::io::checkpointing From 241fbef7bf62874b03d36321840d462f0c5b8d74 Mon Sep 17 00:00:00 2001 From: ampdes Date: Sat, 27 Jul 2024 21:35:12 +0200 Subject: [PATCH 24/60] ADIOS IO and Engine without ptrs --- cpp/demo/checkpointing/main.cpp | 14 +++----- cpp/dolfinx/io/checkpointing.cpp | 57 +++++++++++++++----------------- cpp/dolfinx/io/checkpointing.h | 3 +- 3 files changed, 33 insertions(+), 41 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 99326a97398..8a3f394265d 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -27,18 +27,14 @@ int main(int argc, char* argv[]) mesh::CellType::quadrilateral, part)); // Set up ADIOS2 IO and Engine - std::shared_ptr adios - = std::make_shared(mesh->comm()); - // adios2::ADIOS adios(mesh->comm()); - std::shared_ptr io - = std::make_shared(adios->DeclareIO("mesh-write")); - io->SetEngine("BP5"); - std::shared_ptr engine = std::make_shared( - io->Open("mesh.bp", adios2::Mode::Write)); + adios2::ADIOS adios(mesh->comm()); + adios2::IO io = adios.DeclareIO("mesh-write"); + io.SetEngine("BP5"); + adios2::Engine engine = io.Open("mesh.bp", adios2::Mode::Write); io::checkpointing::write_mesh(io, engine, mesh); - engine->Close(); + engine.Close(); MPI_Finalize(); return 0; diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 935e014839f..df6e87bf069 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -30,8 +30,7 @@ std::map lagrange_variants{ namespace dolfinx::io::checkpointing { template -void write_mesh(std::shared_ptr io, - std::shared_ptr engine, +void write_mesh(adios2::IO io, adios2::Engine engine, std::shared_ptr> mesh) { @@ -80,60 +79,58 @@ void write_mesh(std::shared_ptr io, for (std::size_t i = 0; i < num_cells_local + 1; ++i) offsets_global[i] = offsets[i] + cell_offset * num_dofs_per_cell; - io->DefineAttribute("name", mesh->name); - io->DefineAttribute("dim", geometry.dim()); - io->DefineAttribute("cell_type", - dolfinx::mesh::to_string(cmap.cell_shape())); - io->DefineAttribute("degree", cmap.degree()); - io->DefineAttribute("lagrange_variant", - lagrange_variants[cmap.variant()]); + io.DefineAttribute("name", mesh->name); + io.DefineAttribute("dim", geometry.dim()); + io.DefineAttribute("cell_type", + dolfinx::mesh::to_string(cmap.cell_shape())); + io.DefineAttribute("degree", cmap.degree()); + io.DefineAttribute("lagrange_variant", + lagrange_variants[cmap.variant()]); adios2::Variable var_num_nodes - = io->DefineVariable("num_nodes"); + = io.DefineVariable("num_nodes"); adios2::Variable var_num_cells - = io->DefineVariable("num_cells"); + = io.DefineVariable("num_cells"); adios2::Variable var_num_dofs_per_cell - = io->DefineVariable("num_dofs_per_cell"); + = io.DefineVariable("num_dofs_per_cell"); adios2::Variable var_input_global_indices - = io->DefineVariable( + = io.DefineVariable( "input_global_indices", {num_nodes_global}, {offset}, {num_nodes_local}, adios2::ConstantDims); adios2::Variable var_x - = io->DefineVariable("x", {num_nodes_global, 3}, {offset, 0}, - {num_nodes_local, 3}, adios2::ConstantDims); + = io.DefineVariable("x", {num_nodes_global, 3}, {offset, 0}, + {num_nodes_local, 3}, adios2::ConstantDims); adios2::Variable var_topology_array - = io->DefineVariable( + = io.DefineVariable( "topology_array", {num_cells_global * num_dofs_per_cell}, {cell_offset * num_dofs_per_cell}, {num_cells_local * num_dofs_per_cell}, adios2::ConstantDims); adios2::Variable var_topology_offsets - = io->DefineVariable( + = io.DefineVariable( "topology_offsets", {num_cells_global + 1}, {cell_offset}, {num_cells_local + 1}, adios2::ConstantDims); - engine->BeginStep(); - engine->Put(var_num_nodes, num_nodes_global); - engine->Put(var_num_cells, num_cells_global); - engine->Put(var_num_dofs_per_cell, num_dofs_per_cell); - engine->Put(var_input_global_indices, input_global_indices_span.data()); - engine->Put(var_x, mesh_x.subspan(0, num_nodes_local * 3).data()); - engine->Put(var_topology_array, array_global.data()); - engine->Put(var_topology_offsets, offsets_global.data()); - engine->EndStep(); + engine.BeginStep(); + engine.Put(var_num_nodes, num_nodes_global); + engine.Put(var_num_cells, num_cells_global); + engine.Put(var_num_dofs_per_cell, num_dofs_per_cell); + engine.Put(var_input_global_indices, input_global_indices_span.data()); + engine.Put(var_x, mesh_x.subspan(0, num_nodes_local * 3).data()); + engine.Put(var_topology_array, array_global.data()); + engine.Put(var_topology_offsets, offsets_global.data()); + engine.EndStep(); } template void -write_mesh(std::shared_ptr io, - std::shared_ptr engine, +write_mesh(adios2::IO io, adios2::Engine engine, std::shared_ptr> mesh); template void -write_mesh(std::shared_ptr io, - std::shared_ptr engine, +write_mesh(adios2::IO io, adios2::Engine engine, std::shared_ptr> mesh); } // namespace dolfinx::io::checkpointing diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index b59a4eff1eb..45a713b784a 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -25,8 +25,7 @@ namespace dolfinx::io::checkpointing /// @param[in] adios2engine ADIOS2Engine /// @param[in] mesh Mesh of type float or double to write to the file template -void write_mesh(std::shared_ptr io, - std::shared_ptr engine, +void write_mesh(adios2::IO io, adios2::Engine engine, std::shared_ptr> mesh); } // namespace dolfinx::io::checkpointing From 43f648135bd4ac07c22e16f9b94be0a191ded5e6 Mon Sep 17 00:00:00 2001 From: ampdes Date: Sat, 27 Jul 2024 21:40:15 +0200 Subject: [PATCH 25/60] Pass mesh without shared ptr --- cpp/demo/checkpointing/main.cpp | 2 +- cpp/dolfinx/io/checkpointing.cpp | 22 ++++++++++------------ cpp/dolfinx/io/checkpointing.h | 5 +++-- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 8a3f394265d..41f30526ba7 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -32,7 +32,7 @@ int main(int argc, char* argv[]) io.SetEngine("BP5"); adios2::Engine engine = io.Open("mesh.bp", adios2::Mode::Write); - io::checkpointing::write_mesh(io, engine, mesh); + io::checkpointing::write_mesh(io, engine, *mesh); engine.Close(); diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index df6e87bf069..4a6a6ffa7aa 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -31,16 +31,16 @@ namespace dolfinx::io::checkpointing { template void write_mesh(adios2::IO io, adios2::Engine engine, - std::shared_ptr> mesh) + dolfinx::mesh::Mesh mesh) { - const mesh::Geometry& geometry = mesh->geometry(); - std::shared_ptr topology = mesh->topology(); + const mesh::Geometry& geometry = mesh.geometry(); + std::shared_ptr topology = mesh.topology(); std::int32_t dim = geometry.dim(); std::shared_ptr geom_imap - = mesh->geometry().index_map(); + = mesh.geometry().index_map(); std::uint64_t num_nodes_global = geom_imap->size_global(); std::uint32_t num_nodes_local = geom_imap->size_local(); std::uint64_t offset = geom_imap->local_range()[0]; @@ -51,7 +51,7 @@ void write_mesh(adios2::IO io, adios2::Engine engine, std::uint32_t num_cells_local = topo_imap->size_local(); std::uint64_t cell_offset = topo_imap->local_range()[0]; - const fem::CoordinateElement& cmap = mesh->geometry().cmap(); + const fem::CoordinateElement& cmap = mesh.geometry().cmap(); fem::ElementDofLayout geom_layout = cmap.create_dof_layout(); std::uint32_t num_dofs_per_cell = geom_layout.num_entity_closure_dofs(dim); @@ -79,7 +79,7 @@ void write_mesh(adios2::IO io, adios2::Engine engine, for (std::size_t i = 0; i < num_cells_local + 1; ++i) offsets_global[i] = offsets[i] + cell_offset * num_dofs_per_cell; - io.DefineAttribute("name", mesh->name); + io.DefineAttribute("name", mesh.name); io.DefineAttribute("dim", geometry.dim()); io.DefineAttribute("cell_type", dolfinx::mesh::to_string(cmap.cell_shape())); @@ -125,13 +125,11 @@ void write_mesh(adios2::IO io, adios2::Engine engine, engine.EndStep(); } -template void -write_mesh(adios2::IO io, adios2::Engine engine, - std::shared_ptr> mesh); +template void write_mesh(adios2::IO io, adios2::Engine engine, + dolfinx::mesh::Mesh mesh); -template void -write_mesh(adios2::IO io, adios2::Engine engine, - std::shared_ptr> mesh); +template void write_mesh(adios2::IO io, adios2::Engine engine, + dolfinx::mesh::Mesh mesh); } // namespace dolfinx::io::checkpointing diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 45a713b784a..29b5eed27bc 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -22,11 +22,12 @@ namespace dolfinx::io::checkpointing /// @brief Write mesh to a file. /// -/// @param[in] adios2engine ADIOS2Engine +/// @param[in] io ADIOS2 IO +/// @param[in] engine ADIOS2 Engine /// @param[in] mesh Mesh of type float or double to write to the file template void write_mesh(adios2::IO io, adios2::Engine engine, - std::shared_ptr> mesh); + dolfinx::mesh::Mesh mesh); } // namespace dolfinx::io::checkpointing From 4d71e4c7b0baa36ffc796d31ae2c3fd618f8a952 Mon Sep 17 00:00:00 2001 From: ampdes Date: Sun, 28 Jul 2024 18:02:59 +0200 Subject: [PATCH 26/60] Add python demo --- python/demo/demo_checkpointing.py | 60 +++++++++++++++++++++++++++++++ python/dolfinx/wrappers/io.cpp | 9 +++++ 2 files changed, 69 insertions(+) create mode 100644 python/demo/demo_checkpointing.py diff --git a/python/demo/demo_checkpointing.py b/python/demo/demo_checkpointing.py new file mode 100644 index 00000000000..4a6005ec5f3 --- /dev/null +++ b/python/demo/demo_checkpointing.py @@ -0,0 +1,60 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: light +# format_version: '1.5' +# jupytext_version: 1.13.6 +# --- + +# # Checkpointing +# +# This demo is implemented in {download}`demo_checkpointing.py`. It +# illustrates checkpointing using ADIOS2: +# + +# + +try: + import adios2.bindings +except: + print("This demo requires adios2.bindings.") + exit(0) + +from mpi4py import MPI + +import numpy as np + +from dolfinx import io, mesh +# - + +# Note that it is important to first `from mpi4py import MPI` to +# ensure that MPI is correctly initialised. + +# We create a rectangular {py:class}`Mesh ` using +# {py:func}`create_rectangle `, and +# save it to a file `mesh.bp` + +# + +msh = mesh.create_rectangle( + comm=MPI.COMM_WORLD, + points=((0.0, 0.0), (1.0, 1.0)), + n=(16, 16), + cell_type=mesh.CellType.triangle, +) +# - + +# + +import adios2.bindings +filename = "mesh.bp" +engine_type = "BP5" +adios = adios2.bindings.ADIOS(msh.comm) +adios2_io = adios.DeclareIO("mesh-write") +adios2_io.SetEngine(engine_type) +adios2_engine = adios2_io.Open(filename, adios2.bindings.Mode.Write) +# - + +# + +io.write_mesh(adios2_io, adios2_engine, msh) +adios2_engine.Close() +# - diff --git a/python/dolfinx/wrappers/io.cpp b/python/dolfinx/wrappers/io.cpp index ed7ae36816b..5a04ccf66ad 100644 --- a/python/dolfinx/wrappers/io.cpp +++ b/python/dolfinx/wrappers/io.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -236,6 +237,14 @@ void declare_data_types(nb::module_& m) void io(nb::module_& m) { +#ifdef HAS_ADIOS2 + // dolfinx::io::checkpointing::write_mesh write mesh to file + m.def("write_mesh", &dolfinx::io::checkpointing::write_mesh, nb::arg("io"), + nb::arg("engine"), nb::arg("mesh"), "Write mesh with float type to file using ADIOS2"); + m.def("write_mesh", &dolfinx::io::checkpointing::write_mesh, nb::arg("io"), + nb::arg("engine"), nb::arg("mesh"), "Write mesh with double type to file using ADIOS2"); +#endif + // dolfinx::io::cell vtk cell type converter m.def("get_vtk_cell_type", &dolfinx::io::cells::get_vtk_cell_type, nb::arg("cell"), nb::arg("dim"), "Get VTK cell identifier"); From 659281a8e3ed655bcc5fe179106b5d7392c32276 Mon Sep 17 00:00:00 2001 From: ampdes Date: Wed, 31 Jul 2024 00:02:29 +0200 Subject: [PATCH 27/60] Rename nodes to vertices --- cpp/dolfinx/io/checkpointing.cpp | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 4a6a6ffa7aa..510cf994b38 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -20,7 +20,7 @@ using namespace dolfinx::io; /// @brief ADIOS2 based checkpointing namespace { -std::map lagrange_variants{ +std::map variant_to_string{ {basix::element::lagrange_variant::unset, "unset"}, {basix::element::lagrange_variant::equispaced, "equispaced"}, {basix::element::lagrange_variant::gll_warped, "gll_warped"}, @@ -29,6 +29,7 @@ std::map lagrange_variants{ namespace dolfinx::io::checkpointing { +//----------------------------------------------------------------------------- template void write_mesh(adios2::IO io, adios2::Engine engine, dolfinx::mesh::Mesh mesh) @@ -41,8 +42,8 @@ void write_mesh(adios2::IO io, adios2::Engine engine, std::shared_ptr geom_imap = mesh.geometry().index_map(); - std::uint64_t num_nodes_global = geom_imap->size_global(); - std::uint32_t num_nodes_local = geom_imap->size_local(); + std::uint64_t num_vertices_global = geom_imap->size_global(); + std::uint32_t num_vertices_local = geom_imap->size_local(); std::uint64_t offset = geom_imap->local_range()[0]; const std::shared_ptr topo_imap @@ -58,7 +59,7 @@ void write_mesh(adios2::IO io, adios2::Engine engine, const std::vector input_global_indices = geometry.input_global_indices(); const std::span input_global_indices_span( - input_global_indices.begin(), num_nodes_local); + input_global_indices.begin(), num_vertices_local); const std::span mesh_x = geometry.x(); std::shared_ptr> connectivity @@ -82,13 +83,13 @@ void write_mesh(adios2::IO io, adios2::Engine engine, io.DefineAttribute("name", mesh.name); io.DefineAttribute("dim", geometry.dim()); io.DefineAttribute("cell_type", - dolfinx::mesh::to_string(cmap.cell_shape())); + mesh::to_string(cmap.cell_shape())); io.DefineAttribute("degree", cmap.degree()); io.DefineAttribute("lagrange_variant", - lagrange_variants[cmap.variant()]); + variant_to_string[cmap.variant()]); - adios2::Variable var_num_nodes - = io.DefineVariable("num_nodes"); + adios2::Variable var_num_vertices + = io.DefineVariable("num_vertices"); adios2::Variable var_num_cells = io.DefineVariable("num_cells"); adios2::Variable var_num_dofs_per_cell @@ -96,12 +97,12 @@ void write_mesh(adios2::IO io, adios2::Engine engine, adios2::Variable var_input_global_indices = io.DefineVariable( - "input_global_indices", {num_nodes_global}, {offset}, - {num_nodes_local}, adios2::ConstantDims); + "input_global_indices", {num_vertices_global}, {offset}, + {num_vertices_local}, adios2::ConstantDims); adios2::Variable var_x - = io.DefineVariable("x", {num_nodes_global, 3}, {offset, 0}, - {num_nodes_local, 3}, adios2::ConstantDims); + = io.DefineVariable("x", {num_vertices_global, 3}, {offset, 0}, + {num_vertices_local, 3}, adios2::ConstantDims); adios2::Variable var_topology_array = io.DefineVariable( @@ -115,11 +116,11 @@ void write_mesh(adios2::IO io, adios2::Engine engine, {num_cells_local + 1}, adios2::ConstantDims); engine.BeginStep(); - engine.Put(var_num_nodes, num_nodes_global); + engine.Put(var_num_vertices, num_vertices_global); engine.Put(var_num_cells, num_cells_global); engine.Put(var_num_dofs_per_cell, num_dofs_per_cell); engine.Put(var_input_global_indices, input_global_indices_span.data()); - engine.Put(var_x, mesh_x.subspan(0, num_nodes_local * 3).data()); + engine.Put(var_x, mesh_x.subspan(0, num_vertices_local * 3).data()); engine.Put(var_topology_array, array_global.data()); engine.Put(var_topology_offsets, offsets_global.data()); engine.EndStep(); From 38bb3d5693159f8a452ae7a4cef525f228a16519 Mon Sep 17 00:00:00 2001 From: ampdes Date: Wed, 31 Jul 2024 14:44:22 +0200 Subject: [PATCH 28/60] Scope into sections --- cpp/dolfinx/io/checkpointing.cpp | 179 +++++++++++++++++-------------- 1 file changed, 101 insertions(+), 78 deletions(-) diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 510cf994b38..b07cee6f662 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -38,23 +38,42 @@ void write_mesh(adios2::IO io, adios2::Engine engine, const mesh::Geometry& geometry = mesh.geometry(); std::shared_ptr topology = mesh.topology(); + // Variables/attributes to save std::int32_t dim = geometry.dim(); - - std::shared_ptr geom_imap - = mesh.geometry().index_map(); - std::uint64_t num_vertices_global = geom_imap->size_global(); - std::uint32_t num_vertices_local = geom_imap->size_local(); - std::uint64_t offset = geom_imap->local_range()[0]; - - const std::shared_ptr topo_imap - = topology->index_map(dim); - std::uint64_t num_cells_global = topo_imap->size_global(); - std::uint32_t num_cells_local = topo_imap->size_local(); - std::uint64_t cell_offset = topo_imap->local_range()[0]; - - const fem::CoordinateElement& cmap = mesh.geometry().cmap(); - fem::ElementDofLayout geom_layout = cmap.create_dof_layout(); - std::uint32_t num_dofs_per_cell = geom_layout.num_entity_closure_dofs(dim); + std::uint64_t num_vertices_global, offset, num_cells_global, cell_offset; + std::uint32_t num_vertices_local, num_cells_local, num_dofs_per_cell, degree; + std::string cell_type, lagrange_variant; + std::vector array_global; + std::vector offsets_global; + + std::shared_ptr geom_imap; + + // Vertices information + { + geom_imap = mesh.geometry().index_map(); + num_vertices_global = geom_imap->size_global(); + num_vertices_local = geom_imap->size_local(); + offset = geom_imap->local_range()[0]; + } + + // Cells information + { + const std::shared_ptr topo_imap + = topology->index_map(dim); + num_cells_global = topo_imap->size_global(); + num_cells_local = topo_imap->size_local(); + cell_offset = topo_imap->local_range()[0]; + } + + // Coordinate element information + { + const fem::CoordinateElement& cmap = mesh.geometry().cmap(); + fem::ElementDofLayout geom_layout = cmap.create_dof_layout(); + num_dofs_per_cell = geom_layout.num_entity_closure_dofs(dim); + cell_type = mesh::to_string(cmap.cell_shape()); + degree = cmap.degree(); + lagrange_variant = variant_to_string[cmap.variant()]; + } const std::vector input_global_indices = geometry.input_global_indices(); @@ -62,68 +81,72 @@ void write_mesh(adios2::IO io, adios2::Engine engine, input_global_indices.begin(), num_vertices_local); const std::span mesh_x = geometry.x(); - std::shared_ptr> connectivity - = topology->connectivity(dim, 0); - const std::vector& array = connectivity->array(); - const std::vector& offsets = connectivity->offsets(); - - const std::span array_span(array.begin(), - offsets[num_cells_local]); - std::vector array_global(offsets[num_cells_local]); - - std::vector offsets_global(num_cells_local + 1); - - std::iota(array_global.begin(), array_global.end(), 0); - - geom_imap->local_to_global(array_span, array_global); - - for (std::size_t i = 0; i < num_cells_local + 1; ++i) - offsets_global[i] = offsets[i] + cell_offset * num_dofs_per_cell; - - io.DefineAttribute("name", mesh.name); - io.DefineAttribute("dim", geometry.dim()); - io.DefineAttribute("cell_type", - mesh::to_string(cmap.cell_shape())); - io.DefineAttribute("degree", cmap.degree()); - io.DefineAttribute("lagrange_variant", - variant_to_string[cmap.variant()]); - - adios2::Variable var_num_vertices - = io.DefineVariable("num_vertices"); - adios2::Variable var_num_cells - = io.DefineVariable("num_cells"); - adios2::Variable var_num_dofs_per_cell - = io.DefineVariable("num_dofs_per_cell"); - - adios2::Variable var_input_global_indices - = io.DefineVariable( - "input_global_indices", {num_vertices_global}, {offset}, - {num_vertices_local}, adios2::ConstantDims); - - adios2::Variable var_x - = io.DefineVariable("x", {num_vertices_global, 3}, {offset, 0}, - {num_vertices_local, 3}, adios2::ConstantDims); - - adios2::Variable var_topology_array - = io.DefineVariable( - "topology_array", {num_cells_global * num_dofs_per_cell}, - {cell_offset * num_dofs_per_cell}, - {num_cells_local * num_dofs_per_cell}, adios2::ConstantDims); - - adios2::Variable var_topology_offsets - = io.DefineVariable( - "topology_offsets", {num_cells_global + 1}, {cell_offset}, - {num_cells_local + 1}, adios2::ConstantDims); - - engine.BeginStep(); - engine.Put(var_num_vertices, num_vertices_global); - engine.Put(var_num_cells, num_cells_global); - engine.Put(var_num_dofs_per_cell, num_dofs_per_cell); - engine.Put(var_input_global_indices, input_global_indices_span.data()); - engine.Put(var_x, mesh_x.subspan(0, num_vertices_local * 3).data()); - engine.Put(var_topology_array, array_global.data()); - engine.Put(var_topology_offsets, offsets_global.data()); - engine.EndStep(); + // Connectivity + { + std::shared_ptr> connectivity + = topology->connectivity(dim, 0); + const std::vector& array = connectivity->array(); + const std::vector& offsets = connectivity->offsets(); + + const std::span array_span(array.begin(), + offsets[num_cells_local]); + + array_global.resize(offsets[num_cells_local]); + offsets_global.resize(num_cells_local + 1); + + std::iota(array_global.begin(), array_global.end(), 0); + + geom_imap->local_to_global(array_span, array_global); + + for (std::size_t i = 0; i < num_cells_local + 1; ++i) + offsets_global[i] = offsets[i] + cell_offset * num_dofs_per_cell; + } + + // ADIOS2 write attributes and variables + { + io.DefineAttribute("name", mesh.name); + io.DefineAttribute("dim", dim); + io.DefineAttribute("cell_type", cell_type); + io.DefineAttribute("degree", degree); + io.DefineAttribute("lagrange_variant", lagrange_variant); + + adios2::Variable var_num_vertices + = io.DefineVariable("num_vertices"); + adios2::Variable var_num_cells + = io.DefineVariable("num_cells"); + adios2::Variable var_num_dofs_per_cell + = io.DefineVariable("num_dofs_per_cell"); + + adios2::Variable var_input_global_indices + = io.DefineVariable( + "input_global_indices", {num_vertices_global}, {offset}, + {num_vertices_local}, adios2::ConstantDims); + + adios2::Variable var_x + = io.DefineVariable("x", {num_vertices_global, 3}, {offset, 0}, + {num_vertices_local, 3}, adios2::ConstantDims); + + adios2::Variable var_topology_array + = io.DefineVariable( + "topology_array", {num_cells_global * num_dofs_per_cell}, + {cell_offset * num_dofs_per_cell}, + {num_cells_local * num_dofs_per_cell}, adios2::ConstantDims); + + adios2::Variable var_topology_offsets + = io.DefineVariable( + "topology_offsets", {num_cells_global + 1}, {cell_offset}, + {num_cells_local + 1}, adios2::ConstantDims); + + engine.BeginStep(); + engine.Put(var_num_vertices, num_vertices_global); + engine.Put(var_num_cells, num_cells_global); + engine.Put(var_num_dofs_per_cell, num_dofs_per_cell); + engine.Put(var_input_global_indices, input_global_indices_span.data()); + engine.Put(var_x, mesh_x.subspan(0, num_vertices_local * 3).data()); + engine.Put(var_topology_array, array_global.data()); + engine.Put(var_topology_offsets, offsets_global.data()); + engine.EndStep(); + } } template void write_mesh(adios2::IO io, adios2::Engine engine, From f8d5701ba35cd751fe0ea286c905962d6f569e70 Mon Sep 17 00:00:00 2001 From: ampdes Date: Wed, 31 Jul 2024 15:37:13 +0200 Subject: [PATCH 29/60] Remove python demo --- python/demo/demo_checkpointing.py | 60 ------------------------------- python/dolfinx/wrappers/io.cpp | 9 ----- 2 files changed, 69 deletions(-) delete mode 100644 python/demo/demo_checkpointing.py diff --git a/python/demo/demo_checkpointing.py b/python/demo/demo_checkpointing.py deleted file mode 100644 index 4a6005ec5f3..00000000000 --- a/python/demo/demo_checkpointing.py +++ /dev/null @@ -1,60 +0,0 @@ -# --- -# jupyter: -# jupytext: -# text_representation: -# extension: .py -# format_name: light -# format_version: '1.5' -# jupytext_version: 1.13.6 -# --- - -# # Checkpointing -# -# This demo is implemented in {download}`demo_checkpointing.py`. It -# illustrates checkpointing using ADIOS2: -# - -# + -try: - import adios2.bindings -except: - print("This demo requires adios2.bindings.") - exit(0) - -from mpi4py import MPI - -import numpy as np - -from dolfinx import io, mesh -# - - -# Note that it is important to first `from mpi4py import MPI` to -# ensure that MPI is correctly initialised. - -# We create a rectangular {py:class}`Mesh ` using -# {py:func}`create_rectangle `, and -# save it to a file `mesh.bp` - -# + -msh = mesh.create_rectangle( - comm=MPI.COMM_WORLD, - points=((0.0, 0.0), (1.0, 1.0)), - n=(16, 16), - cell_type=mesh.CellType.triangle, -) -# - - -# + -import adios2.bindings -filename = "mesh.bp" -engine_type = "BP5" -adios = adios2.bindings.ADIOS(msh.comm) -adios2_io = adios.DeclareIO("mesh-write") -adios2_io.SetEngine(engine_type) -adios2_engine = adios2_io.Open(filename, adios2.bindings.Mode.Write) -# - - -# + -io.write_mesh(adios2_io, adios2_engine, msh) -adios2_engine.Close() -# - diff --git a/python/dolfinx/wrappers/io.cpp b/python/dolfinx/wrappers/io.cpp index 5a04ccf66ad..ed7ae36816b 100644 --- a/python/dolfinx/wrappers/io.cpp +++ b/python/dolfinx/wrappers/io.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -237,14 +236,6 @@ void declare_data_types(nb::module_& m) void io(nb::module_& m) { -#ifdef HAS_ADIOS2 - // dolfinx::io::checkpointing::write_mesh write mesh to file - m.def("write_mesh", &dolfinx::io::checkpointing::write_mesh, nb::arg("io"), - nb::arg("engine"), nb::arg("mesh"), "Write mesh with float type to file using ADIOS2"); - m.def("write_mesh", &dolfinx::io::checkpointing::write_mesh, nb::arg("io"), - nb::arg("engine"), nb::arg("mesh"), "Write mesh with double type to file using ADIOS2"); -#endif - // dolfinx::io::cell vtk cell type converter m.def("get_vtk_cell_type", &dolfinx::io::cells::get_vtk_cell_type, nb::arg("cell"), nb::arg("dim"), "Get VTK cell identifier"); From 0940f59ac8c7d010158149ab6034da0ac04ef4d8 Mon Sep 17 00:00:00 2001 From: ampdes Date: Wed, 31 Jul 2024 15:41:39 +0200 Subject: [PATCH 30/60] Remove ADIOS_utils --- cpp/dolfinx/io/ADIOS2_utils.cpp | 36 ----------------- cpp/dolfinx/io/ADIOS2_utils.h | 69 --------------------------------- cpp/dolfinx/io/CMakeLists.txt | 4 +- cpp/dolfinx/io/checkpointing.h | 1 - cpp/dolfinx/io/dolfinx_io.h | 1 - 5 files changed, 1 insertion(+), 110 deletions(-) delete mode 100644 cpp/dolfinx/io/ADIOS2_utils.cpp delete mode 100644 cpp/dolfinx/io/ADIOS2_utils.h diff --git a/cpp/dolfinx/io/ADIOS2_utils.cpp b/cpp/dolfinx/io/ADIOS2_utils.cpp deleted file mode 100644 index 120ed2dabbd..00000000000 --- a/cpp/dolfinx/io/ADIOS2_utils.cpp +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (C) 2024 Abdullah Mujahid, Jørgen S. Dokken and Garth N. Wells -// -// This file is part of DOLFINX (https://www.fenicsproject.org) -// -// SPDX-License-Identifier: LGPL-3.0-or-later - -#ifdef HAS_ADIOS2 - -#include "ADIOS2_utils.h" -#include -#include - -using namespace dolfinx::io; - -//----------------------------------------------------------------------------- -ADIOS2Engine::ADIOS2Engine(MPI_Comm comm, const std::filesystem::path& filename, - std::string tag, std::string engine, - const adios2::Mode mode) - - : _adios(std::make_unique(comm)), - _io(std::make_unique(_adios->DeclareIO(tag))) -{ - _io->SetEngine(engine); - _engine = std::make_unique(_io->Open(filename, mode)); -} -//----------------------------------------------------------------------------- -ADIOS2Engine::~ADIOS2Engine() { close(); } -//----------------------------------------------------------------------------- -void ADIOS2Engine::close() -{ - assert(_engine); - if (*_engine) - _engine->Close(); -} - -#endif diff --git a/cpp/dolfinx/io/ADIOS2_utils.h b/cpp/dolfinx/io/ADIOS2_utils.h deleted file mode 100644 index e926bfd684c..00000000000 --- a/cpp/dolfinx/io/ADIOS2_utils.h +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (C) 2024 Abdullah Mujahid, Jørgen S. Dokken and Garth N. Wells -// -// This file is part of DOLFINX (https://www.fenicsproject.org) -// -// SPDX-License-Identifier: LGPL-3.0-or-later - -#pragma once - -#ifdef HAS_ADIOS2 - -#include -#include -#include -#include - -/// @file ADIOS2_utils.h -/// @brief Utils for ADIOS2 - -namespace dolfinx::io -{ - -/// ADIOS2-based writers/readers -class ADIOS2Engine -{ -public: - /// @brief Create an ADIOS2-based engine writer/reader - /// @param[in] comm The MPI communicator - /// @param[in] filename Name of output file - /// @param[in] tag The ADIOS2 object name - /// @param[in] engine ADIOS2 engine type. See - /// https://adios2.readthedocs.io/en/latest/engines/engines.html. - /// @param[in] mode ADIOS2 mode, default is Write or Read - ADIOS2Engine(MPI_Comm comm, const std::filesystem::path& filename, - std::string tag, std::string engine = "BP5", - const adios2::Mode mode = adios2::Mode::Write); - - /// @brief Move constructor - ADIOS2Engine(ADIOS2Engine&& engine) = default; - - /// @brief Copy constructor - ADIOS2Engine(const ADIOS2Engine&) = delete; - - /// @brief Destructor - ~ADIOS2Engine(); - - /// @brief Move assignment - ADIOS2Engine& operator=(ADIOS2Engine&& engine) = default; - - // Copy assignment - ADIOS2Engine& operator=(const ADIOS2Engine&) = delete; - - /// @brief Close the file - void close(); - - /// @brief Get the IO object - std::shared_ptr io() { return _io; } - - /// @brief Get the Engine object - std::shared_ptr engine() { return _engine; } - -protected: - std::shared_ptr _adios; - std::shared_ptr _io; - std::shared_ptr _engine; -}; - -} // namespace dolfinx::io - -#endif diff --git a/cpp/dolfinx/io/CMakeLists.txt b/cpp/dolfinx/io/CMakeLists.txt index 6e8c6fe59d0..9374d06c067 100644 --- a/cpp/dolfinx/io/CMakeLists.txt +++ b/cpp/dolfinx/io/CMakeLists.txt @@ -1,6 +1,5 @@ set(HEADERS_io ${CMAKE_CURRENT_SOURCE_DIR}/dolfinx_io.h - ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2_utils.h ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.h ${CMAKE_CURRENT_SOURCE_DIR}/cells.h ${CMAKE_CURRENT_SOURCE_DIR}/checkpointing.h @@ -16,8 +15,7 @@ set(HEADERS_io target_sources( dolfinx - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2_utils.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.cpp + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/cells.cpp ${CMAKE_CURRENT_SOURCE_DIR}/checkpointing.cpp ${CMAKE_CURRENT_SOURCE_DIR}/HDF5Interface.cpp diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 29b5eed27bc..03884a36dff 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -8,7 +8,6 @@ #ifdef HAS_ADIOS2 -#include "ADIOS2_utils.h" #include #include #include diff --git a/cpp/dolfinx/io/dolfinx_io.h b/cpp/dolfinx/io/dolfinx_io.h index 1bb24940950..f8f499f049d 100644 --- a/cpp/dolfinx/io/dolfinx_io.h +++ b/cpp/dolfinx/io/dolfinx_io.h @@ -9,7 +9,6 @@ namespace dolfinx::io // DOLFINx io interface -#include #include #include #include From 2a4a986b8a1d57bba6f129d29cc360cfd6026cbf Mon Sep 17 00:00:00 2001 From: ampdes Date: Wed, 31 Jul 2024 19:07:04 +0200 Subject: [PATCH 31/60] Switch to pass by reference --- cpp/dolfinx/io/checkpointing.cpp | 16 ++++++++++------ cpp/dolfinx/io/checkpointing.h | 4 ++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index b07cee6f662..fdfaabc4b29 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -31,8 +31,8 @@ namespace dolfinx::io::checkpointing { //----------------------------------------------------------------------------- template -void write_mesh(adios2::IO io, adios2::Engine engine, - dolfinx::mesh::Mesh mesh) +void write_mesh(adios2::IO& io, adios2::Engine& engine, + dolfinx::mesh::Mesh& mesh) { const mesh::Geometry& geometry = mesh.geometry(); @@ -149,11 +149,15 @@ void write_mesh(adios2::IO io, adios2::Engine engine, } } -template void write_mesh(adios2::IO io, adios2::Engine engine, - dolfinx::mesh::Mesh mesh); +//----------------------------------------------------------------------------- +/// @cond +template void write_mesh(adios2::IO& io, adios2::Engine& engine, + dolfinx::mesh::Mesh& mesh); + +template void write_mesh(adios2::IO& io, adios2::Engine& engine, + dolfinx::mesh::Mesh& mesh); -template void write_mesh(adios2::IO io, adios2::Engine engine, - dolfinx::mesh::Mesh mesh); +/// @endcond } // namespace dolfinx::io::checkpointing diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 03884a36dff..8f1f20edc64 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -25,8 +25,8 @@ namespace dolfinx::io::checkpointing /// @param[in] engine ADIOS2 Engine /// @param[in] mesh Mesh of type float or double to write to the file template -void write_mesh(adios2::IO io, adios2::Engine engine, - dolfinx::mesh::Mesh mesh); +void write_mesh(adios2::IO& io, adios2::Engine& engine, + dolfinx::mesh::Mesh& mesh); } // namespace dolfinx::io::checkpointing From cbc6fb282a8fc585d0f8d5e8787706c642df1e27 Mon Sep 17 00:00:00 2001 From: ampdes Date: Wed, 31 Jul 2024 21:31:18 +0200 Subject: [PATCH 32/60] WIP-buggy code dump for mesh_read --- cpp/dolfinx/io/checkpointing.cpp | 145 ++++++++++++++++++++++++++++++- cpp/dolfinx/io/checkpointing.h | 3 + 2 files changed, 145 insertions(+), 3 deletions(-) diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index fdfaabc4b29..92247d0cfdd 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -13,9 +13,6 @@ #include #include -using namespace dolfinx; -using namespace dolfinx::io; - /// @file checkpointing.h /// @brief ADIOS2 based checkpointing namespace @@ -25,6 +22,13 @@ std::map variant_to_string{ {basix::element::lagrange_variant::equispaced, "equispaced"}, {basix::element::lagrange_variant::gll_warped, "gll_warped"}, }; + +std::map string_to_variant{ + {"unset", basix::element::lagrange_variant::unset}, + {"equispaced", basix::element::lagrange_variant::equispaced}, + {"gll_warped", basix::element::lagrange_variant::gll_warped}, +}; + } // namespace namespace dolfinx::io::checkpointing @@ -159,6 +163,141 @@ template void write_mesh(adios2::IO& io, adios2::Engine& engine, /// @endcond +//----------------------------------------------------------------------------- +dolfinx::mesh::Mesh read_mesh(adios2::IO& io, + adios2::Engine& engine, + MPI_Comm& comm = MPI_COMM_WORLD) +{ + + int rank, size; + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &size); + + engine.BeginStep(); + + // Attributes + std::string name; + std::int32_t dim, degree; + mesh::CellType cell_type; + basix::element::lagrange_variant lagrange_variant; + + // Read attributes + { + adios2::Attribute var_name + = io.InquireAttribute("name"); + adios2::Attribute var_dim + = io.InquireAttribute("dim"); + adios2::Attribute var_cell_type + = io.InquireAttribute("cell_type"); + adios2::Attribute var_degree + = io.InquireAttribute("degree"); + adios2::Attribute var_variant + = io.InquireAttribute("lagrange_variant"); + + name = var_name.Data()[0]; + dim = var_dim.Data()[0]; + cell_type = mesh::to_type(var_cell_type.Data()[0]); + degree = var_degree.Data()[0]; + lagrange_variant = string_to_variant[var_variant.Data()[0]]; + } + + std::cout << dim << std::endl; + + std::uint64_t num_vertices_global; + std::uint64_t num_cells_global; + std::uint32_t num_dofs_per_cell; + + // Read scalar variables + { + adios2::Variable var_num_vertices + = io.InquireVariable("num_vertices"); + adios2::Variable var_num_cells + = io.InquireVariable("num_cells"); + adios2::Variable var_num_dofs_per_cell + = io.InquireVariable("num_dofs_per_cell"); + + engine.Get(var_num_vertices, num_vertices_global); + engine.Get(var_num_cells, num_cells_global); + engine.Get(var_num_dofs_per_cell, num_dofs_per_cell); + } + + // Compute local sizes, offsets + std::array local_range + = dolfinx::MPI::local_range(rank, num_vertices_global, size); + int num_vertices_local = local_range[1] - local_range[0]; + + std::array cell_range + = dolfinx::MPI::local_range(rank, num_cells_global, size); + int num_cells_local = cell_range[1] - cell_range[0]; + + if ("float" == io.VariableType("x")) + using T = float; + else if ("double" == io.VariableType("x")) + using T = double; + + std::cout << T << std::endl; + + std::vector input_global_indices(num_vertices_local); + std::vector mesh_x(num_vertices_local * 3); + std::vector array(num_cells_local * num_dofs_per_cell); + std::vector offsets(num_cells_local + 1); + + adios2::Variable var_input_global_indices + = io.InquireVariable("input_global_indices"); + + adios2::Variable var_x = io.InquireVariable("x"); + + adios2::Variable var_topology_array + = io.InquireVariable("topology_array"); + + adios2::Variable var_topology_offsets + = io.InquireVariable("topology_offsets"); + + if (var_input_global_indices) + { + var_input_global_indices.SetSelection( + {{local_range[0]}, {num_vertices_local}}); + engine.Get(var_input_global_indices, input_global_indices.data(), + adios2::Mode::Deferred); + } + + if (var_x) + { + var_x.SetSelection({{local_range[0], 0}, {num_vertices_local, 3}}); + engine.Get(var_x, mesh_x.data(), adios2::Mode::Deferred); + } + + if (var_topology_array) + { + var_topology_array.SetSelection({{cell_range[0] * num_dofs_per_cell}, + {cell_range[1] * num_dofs_per_cell}}); + engine.Get(var_topology_array, array.data(), adios2::Mode::Deferred); + } + + if (var_topology_offsets) + { + var_topology_offsets.SetSelection({{cell_range[0]}, {cell_range[1] + 1}}); + engine.Get(var_topology_offsets, offsets.data(), adios2::Mode::Deferred); + } + + engine.EndStep(); + engine.Close(); + + std::int32_t cell_offset = offsets[0]; + for (int i = 0; i < offsets.size(); ++i) + offsets[i] -= cell_offset; + + fem::CoordinateElement element + = fem::CoordinateElement(cell_type, degree, lagrange_variant); + + std::array xshape = {num_vertices_local, 3}; + auto part = mesh::create_cell_partitioner(mesh::GhostMode::shared_facet); + + mesh::Mesh mesh = mesh::create_mesh(comm, comm, array, element, comm, + mesh_x, xshape, part); + return mesh; +} + } // namespace dolfinx::io::checkpointing #endif diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 8f1f20edc64..61d5b083104 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -8,6 +8,7 @@ #ifdef HAS_ADIOS2 +#include "boost/variant.hpp" #include #include #include @@ -19,6 +20,8 @@ namespace dolfinx::io::checkpointing { +typedef boost::variant floating_point; + /// @brief Write mesh to a file. /// /// @param[in] io ADIOS2 IO From d82a41260d57df8a65fa06912e691c4b456ea9af Mon Sep 17 00:00:00 2001 From: ampdes Date: Wed, 31 Jul 2024 22:34:41 +0200 Subject: [PATCH 33/60] Fix errors in read_mesh, add version, git_hash --- cpp/demo/checkpointing/main.cpp | 5 +++++ cpp/dolfinx/io/checkpointing.cpp | 29 +++++++++++++++-------------- cpp/dolfinx/io/checkpointing.h | 9 +++++++++ 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 41f30526ba7..c16a30e6719 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -32,6 +33,10 @@ int main(int argc, char* argv[]) io.SetEngine("BP5"); adios2::Engine engine = io.Open("mesh.bp", adios2::Mode::Write); + // TODO: Need to move this inside the checkpointing module + io.DefineAttribute("version", DOLFINX_VERSION_STRING); + io.DefineAttribute("git_hash", DOLFINX_VERSION_GIT); + io::checkpointing::write_mesh(io, engine, *mesh); engine.Close(); diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 92247d0cfdd..e7117caca42 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include /// @file checkpointing.h @@ -164,9 +165,8 @@ template void write_mesh(adios2::IO& io, adios2::Engine& engine, /// @endcond //----------------------------------------------------------------------------- -dolfinx::mesh::Mesh read_mesh(adios2::IO& io, - adios2::Engine& engine, - MPI_Comm& comm = MPI_COMM_WORLD) +dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, + MPI_Comm comm) { int rank, size; @@ -222,20 +222,21 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, } // Compute local sizes, offsets - std::array local_range + std::array _local_range = dolfinx::MPI::local_range(rank, num_vertices_global, size); - int num_vertices_local = local_range[1] - local_range[0]; - std::array cell_range + std::array local_range{(std::uint64_t)_local_range[0], + (std::uint64_t)_local_range[1]}; + std::uint64_t num_vertices_local = local_range[1] - local_range[0]; + + std::array _cell_range = dolfinx::MPI::local_range(rank, num_cells_global, size); - int num_cells_local = cell_range[1] - cell_range[0]; - if ("float" == io.VariableType("x")) - using T = float; - else if ("double" == io.VariableType("x")) - using T = double; + std::array cell_range{(std::uint64_t)_cell_range[0], + (std::uint64_t)_cell_range[1]}; + std::uint64_t num_cells_local = cell_range[1] - cell_range[0]; - std::cout << T << std::endl; + using T = float; std::vector input_global_indices(num_vertices_local); std::vector mesh_x(num_vertices_local * 3); @@ -284,8 +285,8 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, engine.Close(); std::int32_t cell_offset = offsets[0]; - for (int i = 0; i < offsets.size(); ++i) - offsets[i] -= cell_offset; + for (auto offset = offsets.begin(); offset != offsets.end(); ++offset) + *offset -= cell_offset; fem::CoordinateElement element = fem::CoordinateElement(cell_type, degree, lagrange_variant); diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 61d5b083104..5fbf876d4c0 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -31,6 +31,15 @@ template void write_mesh(adios2::IO& io, adios2::Engine& engine, dolfinx::mesh::Mesh& mesh); +/// @brief Write mesh to a file. +/// +/// @param[in] io ADIOS2 IO +/// @param[in] engine ADIOS2 Engine +/// @param[in] comm comm +/// @return mesh reconstructed from the data +dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, + MPI_Comm comm = MPI_COMM_WORLD); + } // namespace dolfinx::io::checkpointing #endif From 5f4a50d0a43e2db1547b221d6f29123b2b3b03d3 Mon Sep 17 00:00:00 2001 From: ampdes Date: Thu, 1 Aug 2024 10:50:13 +0200 Subject: [PATCH 34/60] Add read_mesh of one type --- cpp/demo/checkpointing/main.cpp | 13 +++++ cpp/dolfinx/io/checkpointing.cpp | 87 +++++++++++++++++--------------- 2 files changed, 60 insertions(+), 40 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index c16a30e6719..5c006a5e68b 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -41,6 +41,19 @@ int main(int argc, char* argv[]) engine.Close(); + adios2::IO io_read = adios.DeclareIO("mesh-read"); + io_read.SetEngine("BP5"); + + adios2::Engine reader = io_read.Open("mesh.bp", adios2::Mode::Read); + auto mesh_read = io::checkpointing::read_mesh(io_read, reader, mesh->comm()); + reader.Close(); + + adios2::IO io_write = adios.DeclareIO("mesh-rewrite"); + io_write.SetEngine("BP5"); + + adios2::Engine writer = io_write.Open("mesh2.bp", adios2::Mode::Write); + io::checkpointing::write_mesh(io_write, writer, mesh_read); + MPI_Finalize(); return 0; } diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index e7117caca42..c726a87c860 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -201,8 +201,7 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, lagrange_variant = string_to_variant[var_variant.Data()[0]]; } - std::cout << dim << std::endl; - + // Scalar variables std::uint64_t num_vertices_global; std::uint64_t num_cells_global; std::uint32_t num_dofs_per_cell; @@ -239,63 +238,71 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, using T = float; std::vector input_global_indices(num_vertices_local); - std::vector mesh_x(num_vertices_local * 3); + std::vector x(num_vertices_local * 3); std::vector array(num_cells_local * num_dofs_per_cell); std::vector offsets(num_cells_local + 1); - adios2::Variable var_input_global_indices - = io.InquireVariable("input_global_indices"); - - adios2::Variable var_x = io.InquireVariable("x"); + { + adios2::Variable var_input_global_indices + = io.InquireVariable("input_global_indices"); - adios2::Variable var_topology_array - = io.InquireVariable("topology_array"); + adios2::Variable var_x = io.InquireVariable("x"); - adios2::Variable var_topology_offsets - = io.InquireVariable("topology_offsets"); + adios2::Variable var_topology_array + = io.InquireVariable("topology_array"); - if (var_input_global_indices) - { - var_input_global_indices.SetSelection( - {{local_range[0]}, {num_vertices_local}}); - engine.Get(var_input_global_indices, input_global_indices.data(), - adios2::Mode::Deferred); - } + adios2::Variable var_topology_offsets + = io.InquireVariable("topology_offsets"); + + if (var_input_global_indices) + { + var_input_global_indices.SetSelection( + {{local_range[0]}, {num_vertices_local}}); + engine.Get(var_input_global_indices, input_global_indices.data(), + adios2::Mode::Deferred); + } + + if (var_x) + { + var_x.SetSelection({{local_range[0], 0}, {num_vertices_local, 3}}); + engine.Get(var_x, x.data(), adios2::Mode::Deferred); + } + + if (var_topology_array) + { + var_topology_array.SetSelection({{cell_range[0] * num_dofs_per_cell}, + {cell_range[1] * num_dofs_per_cell}}); + engine.Get(var_topology_array, array.data(), adios2::Mode::Deferred); + } + + if (var_topology_offsets) + { + var_topology_offsets.SetSelection({{cell_range[0]}, {cell_range[1] + 1}}); + engine.Get(var_topology_offsets, offsets.data(), adios2::Mode::Deferred); + } - if (var_x) - { - var_x.SetSelection({{local_range[0], 0}, {num_vertices_local, 3}}); - engine.Get(var_x, mesh_x.data(), adios2::Mode::Deferred); - } + engine.EndStep(); - if (var_topology_array) - { - var_topology_array.SetSelection({{cell_range[0] * num_dofs_per_cell}, - {cell_range[1] * num_dofs_per_cell}}); - engine.Get(var_topology_array, array.data(), adios2::Mode::Deferred); + std::int32_t cell_offset = offsets[0]; + for (auto offset = offsets.begin(); offset != offsets.end(); ++offset) + *offset -= cell_offset; } - if (var_topology_offsets) + std::vector x_reduced(num_vertices_local * dim); + for (std::uint32_t i = 0; i < num_vertices_local; ++i) { - var_topology_offsets.SetSelection({{cell_range[0]}, {cell_range[1] + 1}}); - engine.Get(var_topology_offsets, offsets.data(), adios2::Mode::Deferred); + for (std::uint32_t j = 0; j < (std::uint32_t)dim; ++j) + x_reduced[i * dim + j] = x[i * 3 + j]; } - engine.EndStep(); - engine.Close(); - - std::int32_t cell_offset = offsets[0]; - for (auto offset = offsets.begin(); offset != offsets.end(); ++offset) - *offset -= cell_offset; - fem::CoordinateElement element = fem::CoordinateElement(cell_type, degree, lagrange_variant); - std::array xshape = {num_vertices_local, 3}; + std::array xshape = {num_vertices_local, (std::uint32_t)dim}; auto part = mesh::create_cell_partitioner(mesh::GhostMode::shared_facet); mesh::Mesh mesh = mesh::create_mesh(comm, comm, array, element, comm, - mesh_x, xshape, part); + x_reduced, xshape, part); return mesh; } From d537461dba75fb48b6e8eb55d7ab99c5aedaf7a0 Mon Sep 17 00:00:00 2001 From: ampdes Date: Thu, 1 Aug 2024 11:27:05 +0200 Subject: [PATCH 35/60] Fix docs --- cpp/dolfinx/io/checkpointing.cpp | 5 +++++ cpp/dolfinx/io/checkpointing.h | 4 +--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index c726a87c860..9913ba66d08 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -235,6 +235,11 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, (std::uint64_t)_cell_range[1]}; std::uint64_t num_cells_local = cell_range[1] - cell_range[0]; + // TODO: Determine type + // if ("float" == io.VariableType("x")) + // using T = float; + // else if ("double" == io.VariableType("x")) + // using T = double; using T = float; std::vector input_global_indices(num_vertices_local); diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 5fbf876d4c0..bc4caa65277 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -20,8 +20,6 @@ namespace dolfinx::io::checkpointing { -typedef boost::variant floating_point; - /// @brief Write mesh to a file. /// /// @param[in] io ADIOS2 IO @@ -31,7 +29,7 @@ template void write_mesh(adios2::IO& io, adios2::Engine& engine, dolfinx::mesh::Mesh& mesh); -/// @brief Write mesh to a file. +/// @brief Read mesh from a file. /// /// @param[in] io ADIOS2 IO /// @param[in] engine ADIOS2 Engine From 4178385645c39e7d6d108360ed9480fa47e1b0f1 Mon Sep 17 00:00:00 2001 From: ampdes Date: Thu, 1 Aug 2024 17:18:40 +0200 Subject: [PATCH 36/60] Templated read_mesh --- cpp/demo/checkpointing/main.cpp | 34 ++++++++++++++++++++++++++------ cpp/dolfinx/io/checkpointing.cpp | 22 ++++++++++++--------- cpp/dolfinx/io/checkpointing.h | 5 +++-- 3 files changed, 44 insertions(+), 17 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 5c006a5e68b..52d9c8bed11 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -45,14 +45,36 @@ int main(int argc, char* argv[]) io_read.SetEngine("BP5"); adios2::Engine reader = io_read.Open("mesh.bp", adios2::Mode::Read); - auto mesh_read = io::checkpointing::read_mesh(io_read, reader, mesh->comm()); - reader.Close(); - adios2::IO io_write = adios.DeclareIO("mesh-rewrite"); - io_write.SetEngine("BP5"); + // TODO: move type deduction inside checkpointing + if ("float" == io.VariableType("x")) + { + using T = float; + mesh::Mesh mesh_read + = io::checkpointing::read_mesh(io_read, reader, mesh->comm()); + reader.Close(); - adios2::Engine writer = io_write.Open("mesh2.bp", adios2::Mode::Write); - io::checkpointing::write_mesh(io_write, writer, mesh_read); + adios2::IO io_write = adios.DeclareIO("mesh-rewrite"); + io_write.SetEngine("BP5"); + + adios2::Engine writer = io_write.Open("mesh2.bp", adios2::Mode::Write); + io::checkpointing::write_mesh(io_write, writer, mesh_read); + writer.Close(); + } + else if ("double" == io.VariableType("x")) + { + using T = double; + mesh::Mesh mesh_read + = io::checkpointing::read_mesh(io_read, reader, mesh->comm()); + reader.Close(); + + adios2::IO io_write = adios.DeclareIO("mesh-rewrite"); + io_write.SetEngine("BP5"); + + adios2::Engine writer = io_write.Open("mesh2.bp", adios2::Mode::Write); + io::checkpointing::write_mesh(io_write, writer, mesh_read); + writer.Close(); + } MPI_Finalize(); return 0; diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 9913ba66d08..7c7655405b2 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -165,8 +165,9 @@ template void write_mesh(adios2::IO& io, adios2::Engine& engine, /// @endcond //----------------------------------------------------------------------------- -dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, - MPI_Comm comm) +template +dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, + MPI_Comm comm) { int rank, size; @@ -235,13 +236,6 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, (std::uint64_t)_cell_range[1]}; std::uint64_t num_cells_local = cell_range[1] - cell_range[0]; - // TODO: Determine type - // if ("float" == io.VariableType("x")) - // using T = float; - // else if ("double" == io.VariableType("x")) - // using T = double; - using T = float; - std::vector input_global_indices(num_vertices_local); std::vector x(num_vertices_local * 3); std::vector array(num_cells_local * num_dofs_per_cell); @@ -311,6 +305,16 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, return mesh; } +//----------------------------------------------------------------------------- +/// @cond +template dolfinx::mesh::Mesh +read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm); + +template dolfinx::mesh::Mesh +read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm); + +/// @endcond + } // namespace dolfinx::io::checkpointing #endif diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index bc4caa65277..4fb351eb843 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -35,8 +35,9 @@ void write_mesh(adios2::IO& io, adios2::Engine& engine, /// @param[in] engine ADIOS2 Engine /// @param[in] comm comm /// @return mesh reconstructed from the data -dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, - MPI_Comm comm = MPI_COMM_WORLD); +template +dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, + MPI_Comm comm = MPI_COMM_WORLD); } // namespace dolfinx::io::checkpointing From 6d341679c28f7f25a0fdb17a740168223c431b14 Mon Sep 17 00:00:00 2001 From: ampdes Date: Thu, 1 Aug 2024 21:36:26 +0200 Subject: [PATCH 37/60] Add python demo for ADIOS2 wrapper write --- cpp/demo/checkpointing/main.cpp | 6 +++ cpp/dolfinx/io/ADIOS2_utils.cpp | 44 ++++++++++++++++++++ cpp/dolfinx/io/ADIOS2_utils.h | 69 +++++++++++++++++++++++++++++++ cpp/dolfinx/io/CMakeLists.txt | 4 +- cpp/dolfinx/io/checkpointing.cpp | 15 +++++++ cpp/dolfinx/io/checkpointing.h | 7 +++- cpp/dolfinx/io/dolfinx_io.h | 1 + python/demo/demo_checkpointing.py | 65 +++++++++++++++++++++++++++++ python/dolfinx/io/__init__.py | 12 +++++- python/dolfinx/io/utils.py | 22 +++++++++- python/dolfinx/wrappers/io.cpp | 26 ++++++++++++ 11 files changed, 266 insertions(+), 5 deletions(-) create mode 100644 cpp/dolfinx/io/ADIOS2_utils.cpp create mode 100644 cpp/dolfinx/io/ADIOS2_utils.h create mode 100644 python/demo/demo_checkpointing.py diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 52d9c8bed11..965df4ac13d 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -76,6 +77,11 @@ int main(int argc, char* argv[]) writer.Close(); } + auto container + = ADIOS2Container(mesh->comm(), "test.bp", "test-write", "BP5", "write"); + + io::checkpointing::write_test(container); + MPI_Finalize(); return 0; } diff --git a/cpp/dolfinx/io/ADIOS2_utils.cpp b/cpp/dolfinx/io/ADIOS2_utils.cpp new file mode 100644 index 00000000000..3d07371d13a --- /dev/null +++ b/cpp/dolfinx/io/ADIOS2_utils.cpp @@ -0,0 +1,44 @@ +// Copyright (C) 2024 Abdullah Mujahid, Jørgen S. Dokken and Garth N. Wells +// +// This file is part of DOLFINX (https://www.fenicsproject.org) +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#ifdef HAS_ADIOS2 + +#include "ADIOS2_utils.h" +#include +#include + +namespace +{ +std::map string_to_mode{ + {"write", adios2::Mode::Write}, + {"read", adios2::Mode::Read}, +}; +} // namespace + +using namespace dolfinx::io; + +//----------------------------------------------------------------------------- +ADIOS2Container::ADIOS2Container(MPI_Comm comm, std::string filename, + std::string tag, std::string engine, + std::string mode) + + : _adios(std::make_unique(comm)), + _io(std::make_unique(_adios->DeclareIO(tag))) +{ + _io->SetEngine(engine); + _engine = std::make_unique(_io->Open(filename, string_to_mode[mode])); +} +//----------------------------------------------------------------------------- +ADIOS2Container::~ADIOS2Container() { close(); } +//----------------------------------------------------------------------------- +void ADIOS2Container::close() +{ + assert(_engine); + if (*_engine) + _engine->Close(); +} + +#endif \ No newline at end of file diff --git a/cpp/dolfinx/io/ADIOS2_utils.h b/cpp/dolfinx/io/ADIOS2_utils.h new file mode 100644 index 00000000000..2de9b1bf8f2 --- /dev/null +++ b/cpp/dolfinx/io/ADIOS2_utils.h @@ -0,0 +1,69 @@ +// Copyright (C) 2024 Abdullah Mujahid, Jørgen S. Dokken and Garth N. Wells +// +// This file is part of DOLFINX (https://www.fenicsproject.org) +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#pragma once + +#ifdef HAS_ADIOS2 + +#include +#include +#include +#include + +/// @file ADIOS2_utils.h +/// @brief Utils for ADIOS2 + +namespace dolfinx::io +{ + +/// ADIOS2-based writers/readers +class ADIOS2Container +{ +public: + /// @brief Create an ADIOS2-based engine writer/reader + /// @param[in] comm The MPI communicator + /// @param[in] filename Name of output file + /// @param[in] tag The ADIOS2 object name + /// @param[in] engine ADIOS2 engine type. See + /// https://adios2.readthedocs.io/en/latest/engines/engines.html. + /// @param[in] mode ADIOS2 mode, default is Write or Read + ADIOS2Container(MPI_Comm comm, std::string filename, + std::string tag, std::string engine = "BP5", + std::string mode = "write"); + + /// @brief Move constructor + ADIOS2Container(ADIOS2Container&& engine) = default; + + /// @brief Copy constructor + ADIOS2Container(const ADIOS2Container&) = delete; + + /// @brief Destructor + ~ADIOS2Container(); + + /// @brief Move assignment + ADIOS2Container& operator=(ADIOS2Container&& engine) = default; + + // Copy assignment + ADIOS2Container& operator=(const ADIOS2Container&) = delete; + + /// @brief Close the file + void close(); + + /// @brief Get the IO object + std::shared_ptr io() { return _io; } + + /// @brief Close the Engine object + std::shared_ptr engine() { return _engine; } + +protected: + std::shared_ptr _adios; + std::shared_ptr _io; + std::shared_ptr _engine; +}; + +} // namespace dolfinx::io + +#endif \ No newline at end of file diff --git a/cpp/dolfinx/io/CMakeLists.txt b/cpp/dolfinx/io/CMakeLists.txt index 9374d06c067..6e8c6fe59d0 100644 --- a/cpp/dolfinx/io/CMakeLists.txt +++ b/cpp/dolfinx/io/CMakeLists.txt @@ -1,5 +1,6 @@ set(HEADERS_io ${CMAKE_CURRENT_SOURCE_DIR}/dolfinx_io.h + ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2_utils.h ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.h ${CMAKE_CURRENT_SOURCE_DIR}/cells.h ${CMAKE_CURRENT_SOURCE_DIR}/checkpointing.h @@ -15,7 +16,8 @@ set(HEADERS_io target_sources( dolfinx - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.cpp + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2_utils.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/cells.cpp ${CMAKE_CURRENT_SOURCE_DIR}/checkpointing.cpp ${CMAKE_CURRENT_SOURCE_DIR}/HDF5Interface.cpp diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 7c7655405b2..8b8e07b197c 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -315,6 +315,21 @@ read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm); /// @endcond +void write_test(ADIOS2Container& container) +{ + auto io = container.io(); + auto engine = container.engine(); + + io->DefineAttribute("name", "Test write"); + adios2::Variable var_num + = io->DefineVariable("num"); + + std::uint64_t num = 100; + + engine->BeginStep(); + engine->Put(var_num, num); + engine->EndStep(); +} } // namespace dolfinx::io::checkpointing #endif diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 4fb351eb843..cd64d924b00 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -8,7 +8,7 @@ #ifdef HAS_ADIOS2 -#include "boost/variant.hpp" +#include "ADIOS2_utils.h" #include #include #include @@ -39,6 +39,11 @@ template dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm = MPI_COMM_WORLD); +/// @brief Test function to see python API for ADIOS2 wrapper +/// +/// @param[in] ADIOS2Container ADIOS2Container +void write_test(ADIOS2Container& container); + } // namespace dolfinx::io::checkpointing #endif diff --git a/cpp/dolfinx/io/dolfinx_io.h b/cpp/dolfinx/io/dolfinx_io.h index f8f499f049d..1bb24940950 100644 --- a/cpp/dolfinx/io/dolfinx_io.h +++ b/cpp/dolfinx/io/dolfinx_io.h @@ -9,6 +9,7 @@ namespace dolfinx::io // DOLFINx io interface +#include #include #include #include diff --git a/python/demo/demo_checkpointing.py b/python/demo/demo_checkpointing.py new file mode 100644 index 00000000000..d4d3d6090ba --- /dev/null +++ b/python/demo/demo_checkpointing.py @@ -0,0 +1,65 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: light +# format_version: '1.5' +# jupytext_version: 1.13.6 +# --- + +# # Checkpointing +# +# This demo is implemented in {download}`demo_checkpointing.py`. It +# illustrates checkpointing using ADIOS2: +# + +# + +# import importlib.util + +# if importlib.util.find_spec("adios2") is not None: +# import dolfinx + +# if not dolfinx.has_adios2: +# print("This demo requires DOLFINx to be compiled with ADIOS2 enabled.") +# exit(0) +# else: +# print("This demo requires ADIOS2.") +# exit(0) + +from mpi4py import MPI + +import numpy as np + +from dolfinx import io, mesh +# - + +# Note that it is important to first `from mpi4py import MPI` to +# ensure that MPI is correctly initialised. + +# We create a rectangular {py:class}`Mesh ` using +# {py:func}`create_rectangle `, and +# save it to a file `mesh.bp` + +# + +msh = mesh.create_rectangle( + comm=MPI.COMM_WORLD, + points=((0.0, 0.0), (1.0, 1.0)), + n=(16, 16), + cell_type=mesh.CellType.triangle, +) +# - + +# + +filename = "test.bp" +engine_type = "BP5" +tag = "test-write" +mode = "write" + +container = io.ADIOS2(msh.comm, filename, tag, engine_type, mode) +# - + +# + +io.write_test(container) +container.close() +# - \ No newline at end of file diff --git a/python/dolfinx/io/__init__.py b/python/dolfinx/io/__init__.py index 3a53be8d7eb..2b672aced08 100644 --- a/python/dolfinx/io/__init__.py +++ b/python/dolfinx/io/__init__.py @@ -7,7 +7,7 @@ from dolfinx import cpp as _cpp from dolfinx.io import gmshio -from dolfinx.io.utils import VTKFile, XDMFFile, distribute_entity_data +from dolfinx.io.utils import VTKFile, XDMFFile, distribute_entity_data, ADIOS2, write_test __all__ = ["gmshio", "distribute_entity_data", "VTKFile", "XDMFFile"] @@ -15,4 +15,12 @@ # FidesWriter and VTXWriter require ADIOS2 from dolfinx.io.utils import FidesMeshPolicy, FidesWriter, VTXMeshPolicy, VTXWriter - __all__ = [*__all__, "FidesWriter", "VTXWriter", "FidesMeshPolicy", "VTXMeshPolicy"] + __all__ = [ + *__all__, + "FidesWriter", + "VTXWriter", + "FidesMeshPolicy", + "VTXMeshPolicy", + "ADIOS2", + "write_test", + ] diff --git a/python/dolfinx/io/utils.py b/python/dolfinx/io/utils.py index a479f898998..5574a917d7c 100644 --- a/python/dolfinx/io/utils.py +++ b/python/dolfinx/io/utils.py @@ -23,7 +23,14 @@ from dolfinx.fem import Function from dolfinx.mesh import GhostMode, Mesh, MeshTags -__all__ = ["VTKFile", "XDMFFile", "cell_perm_gmsh", "cell_perm_vtk", "distribute_entity_data"] +__all__ = [ + "VTKFile", + "XDMFFile", + "cell_perm_gmsh", + "cell_perm_vtk", + "distribute_entity_data", + "ADIOS2", +] def _extract_cpp_objects(functions: typing.Union[Mesh, Function, tuple[Function], list[Function]]): @@ -281,6 +288,19 @@ def read_meshtags(self, mesh, name, xpath="/Xdmf/Domain"): return MeshTags(mt) +class ADIOS2(_cpp.io.ADIOS2): + def __enter__(self): + return self + + def __exit__(self, exception_type, exception_value, traceback): + self.close() + + +def write_test(container: ADIOS2) -> None: + """Write to a file using ADIOS2""" + return _cpp.io.write_test(container) + + def distribute_entity_data( mesh: Mesh, entity_dim: int, entities: npt.NDArray[np.int64], values: np.ndarray ) -> tuple[npt.NDArray[np.int64], np.ndarray]: diff --git a/python/dolfinx/wrappers/io.cpp b/python/dolfinx/wrappers/io.cpp index ed7ae36816b..32764236415 100644 --- a/python/dolfinx/wrappers/io.cpp +++ b/python/dolfinx/wrappers/io.cpp @@ -11,9 +11,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -236,6 +238,30 @@ void declare_data_types(nb::module_& m) void io(nb::module_& m) { +#ifdef HAS_ADIOS2 + // dolfinx::io::ADIOS2Container + nb::class_ ADIOS2(m, "ADIOS2"); + + ADIOS2 + .def( + "__init__", + [](dolfinx::io::ADIOS2Container* v, MPICommWrapper comm, + const std::string filename, std::string tag, + std::string engine = "BP5", std::string mode = "write") + { + new (v) dolfinx::io::ADIOS2Container(comm.get(), filename, tag, + engine, mode); + }, + nb::arg("comm"), nb::arg("filename"), nb::arg("tag"), + nb::arg("engine"), nb::arg("mode")) + .def("close", &dolfinx::io::ADIOS2Container::close); + + // dolfinx::io::checkpointing::write_test + m.def("write_test", &dolfinx::io::checkpointing::write_test, + nb::arg("container"), "Write test to file using ADIOS2"); + +#endif + // dolfinx::io::cell vtk cell type converter m.def("get_vtk_cell_type", &dolfinx::io::cells::get_vtk_cell_type, nb::arg("cell"), nb::arg("dim"), "Get VTK cell identifier"); From 3f6ce602d263c264a98b84df9bf89a8539299fea Mon Sep 17 00:00:00 2001 From: ampdes Date: Thu, 1 Aug 2024 21:47:05 +0200 Subject: [PATCH 38/60] isort and formatting --- python/demo/demo_checkpointing.py | 5 ++--- python/dolfinx/io/__init__.py | 11 +++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/python/demo/demo_checkpointing.py b/python/demo/demo_checkpointing.py index d4d3d6090ba..420a85339cc 100644 --- a/python/demo/demo_checkpointing.py +++ b/python/demo/demo_checkpointing.py @@ -29,9 +29,8 @@ from mpi4py import MPI -import numpy as np - from dolfinx import io, mesh + # - # Note that it is important to first `from mpi4py import MPI` to @@ -62,4 +61,4 @@ # + io.write_test(container) container.close() -# - \ No newline at end of file +# - diff --git a/python/dolfinx/io/__init__.py b/python/dolfinx/io/__init__.py index 2b672aced08..0777dfc6e72 100644 --- a/python/dolfinx/io/__init__.py +++ b/python/dolfinx/io/__init__.py @@ -7,13 +7,20 @@ from dolfinx import cpp as _cpp from dolfinx.io import gmshio -from dolfinx.io.utils import VTKFile, XDMFFile, distribute_entity_data, ADIOS2, write_test +from dolfinx.io.utils import VTKFile, XDMFFile, distribute_entity_data __all__ = ["gmshio", "distribute_entity_data", "VTKFile", "XDMFFile"] if _cpp.common.has_adios2: # FidesWriter and VTXWriter require ADIOS2 - from dolfinx.io.utils import FidesMeshPolicy, FidesWriter, VTXMeshPolicy, VTXWriter + from dolfinx.io.utils import ( + ADIOS2, + FidesMeshPolicy, + FidesWriter, + VTXMeshPolicy, + VTXWriter, + write_test, + ) __all__ = [ *__all__, From 816233907463e803ad6e90c9f0c21d38e64dd9a3 Mon Sep 17 00:00:00 2001 From: ampdes Date: Thu, 1 Aug 2024 22:06:44 +0200 Subject: [PATCH 39/60] Fix docs --- cpp/dolfinx/io/checkpointing.cpp | 1 + cpp/dolfinx/io/checkpointing.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 8b8e07b197c..793cde691da 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -315,6 +315,7 @@ read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm); /// @endcond +//----------------------------------------------------------------------------- void write_test(ADIOS2Container& container) { auto io = container.io(); diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index cd64d924b00..f673f5cd6cd 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -41,7 +41,7 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, /// @brief Test function to see python API for ADIOS2 wrapper /// -/// @param[in] ADIOS2Container ADIOS2Container +/// @param[in] container ADIOS2Container void write_test(ADIOS2Container& container); } // namespace dolfinx::io::checkpointing From b93cc2ec48b8c438dd99420a9eda7a315112f0e6 Mon Sep 17 00:00:00 2001 From: ampdes Date: Thu, 1 Aug 2024 22:33:02 +0200 Subject: [PATCH 40/60] Add missing has_adios2 --- python/demo/demo_checkpointing.py | 18 ++++++++-------- python/dolfinx/__init__.py | 5 ++++- python/dolfinx/io/utils.py | 35 ++++++++++++++++++------------- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/python/demo/demo_checkpointing.py b/python/demo/demo_checkpointing.py index 420a85339cc..379fc28581f 100644 --- a/python/demo/demo_checkpointing.py +++ b/python/demo/demo_checkpointing.py @@ -15,17 +15,17 @@ # # + -# import importlib.util +import importlib.util -# if importlib.util.find_spec("adios2") is not None: -# import dolfinx +if importlib.util.find_spec("adios2") is not None: + import dolfinx -# if not dolfinx.has_adios2: -# print("This demo requires DOLFINx to be compiled with ADIOS2 enabled.") -# exit(0) -# else: -# print("This demo requires ADIOS2.") -# exit(0) + if not dolfinx.has_adios2: + print("This demo requires DOLFINx to be compiled with ADIOS2 enabled.") + exit(0) +else: + print("This demo requires ADIOS2.") + exit(0) from mpi4py import MPI diff --git a/python/dolfinx/__init__.py b/python/dolfinx/__init__.py index dcdfa35aa3f..b442d99258d 100644 --- a/python/dolfinx/__init__.py +++ b/python/dolfinx/__init__.py @@ -30,10 +30,11 @@ from dolfinx.common import ( TimingType, git_commit_hash, + has_adios2, has_debug, has_kahip, - has_petsc, has_parmetis, + has_petsc, list_timings, timing, ) @@ -70,9 +71,11 @@ def get_include(user=False): "utils", "TimingType", "git_commit_hash", + "has_adios2", "has_debug", "has_kahip", "has_parmetis", + "has_petsc", "list_timings", "timing", ] diff --git a/python/dolfinx/io/utils.py b/python/dolfinx/io/utils.py index 5574a917d7c..6cd64abfc0b 100644 --- a/python/dolfinx/io/utils.py +++ b/python/dolfinx/io/utils.py @@ -29,7 +29,6 @@ "cell_perm_gmsh", "cell_perm_vtk", "distribute_entity_data", - "ADIOS2", ] @@ -45,7 +44,15 @@ def _extract_cpp_objects(functions: typing.Union[Mesh, Function, tuple[Function] if _cpp.common.has_adios2: from dolfinx.cpp.io import FidesMeshPolicy, VTXMeshPolicy # F401 - __all__ = [*__all__, "FidesWriter", "VTXWriter", "FidesMeshPolicy", "VTXMeshPolicy"] + __all__ = [ + *__all__, + "FidesWriter", + "VTXWriter", + "FidesMeshPolicy", + "VTXMeshPolicy", + "ADIOS2", + "write_test", + ] class VTXWriter: """Writer for VTX files, using ADIOS2 to create the files. @@ -193,6 +200,17 @@ def write(self, t: float): def close(self): self._cpp_object.close() + class ADIOS2(_cpp.io.ADIOS2): + def __enter__(self): + return self + + def __exit__(self, exception_type, exception_value, traceback): + self.close() + + def write_test(container: ADIOS2) -> None: + """Write to a file using ADIOS2""" + return _cpp.io.write_test(container) + class VTKFile(_cpp.io.VTKFile): """Interface to VTK files. @@ -288,19 +306,6 @@ def read_meshtags(self, mesh, name, xpath="/Xdmf/Domain"): return MeshTags(mt) -class ADIOS2(_cpp.io.ADIOS2): - def __enter__(self): - return self - - def __exit__(self, exception_type, exception_value, traceback): - self.close() - - -def write_test(container: ADIOS2) -> None: - """Write to a file using ADIOS2""" - return _cpp.io.write_test(container) - - def distribute_entity_data( mesh: Mesh, entity_dim: int, entities: npt.NDArray[np.int64], values: np.ndarray ) -> tuple[npt.NDArray[np.int64], np.ndarray]: From 73a26436589e86e16839c95d54bafc33bbf8d60e Mon Sep 17 00:00:00 2001 From: ampdes Date: Fri, 2 Aug 2024 00:00:14 +0200 Subject: [PATCH 41/60] Update docs --- python/doc/source/demos.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/doc/source/demos.rst b/python/doc/source/demos.rst index bbbcb889137..5326d08a6c1 100644 --- a/python/doc/source/demos.rst +++ b/python/doc/source/demos.rst @@ -65,6 +65,7 @@ Interpolation, IO and visualisation demos/demo_pyvista.md demos/demo_interpolation-io.md + demos/demo_checkpointing.md Advanced iterative solvers @@ -103,6 +104,7 @@ List of all demos demos/demo_static-condensation.md demos/demo_pyvista.md demos/demo_interpolation-io.md + demos/demo_checkpointing.md demos/demo_types.md demos/demo_lagrange_variants.md demos/demo_tnt-elements.md From c81221d3d215ec56ac734c1ff4c8d6e3e0683bda Mon Sep 17 00:00:00 2001 From: ampdes Date: Fri, 2 Aug 2024 13:20:05 +0200 Subject: [PATCH 42/60] Rename to ADIOS2Wrapper and engine_type --- cpp/demo/checkpointing/main.cpp | 2 +- cpp/dolfinx/io/ADIOS2_utils.cpp | 15 ++++++++------- cpp/dolfinx/io/ADIOS2_utils.h | 19 +++++++++---------- cpp/dolfinx/io/checkpointing.cpp | 6 +++--- cpp/dolfinx/io/checkpointing.h | 4 ++-- python/demo/demo_checkpointing.py | 6 +++--- python/dolfinx/io/utils.py | 4 ++-- python/dolfinx/wrappers/io.cpp | 16 ++++++++-------- 8 files changed, 36 insertions(+), 36 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 965df4ac13d..f911d21d1ff 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -78,7 +78,7 @@ int main(int argc, char* argv[]) } auto container - = ADIOS2Container(mesh->comm(), "test.bp", "test-write", "BP5", "write"); + = ADIOS2Wrapper(mesh->comm(), "test.bp", "test-write", "BP5", "write"); io::checkpointing::write_test(container); diff --git a/cpp/dolfinx/io/ADIOS2_utils.cpp b/cpp/dolfinx/io/ADIOS2_utils.cpp index 3d07371d13a..428dac770f9 100644 --- a/cpp/dolfinx/io/ADIOS2_utils.cpp +++ b/cpp/dolfinx/io/ADIOS2_utils.cpp @@ -21,20 +21,21 @@ std::map string_to_mode{ using namespace dolfinx::io; //----------------------------------------------------------------------------- -ADIOS2Container::ADIOS2Container(MPI_Comm comm, std::string filename, - std::string tag, std::string engine, - std::string mode) +ADIOS2Wrapper::ADIOS2Wrapper(MPI_Comm comm, std::string filename, + std::string tag, std::string engine_type, + std::string mode) : _adios(std::make_unique(comm)), _io(std::make_unique(_adios->DeclareIO(tag))) { - _io->SetEngine(engine); - _engine = std::make_unique(_io->Open(filename, string_to_mode[mode])); + _io->SetEngine(engine_type); + _engine = std::make_unique( + _io->Open(filename, string_to_mode[mode])); } //----------------------------------------------------------------------------- -ADIOS2Container::~ADIOS2Container() { close(); } +ADIOS2Wrapper::~ADIOS2Wrapper() { close(); } //----------------------------------------------------------------------------- -void ADIOS2Container::close() +void ADIOS2Wrapper::close() { assert(_engine); if (*_engine) diff --git a/cpp/dolfinx/io/ADIOS2_utils.h b/cpp/dolfinx/io/ADIOS2_utils.h index 2de9b1bf8f2..abdec425101 100644 --- a/cpp/dolfinx/io/ADIOS2_utils.h +++ b/cpp/dolfinx/io/ADIOS2_utils.h @@ -20,34 +20,33 @@ namespace dolfinx::io { /// ADIOS2-based writers/readers -class ADIOS2Container +class ADIOS2Wrapper { public: /// @brief Create an ADIOS2-based engine writer/reader /// @param[in] comm The MPI communicator /// @param[in] filename Name of output file /// @param[in] tag The ADIOS2 object name - /// @param[in] engine ADIOS2 engine type. See + /// @param[in] engine_type ADIOS2 engine type. See /// https://adios2.readthedocs.io/en/latest/engines/engines.html. /// @param[in] mode ADIOS2 mode, default is Write or Read - ADIOS2Container(MPI_Comm comm, std::string filename, - std::string tag, std::string engine = "BP5", - std::string mode = "write"); + ADIOS2Wrapper(MPI_Comm comm, std::string filename, std::string tag, + std::string engine_type = "BP5", std::string mode = "write"); /// @brief Move constructor - ADIOS2Container(ADIOS2Container&& engine) = default; + ADIOS2Wrapper(ADIOS2Wrapper&& ADIOS2) = default; /// @brief Copy constructor - ADIOS2Container(const ADIOS2Container&) = delete; + ADIOS2Wrapper(const ADIOS2Wrapper&) = delete; /// @brief Destructor - ~ADIOS2Container(); + ~ADIOS2Wrapper(); /// @brief Move assignment - ADIOS2Container& operator=(ADIOS2Container&& engine) = default; + ADIOS2Wrapper& operator=(ADIOS2Wrapper&& ADIOS2) = default; // Copy assignment - ADIOS2Container& operator=(const ADIOS2Container&) = delete; + ADIOS2Wrapper& operator=(const ADIOS2Wrapper&) = delete; /// @brief Close the file void close(); diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 793cde691da..7856d9e96c4 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -316,10 +316,10 @@ read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm); /// @endcond //----------------------------------------------------------------------------- -void write_test(ADIOS2Container& container) +void write_test(ADIOS2Wrapper& ADIOS2) { - auto io = container.io(); - auto engine = container.engine(); + auto io = ADIOS2.io(); + auto engine = ADIOS2.engine(); io->DefineAttribute("name", "Test write"); adios2::Variable var_num diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index f673f5cd6cd..f1a88f7040c 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -41,8 +41,8 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, /// @brief Test function to see python API for ADIOS2 wrapper /// -/// @param[in] container ADIOS2Container -void write_test(ADIOS2Container& container); +/// @param[in] ADIOS2 ADIOS2Wrapper +void write_test(ADIOS2Wrapper& ADIOS2); } // namespace dolfinx::io::checkpointing diff --git a/python/demo/demo_checkpointing.py b/python/demo/demo_checkpointing.py index 379fc28581f..f582ecd5422 100644 --- a/python/demo/demo_checkpointing.py +++ b/python/demo/demo_checkpointing.py @@ -55,10 +55,10 @@ tag = "test-write" mode = "write" -container = io.ADIOS2(msh.comm, filename, tag, engine_type, mode) +adios2 = io.ADIOS2(msh.comm, filename, tag, engine_type, mode) # - # + -io.write_test(container) -container.close() +io.write_test(adios2) +adios2.close() # - diff --git a/python/dolfinx/io/utils.py b/python/dolfinx/io/utils.py index 6cd64abfc0b..898fcc488cb 100644 --- a/python/dolfinx/io/utils.py +++ b/python/dolfinx/io/utils.py @@ -207,9 +207,9 @@ def __enter__(self): def __exit__(self, exception_type, exception_value, traceback): self.close() - def write_test(container: ADIOS2) -> None: + def write_test(ADIOS2: ADIOS2) -> None: """Write to a file using ADIOS2""" - return _cpp.io.write_test(container) + return _cpp.io.write_test(ADIOS2) class VTKFile(_cpp.io.VTKFile): diff --git a/python/dolfinx/wrappers/io.cpp b/python/dolfinx/wrappers/io.cpp index 32764236415..cf85809ffea 100644 --- a/python/dolfinx/wrappers/io.cpp +++ b/python/dolfinx/wrappers/io.cpp @@ -239,22 +239,22 @@ void declare_data_types(nb::module_& m) void io(nb::module_& m) { #ifdef HAS_ADIOS2 - // dolfinx::io::ADIOS2Container - nb::class_ ADIOS2(m, "ADIOS2"); + // dolfinx::io::ADIOS2Wrapper + nb::class_ ADIOS2(m, "ADIOS2"); ADIOS2 .def( "__init__", - [](dolfinx::io::ADIOS2Container* v, MPICommWrapper comm, + [](dolfinx::io::ADIOS2Wrapper* v, MPICommWrapper comm, const std::string filename, std::string tag, - std::string engine = "BP5", std::string mode = "write") + std::string engine_type = "BP5", std::string mode = "write") { - new (v) dolfinx::io::ADIOS2Container(comm.get(), filename, tag, - engine, mode); + new (v) dolfinx::io::ADIOS2Wrapper(comm.get(), filename, tag, + engine_type, mode); }, nb::arg("comm"), nb::arg("filename"), nb::arg("tag"), - nb::arg("engine"), nb::arg("mode")) - .def("close", &dolfinx::io::ADIOS2Container::close); + nb::arg("engine_type"), nb::arg("mode")) + .def("close", &dolfinx::io::ADIOS2Wrapper::close); // dolfinx::io::checkpointing::write_test m.def("write_test", &dolfinx::io::checkpointing::write_test, From 20708f55025b6d39128a71d98a59034e7fbeddbe Mon Sep 17 00:00:00 2001 From: ampdes Date: Fri, 2 Aug 2024 17:24:04 +0200 Subject: [PATCH 43/60] Add python API for write_mesh --- cpp/demo/checkpointing/main.cpp | 67 +++++++++++------- cpp/dolfinx/io/checkpointing.cpp | 112 ++++++++++++++++-------------- cpp/dolfinx/io/checkpointing.h | 11 ++- python/demo/demo_checkpointing.py | 22 ++++-- python/dolfinx/io/__init__.py | 2 + python/dolfinx/io/utils.py | 11 +++ python/dolfinx/wrappers/io.cpp | 15 +++- 7 files changed, 147 insertions(+), 93 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index f911d21d1ff..729bd297ec8 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -29,52 +29,65 @@ int main(int argc, char* argv[]) mesh::CellType::quadrilateral, part)); // Set up ADIOS2 IO and Engine - adios2::ADIOS adios(mesh->comm()); - adios2::IO io = adios.DeclareIO("mesh-write"); - io.SetEngine("BP5"); - adios2::Engine engine = io.Open("mesh.bp", adios2::Mode::Write); + auto adios + = ADIOS2Wrapper(mesh->comm(), "mesh.bp", "mesh-write", "BP5", "write"); + + auto io = adios.io(); // TODO: Need to move this inside the checkpointing module - io.DefineAttribute("version", DOLFINX_VERSION_STRING); - io.DefineAttribute("git_hash", DOLFINX_VERSION_GIT); + io->DefineAttribute("version", DOLFINX_VERSION_STRING); + io->DefineAttribute("git_hash", DOLFINX_VERSION_GIT); + + // io::checkpointing::write_mesh(io, engine, *mesh); + io::checkpointing::write_mesh(adios, *mesh); + + // adios.close(); - io::checkpointing::write_mesh(io, engine, *mesh); + // ---------------------------------------------------------------------- + auto adios_read + = ADIOS2Wrapper(mesh->comm(), "mesh.bp", "mesh-read", "BP5", "read"); - engine.Close(); + auto io_read = adios_read.io(); + auto engine_read = adios_read.engine(); - adios2::IO io_read = adios.DeclareIO("mesh-read"); - io_read.SetEngine("BP5"); + // Following throws an error. engine_read->BeginStep() is needed to + // read the VariableType, but then EndStep() fails with message + // EndStep() called without a successful BeginStep() - adios2::Engine reader = io_read.Open("mesh.bp", adios2::Mode::Read); + // engine_read->BeginStep(); + std::string floating_point = io->VariableType("x"); + // engine_read->EndStep(); // TODO: move type deduction inside checkpointing - if ("float" == io.VariableType("x")) + if ("float" == floating_point) { using T = float; mesh::Mesh mesh_read - = io::checkpointing::read_mesh(io_read, reader, mesh->comm()); - reader.Close(); + = io::checkpointing::read_mesh(adios_read, mesh->comm()); + adios_read.close(); - adios2::IO io_write = adios.DeclareIO("mesh-rewrite"); - io_write.SetEngine("BP5"); + auto adios_write = ADIOS2Wrapper(mesh->comm(), "mesh2.bp", "mesh-rewrite", + "BP5", "write"); - adios2::Engine writer = io_write.Open("mesh2.bp", adios2::Mode::Write); - io::checkpointing::write_mesh(io_write, writer, mesh_read); - writer.Close(); + auto io_write = adios_write.io(); + + io::checkpointing::write_mesh(adios_write, mesh_read); + adios_write.close(); } - else if ("double" == io.VariableType("x")) + else if ("double" == floating_point) { using T = double; mesh::Mesh mesh_read - = io::checkpointing::read_mesh(io_read, reader, mesh->comm()); - reader.Close(); + = io::checkpointing::read_mesh(adios_read, mesh->comm()); + adios_read.close(); + + auto adios_write = ADIOS2Wrapper(mesh->comm(), "mesh2.bp", "mesh-rewrite", + "BP5", "write"); - adios2::IO io_write = adios.DeclareIO("mesh-rewrite"); - io_write.SetEngine("BP5"); + auto io_write = adios_write.io(); - adios2::Engine writer = io_write.Open("mesh2.bp", adios2::Mode::Write); - io::checkpointing::write_mesh(io_write, writer, mesh_read); - writer.Close(); + io::checkpointing::write_mesh(adios_write, mesh_read); + adios_write.close(); } auto container diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 7856d9e96c4..75ce5927dba 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -36,8 +36,7 @@ namespace dolfinx::io::checkpointing { //----------------------------------------------------------------------------- template -void write_mesh(adios2::IO& io, adios2::Engine& engine, - dolfinx::mesh::Mesh& mesh) +void write_mesh(ADIOS2Wrapper& ADIOS2, dolfinx::mesh::Mesh& mesh) { const mesh::Geometry& geometry = mesh.geometry(); @@ -109,72 +108,77 @@ void write_mesh(adios2::IO& io, adios2::Engine& engine, // ADIOS2 write attributes and variables { - io.DefineAttribute("name", mesh.name); - io.DefineAttribute("dim", dim); - io.DefineAttribute("cell_type", cell_type); - io.DefineAttribute("degree", degree); - io.DefineAttribute("lagrange_variant", lagrange_variant); + auto io = ADIOS2.io(); + auto engine = ADIOS2.engine(); + + io->DefineAttribute("name", mesh.name); + io->DefineAttribute("dim", dim); + io->DefineAttribute("cell_type", cell_type); + io->DefineAttribute("degree", degree); + io->DefineAttribute("lagrange_variant", lagrange_variant); adios2::Variable var_num_vertices - = io.DefineVariable("num_vertices"); + = io->DefineVariable("num_vertices"); adios2::Variable var_num_cells - = io.DefineVariable("num_cells"); + = io->DefineVariable("num_cells"); adios2::Variable var_num_dofs_per_cell - = io.DefineVariable("num_dofs_per_cell"); + = io->DefineVariable("num_dofs_per_cell"); adios2::Variable var_input_global_indices - = io.DefineVariable( + = io->DefineVariable( "input_global_indices", {num_vertices_global}, {offset}, {num_vertices_local}, adios2::ConstantDims); adios2::Variable var_x - = io.DefineVariable("x", {num_vertices_global, 3}, {offset, 0}, - {num_vertices_local, 3}, adios2::ConstantDims); + = io->DefineVariable("x", {num_vertices_global, 3}, {offset, 0}, + {num_vertices_local, 3}, adios2::ConstantDims); adios2::Variable var_topology_array - = io.DefineVariable( + = io->DefineVariable( "topology_array", {num_cells_global * num_dofs_per_cell}, {cell_offset * num_dofs_per_cell}, {num_cells_local * num_dofs_per_cell}, adios2::ConstantDims); adios2::Variable var_topology_offsets - = io.DefineVariable( + = io->DefineVariable( "topology_offsets", {num_cells_global + 1}, {cell_offset}, {num_cells_local + 1}, adios2::ConstantDims); - engine.BeginStep(); - engine.Put(var_num_vertices, num_vertices_global); - engine.Put(var_num_cells, num_cells_global); - engine.Put(var_num_dofs_per_cell, num_dofs_per_cell); - engine.Put(var_input_global_indices, input_global_indices_span.data()); - engine.Put(var_x, mesh_x.subspan(0, num_vertices_local * 3).data()); - engine.Put(var_topology_array, array_global.data()); - engine.Put(var_topology_offsets, offsets_global.data()); - engine.EndStep(); + engine->BeginStep(); + engine->Put(var_num_vertices, num_vertices_global); + engine->Put(var_num_cells, num_cells_global); + engine->Put(var_num_dofs_per_cell, num_dofs_per_cell); + engine->Put(var_input_global_indices, input_global_indices_span.data()); + engine->Put(var_x, mesh_x.subspan(0, num_vertices_local * 3).data()); + engine->Put(var_topology_array, array_global.data()); + engine->Put(var_topology_offsets, offsets_global.data()); + engine->EndStep(); } } //----------------------------------------------------------------------------- /// @cond -template void write_mesh(adios2::IO& io, adios2::Engine& engine, +template void write_mesh(ADIOS2Wrapper& ADIOS2, dolfinx::mesh::Mesh& mesh); -template void write_mesh(adios2::IO& io, adios2::Engine& engine, +template void write_mesh(ADIOS2Wrapper& ADIOS2, dolfinx::mesh::Mesh& mesh); /// @endcond //----------------------------------------------------------------------------- template -dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, - MPI_Comm comm) +dolfinx::mesh::Mesh read_mesh(ADIOS2Wrapper& ADIOS2, MPI_Comm comm) { int rank, size; MPI_Comm_rank(comm, &rank); MPI_Comm_size(comm, &size); - engine.BeginStep(); + auto io = ADIOS2.io(); + auto engine = ADIOS2.engine(); + + engine->BeginStep(); // Attributes std::string name; @@ -185,15 +189,15 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, // Read attributes { adios2::Attribute var_name - = io.InquireAttribute("name"); + = io->InquireAttribute("name"); adios2::Attribute var_dim - = io.InquireAttribute("dim"); + = io->InquireAttribute("dim"); adios2::Attribute var_cell_type - = io.InquireAttribute("cell_type"); + = io->InquireAttribute("cell_type"); adios2::Attribute var_degree - = io.InquireAttribute("degree"); + = io->InquireAttribute("degree"); adios2::Attribute var_variant - = io.InquireAttribute("lagrange_variant"); + = io->InquireAttribute("lagrange_variant"); name = var_name.Data()[0]; dim = var_dim.Data()[0]; @@ -210,15 +214,15 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, // Read scalar variables { adios2::Variable var_num_vertices - = io.InquireVariable("num_vertices"); + = io->InquireVariable("num_vertices"); adios2::Variable var_num_cells - = io.InquireVariable("num_cells"); + = io->InquireVariable("num_cells"); adios2::Variable var_num_dofs_per_cell - = io.InquireVariable("num_dofs_per_cell"); + = io->InquireVariable("num_dofs_per_cell"); - engine.Get(var_num_vertices, num_vertices_global); - engine.Get(var_num_cells, num_cells_global); - engine.Get(var_num_dofs_per_cell, num_dofs_per_cell); + engine->Get(var_num_vertices, num_vertices_global); + engine->Get(var_num_cells, num_cells_global); + engine->Get(var_num_dofs_per_cell, num_dofs_per_cell); } // Compute local sizes, offsets @@ -243,44 +247,44 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, { adios2::Variable var_input_global_indices - = io.InquireVariable("input_global_indices"); + = io->InquireVariable("input_global_indices"); - adios2::Variable var_x = io.InquireVariable("x"); + adios2::Variable var_x = io->InquireVariable("x"); adios2::Variable var_topology_array - = io.InquireVariable("topology_array"); + = io->InquireVariable("topology_array"); adios2::Variable var_topology_offsets - = io.InquireVariable("topology_offsets"); + = io->InquireVariable("topology_offsets"); if (var_input_global_indices) { var_input_global_indices.SetSelection( {{local_range[0]}, {num_vertices_local}}); - engine.Get(var_input_global_indices, input_global_indices.data(), - adios2::Mode::Deferred); + engine->Get(var_input_global_indices, input_global_indices.data(), + adios2::Mode::Deferred); } if (var_x) { var_x.SetSelection({{local_range[0], 0}, {num_vertices_local, 3}}); - engine.Get(var_x, x.data(), adios2::Mode::Deferred); + engine->Get(var_x, x.data(), adios2::Mode::Deferred); } if (var_topology_array) { var_topology_array.SetSelection({{cell_range[0] * num_dofs_per_cell}, {cell_range[1] * num_dofs_per_cell}}); - engine.Get(var_topology_array, array.data(), adios2::Mode::Deferred); + engine->Get(var_topology_array, array.data(), adios2::Mode::Deferred); } if (var_topology_offsets) { var_topology_offsets.SetSelection({{cell_range[0]}, {cell_range[1] + 1}}); - engine.Get(var_topology_offsets, offsets.data(), adios2::Mode::Deferred); + engine->Get(var_topology_offsets, offsets.data(), adios2::Mode::Deferred); } - engine.EndStep(); + engine->EndStep(); std::int32_t cell_offset = offsets[0]; for (auto offset = offsets.begin(); offset != offsets.end(); ++offset) @@ -307,11 +311,11 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, //----------------------------------------------------------------------------- /// @cond -template dolfinx::mesh::Mesh -read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm); +template dolfinx::mesh::Mesh read_mesh(ADIOS2Wrapper& ADIOS2, + MPI_Comm comm); -template dolfinx::mesh::Mesh -read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm); +template dolfinx::mesh::Mesh read_mesh(ADIOS2Wrapper& ADIOS2, + MPI_Comm comm); /// @endcond diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index f1a88f7040c..0d0d66afbfc 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -22,21 +22,18 @@ namespace dolfinx::io::checkpointing /// @brief Write mesh to a file. /// -/// @param[in] io ADIOS2 IO -/// @param[in] engine ADIOS2 Engine +/// @param[in] ADIOS2 ADIOS2Wrapper /// @param[in] mesh Mesh of type float or double to write to the file template -void write_mesh(adios2::IO& io, adios2::Engine& engine, - dolfinx::mesh::Mesh& mesh); +void write_mesh(ADIOS2Wrapper& ADIOS2, dolfinx::mesh::Mesh& mesh); /// @brief Read mesh from a file. /// -/// @param[in] io ADIOS2 IO -/// @param[in] engine ADIOS2 Engine +/// @param[in] ADIOS2 ADIOS2Wrapper /// @param[in] comm comm /// @return mesh reconstructed from the data template -dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, +dolfinx::mesh::Mesh read_mesh(ADIOS2Wrapper& ADIOS2, MPI_Comm comm = MPI_COMM_WORLD); /// @brief Test function to see python API for ADIOS2 wrapper diff --git a/python/demo/demo_checkpointing.py b/python/demo/demo_checkpointing.py index f582ecd5422..73b938df477 100644 --- a/python/demo/demo_checkpointing.py +++ b/python/demo/demo_checkpointing.py @@ -44,21 +44,35 @@ msh = mesh.create_rectangle( comm=MPI.COMM_WORLD, points=((0.0, 0.0), (1.0, 1.0)), - n=(16, 16), + n=(4, 4), cell_type=mesh.CellType.triangle, ) # - # + -filename = "test.bp" +filename = "mesh.bp" engine_type = "BP5" -tag = "test-write" +tag = "mesh-write" mode = "write" adios2 = io.ADIOS2(msh.comm, filename, tag, engine_type, mode) # - # + -io.write_test(adios2) +io.write_mesh(adios2, msh) adios2.close() # - + +# + +filename = "test.bp" +engine_type = "BP5" +tag = "test-write" +mode = "write" + +adios2_test = io.ADIOS2(msh.comm, filename, tag, engine_type, mode) +# - + +# + +io.write_test(adios2_test) +adios2_test.close() +# - diff --git a/python/dolfinx/io/__init__.py b/python/dolfinx/io/__init__.py index 0777dfc6e72..d9876e5928c 100644 --- a/python/dolfinx/io/__init__.py +++ b/python/dolfinx/io/__init__.py @@ -19,6 +19,7 @@ FidesWriter, VTXMeshPolicy, VTXWriter, + write_mesh, write_test, ) @@ -29,5 +30,6 @@ "FidesMeshPolicy", "VTXMeshPolicy", "ADIOS2", + "write_mesh", "write_test", ] diff --git a/python/dolfinx/io/utils.py b/python/dolfinx/io/utils.py index 898fcc488cb..4f6b8a2d5e1 100644 --- a/python/dolfinx/io/utils.py +++ b/python/dolfinx/io/utils.py @@ -51,6 +51,7 @@ def _extract_cpp_objects(functions: typing.Union[Mesh, Function, tuple[Function] "FidesMeshPolicy", "VTXMeshPolicy", "ADIOS2", + "write_mesh", "write_test", ] @@ -207,6 +208,16 @@ def __enter__(self): def __exit__(self, exception_type, exception_value, traceback): self.close() + def write_mesh(ADIOS2: ADIOS2, mesh: Mesh) -> None: + """Write mesh to a file using ADIOS2""" + dtype = mesh.geometry.x.dtype # type: ignore + if np.issubdtype(dtype, np.float32): + _writer = _cpp.io.write_mesh_float32 + elif np.issubdtype(dtype, np.float64): + _writer = _cpp.io.write_mesh_float64 + + return _writer(ADIOS2, mesh._cpp_object) + def write_test(ADIOS2: ADIOS2) -> None: """Write to a file using ADIOS2""" return _cpp.io.write_test(ADIOS2) diff --git a/python/dolfinx/wrappers/io.cpp b/python/dolfinx/wrappers/io.cpp index cf85809ffea..7401ac6fb85 100644 --- a/python/dolfinx/wrappers/io.cpp +++ b/python/dolfinx/wrappers/io.cpp @@ -236,6 +236,16 @@ void declare_data_types(nb::module_& m) } // namespace +template +void declare_write_mesh(nb::module_& m, std::string type) +{ + // dolfinx::io::checkpointing::write_test + std::string pyfunction_write_mesh_name = std::string("write_mesh_") + type; + m.def(pyfunction_write_mesh_name.c_str(), + &dolfinx::io::checkpointing::write_mesh, nb::arg("adios2"), + nb::arg("mesh"), "Write test to file using ADIOS2"); +} + void io(nb::module_& m) { #ifdef HAS_ADIOS2 @@ -258,7 +268,10 @@ void io(nb::module_& m) // dolfinx::io::checkpointing::write_test m.def("write_test", &dolfinx::io::checkpointing::write_test, - nb::arg("container"), "Write test to file using ADIOS2"); + nb::arg("adios2"), "Write test to file using ADIOS2"); + + declare_write_mesh(m, "float32"); + declare_write_mesh(m, "float64"); #endif From 5f7d50507316c22a722f8703fa404ba5d294dc57 Mon Sep 17 00:00:00 2001 From: ampdes Date: Fri, 2 Aug 2024 19:22:47 +0200 Subject: [PATCH 44/60] Fix includes and docs --- cpp/dolfinx/io/ADIOS2_utils.h | 2 +- python/dolfinx/wrappers/io.cpp | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cpp/dolfinx/io/ADIOS2_utils.h b/cpp/dolfinx/io/ADIOS2_utils.h index abdec425101..0ef73a1562f 100644 --- a/cpp/dolfinx/io/ADIOS2_utils.h +++ b/cpp/dolfinx/io/ADIOS2_utils.h @@ -65,4 +65,4 @@ class ADIOS2Wrapper } // namespace dolfinx::io -#endif \ No newline at end of file +#endif diff --git a/python/dolfinx/wrappers/io.cpp b/python/dolfinx/wrappers/io.cpp index 7401ac6fb85..025922d91da 100644 --- a/python/dolfinx/wrappers/io.cpp +++ b/python/dolfinx/wrappers/io.cpp @@ -234,17 +234,19 @@ void declare_data_types(nb::module_& m) nb::arg("values").noconvert()); } -} // namespace - +#ifdef HAS_ADIOS2 template void declare_write_mesh(nb::module_& m, std::string type) { - // dolfinx::io::checkpointing::write_test + // dolfinx::io::checkpointing::write_mesh std::string pyfunction_write_mesh_name = std::string("write_mesh_") + type; m.def(pyfunction_write_mesh_name.c_str(), &dolfinx::io::checkpointing::write_mesh, nb::arg("adios2"), nb::arg("mesh"), "Write test to file using ADIOS2"); } +#endif + +} // namespace void io(nb::module_& m) { From 55b266c456da5a6e3d643e0388aae5b93ff7c30c Mon Sep 17 00:00:00 2001 From: ampdes Date: Sat, 3 Aug 2024 15:17:30 +0200 Subject: [PATCH 45/60] Add read_mesh * Read mesh of variant float or double * Add python API for read_mesh * Remove write_test --- cpp/demo/checkpointing/main.cpp | 107 ++++++++++++++++++------------ cpp/dolfinx/io/checkpointing.cpp | 30 ++++++--- cpp/dolfinx/io/checkpointing.h | 15 ++++- python/demo/demo_checkpointing.py | 20 ++++-- python/dolfinx/io/__init__.py | 4 +- python/dolfinx/io/utils.py | 10 +-- python/dolfinx/wrappers/io.cpp | 17 +++-- 7 files changed, 134 insertions(+), 69 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 729bd297ec8..be745b94877 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -13,6 +13,8 @@ #include #include #include +#include +#include using namespace dolfinx; using namespace dolfinx::io; @@ -41,59 +43,80 @@ int main(int argc, char* argv[]) // io::checkpointing::write_mesh(io, engine, *mesh); io::checkpointing::write_mesh(adios, *mesh); - // adios.close(); + adios.close(); // ---------------------------------------------------------------------- - auto adios_read + auto adios_query = ADIOS2Wrapper(mesh->comm(), "mesh.bp", "mesh-read", "BP5", "read"); - auto io_read = adios_read.io(); - auto engine_read = adios_read.engine(); - - // Following throws an error. engine_read->BeginStep() is needed to - // read the VariableType, but then EndStep() fails with message - // EndStep() called without a successful BeginStep() - - // engine_read->BeginStep(); - std::string floating_point = io->VariableType("x"); - // engine_read->EndStep(); - - // TODO: move type deduction inside checkpointing - if ("float" == floating_point) - { - using T = float; - mesh::Mesh mesh_read - = io::checkpointing::read_mesh(adios_read, mesh->comm()); - adios_read.close(); - - auto adios_write = ADIOS2Wrapper(mesh->comm(), "mesh2.bp", "mesh-rewrite", - "BP5", "write"); + auto adios_read + = ADIOS2Wrapper(mesh->comm(), "mesh.bp", "mesh-read", "BP5", "read"); - auto io_write = adios_write.io(); + // Can't use the same engine to query as well as read + // since, in that case BeginStep and EndStep will be called twice + // on a dataset written with a single Step + auto mesh_read_variant = io::checkpointing::read_mesh_variant( + adios_query, adios_read, mesh->comm()); - io::checkpointing::write_mesh(adios_write, mesh_read); - adios_write.close(); - } - else if ("double" == floating_point) - { - using T = double; - mesh::Mesh mesh_read - = io::checkpointing::read_mesh(adios_read, mesh->comm()); - adios_read.close(); + // We cannot resolve easily the variant + // Hence the user can query the type and + // call the correct read_mesh + auto mesh_read = std::get<0>(mesh_read_variant); + // auto mesh_read = std::get>(mesh_read_variant); - auto adios_write = ADIOS2Wrapper(mesh->comm(), "mesh2.bp", "mesh-rewrite", - "BP5", "write"); + auto adios_write + = ADIOS2Wrapper(mesh->comm(), "mesh2.bp", "mesh-rewrite", "BP5", "write"); - auto io_write = adios_write.io(); + io::checkpointing::write_mesh(adios_write, mesh_read); - io::checkpointing::write_mesh(adios_write, mesh_read); - adios_write.close(); - } + // auto io_query = adios_query.io(); + // auto engine_query = adios_query.engine(); - auto container - = ADIOS2Wrapper(mesh->comm(), "test.bp", "test-write", "BP5", "write"); + // Following throws an error. engine_read->BeginStep() is needed to + // read the VariableType, but then EndStep() fails with message + // EndStep() called without a successful BeginStep() - io::checkpointing::write_test(container); + // engine_query->BeginStep(); + // std::string floating_point = io_query->VariableType("x"); + // engine_query->EndStep(); + + // // std::string floating_point = + // dolfinx::io::checkpointing::query_type(adios_read); std::cout << + // floating_point; + + // // TODO: move type deduction inside checkpointing + // if ("float" == floating_point) + // { + // using T = float; + // mesh::Mesh mesh_read + // = io::checkpointing::read_mesh(adios_read, mesh->comm()); + // adios_read.close(); + + // auto adios_write = ADIOS2Wrapper(mesh->comm(), "mesh2.bp", + // "mesh-rewrite", + // "BP5", "write"); + + // auto io_write = adios_write.io(); + + // io::checkpointing::write_mesh(adios_write, mesh_read); + // adios_write.close(); + // } + // else if ("double" == floating_point) + // { + // using T = double; + // mesh::Mesh mesh_read + // = io::checkpointing::read_mesh(adios_read, mesh->comm()); + // adios_read.close(); + + // auto adios_write = ADIOS2Wrapper(mesh->comm(), "mesh2.bp", + // "mesh-rewrite", + // "BP5", "write"); + + // auto io_write = adios_write.io(); + + // io::checkpointing::write_mesh(adios_write, mesh_read); + // adios_write.close(); + // } MPI_Finalize(); return 0; diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 75ce5927dba..185f87c895e 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -320,21 +320,35 @@ template dolfinx::mesh::Mesh read_mesh(ADIOS2Wrapper& ADIOS2, /// @endcond //----------------------------------------------------------------------------- -void write_test(ADIOS2Wrapper& ADIOS2) +std::string query_type(ADIOS2Wrapper& ADIOS2) { auto io = ADIOS2.io(); auto engine = ADIOS2.engine(); + engine->BeginStep(); + std::string floating_point = io->VariableType("x"); + engine->EndStep(); - io->DefineAttribute("name", "Test write"); - adios2::Variable var_num - = io->DefineVariable("num"); + return floating_point; +} - std::uint64_t num = 100; +//----------------------------------------------------------------------------- +std::variant, dolfinx::mesh::Mesh> +read_mesh_variant(ADIOS2Wrapper& _ADIOS2, ADIOS2Wrapper& ADIOS2, MPI_Comm comm) +{ + std::string floating_point = query_type(_ADIOS2); - engine->BeginStep(); - engine->Put(var_num, num); - engine->EndStep(); + if (floating_point == "float") + { + dolfinx::mesh::Mesh mesh = read_mesh(ADIOS2, comm); + return mesh; + } + else // floating_point == "double" + { + dolfinx::mesh::Mesh mesh = read_mesh(ADIOS2, comm); + return mesh; + } } + } // namespace dolfinx::io::checkpointing #endif diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 0d0d66afbfc..ed785b33ef3 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -36,10 +36,21 @@ template dolfinx::mesh::Mesh read_mesh(ADIOS2Wrapper& ADIOS2, MPI_Comm comm = MPI_COMM_WORLD); -/// @brief Test function to see python API for ADIOS2 wrapper +/// @brief Read mesh from a file. +/// +/// @param[in] _ADIOS2 ADIOS2Wrapper +/// @param[in] ADIOS2 ADIOS2Wrapper +/// @param[in] comm comm +/// @return mesh reconstructed from the data +std::variant, dolfinx::mesh::Mesh> +read_mesh_variant(ADIOS2Wrapper& _ADIOS2, ADIOS2Wrapper& ADIOS2, + MPI_Comm comm = MPI_COMM_WORLD); + +/// @brief Query floating_point type stored in the file /// /// @param[in] ADIOS2 ADIOS2Wrapper -void write_test(ADIOS2Wrapper& ADIOS2); +/// @return type float or double as a string +std::string query_type(ADIOS2Wrapper& ADIOS2); } // namespace dolfinx::io::checkpointing diff --git a/python/demo/demo_checkpointing.py b/python/demo/demo_checkpointing.py index 73b938df477..835fd51985e 100644 --- a/python/demo/demo_checkpointing.py +++ b/python/demo/demo_checkpointing.py @@ -63,16 +63,22 @@ adios2.close() # - -# + -filename = "test.bp" +filename = "mesh.bp" engine_type = "BP5" -tag = "test-write" -mode = "write" +tag = "mesh-read" +mode = "read" -adios2_test = io.ADIOS2(msh.comm, filename, tag, engine_type, mode) +adios2_query = io.ADIOS2(msh.comm, filename, tag, engine_type, mode) +adios2_read = io.ADIOS2(msh.comm, filename, tag, engine_type, mode) +# - + +# + +msh_read = io.read_mesh(adios2_query, adios2_read, msh.comm) # - # + -io.write_test(adios2_test) -adios2_test.close() +print(type(msh_read)) +print(msh_read.name) +print(msh_read.geometry.x.dtype) +print(msh_read.geometry.x) # - diff --git a/python/dolfinx/io/__init__.py b/python/dolfinx/io/__init__.py index d9876e5928c..4cd6420b68e 100644 --- a/python/dolfinx/io/__init__.py +++ b/python/dolfinx/io/__init__.py @@ -19,8 +19,8 @@ FidesWriter, VTXMeshPolicy, VTXWriter, + read_mesh, write_mesh, - write_test, ) __all__ = [ @@ -31,5 +31,5 @@ "VTXMeshPolicy", "ADIOS2", "write_mesh", - "write_test", + "read_mesh", ] diff --git a/python/dolfinx/io/utils.py b/python/dolfinx/io/utils.py index 4f6b8a2d5e1..bc762d6222c 100644 --- a/python/dolfinx/io/utils.py +++ b/python/dolfinx/io/utils.py @@ -51,8 +51,8 @@ def _extract_cpp_objects(functions: typing.Union[Mesh, Function, tuple[Function] "FidesMeshPolicy", "VTXMeshPolicy", "ADIOS2", + "read_mesh", "write_mesh", - "write_test", ] class VTXWriter: @@ -218,9 +218,11 @@ def write_mesh(ADIOS2: ADIOS2, mesh: Mesh) -> None: return _writer(ADIOS2, mesh._cpp_object) - def write_test(ADIOS2: ADIOS2) -> None: - """Write to a file using ADIOS2""" - return _cpp.io.write_test(ADIOS2) + def read_mesh(_ADIOS2: ADIOS2, ADIOS2: ADIOS2, comm: _MPI.Comm) -> Mesh: + """Read mesh from a file using ADIOS2""" + msh = _cpp.io.read_mesh(_ADIOS2, ADIOS2, comm) + + return msh class VTKFile(_cpp.io.VTKFile): diff --git a/python/dolfinx/wrappers/io.cpp b/python/dolfinx/wrappers/io.cpp index 025922d91da..6af675afbe8 100644 --- a/python/dolfinx/wrappers/io.cpp +++ b/python/dolfinx/wrappers/io.cpp @@ -242,8 +242,9 @@ void declare_write_mesh(nb::module_& m, std::string type) std::string pyfunction_write_mesh_name = std::string("write_mesh_") + type; m.def(pyfunction_write_mesh_name.c_str(), &dolfinx::io::checkpointing::write_mesh, nb::arg("adios2"), - nb::arg("mesh"), "Write test to file using ADIOS2"); + nb::arg("mesh"), "Write mesh to file using ADIOS2"); } + #endif } // namespace @@ -268,9 +269,17 @@ void io(nb::module_& m) nb::arg("engine_type"), nb::arg("mode")) .def("close", &dolfinx::io::ADIOS2Wrapper::close); - // dolfinx::io::checkpointing::write_test - m.def("write_test", &dolfinx::io::checkpointing::write_test, - nb::arg("adios2"), "Write test to file using ADIOS2"); + // dolfinx::io::checkpointing::read_mesh_variant + m.def( + "read_mesh", + [](dolfinx::io::ADIOS2Wrapper& _ADIOS2, + dolfinx::io::ADIOS2Wrapper& ADIOS2, MPICommWrapper comm) + { + return dolfinx::io::checkpointing::read_mesh_variant(_ADIOS2, ADIOS2, + comm.get()); + }, + nb::arg("_adios2"), nb::arg("adios2"), nb::arg("comm"), + "Read mesh from file using ADIOS2"); declare_write_mesh(m, "float32"); declare_write_mesh(m, "float64"); From 1c548dbbe7f9b1e0c0d900b406c59ff77e5e5b2b Mon Sep 17 00:00:00 2001 From: ampdes Date: Tue, 6 Aug 2024 22:14:12 +0200 Subject: [PATCH 46/60] Update from review --- cpp/demo/checkpointing/main.cpp | 89 +++++++++++------ cpp/dolfinx/io/ADIOS2_utils.cpp | 6 +- cpp/dolfinx/io/ADIOS2_utils.h | 2 +- cpp/dolfinx/io/checkpointing.cpp | 154 +++++++++++++++--------------- cpp/dolfinx/io/checkpointing.h | 31 +++--- python/demo/demo_checkpointing.py | 35 +++---- 6 files changed, 176 insertions(+), 141 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index be745b94877..62165787ec7 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -24,50 +24,83 @@ int main(int argc, char* argv[]) dolfinx::init_logging(argc, argv); MPI_Init(&argc, &argv); + // { + // int i=0; + // while (i == 0) + // sleep(5); + // } + // Create mesh and function space auto part = mesh::create_cell_partitioner(mesh::GhostMode::shared_facet); auto mesh = std::make_shared>(mesh::create_rectangle( MPI_COMM_WORLD, {{{0.0, 0.0}, {1.0, 1.0}}}, {4, 4}, mesh::CellType::quadrilateral, part)); - // Set up ADIOS2 IO and Engine - auto adios - = ADIOS2Wrapper(mesh->comm(), "mesh.bp", "mesh-write", "BP5", "write"); + // // Set up ADIOS2 IO and Engine + // auto adios + // = ADIOS2Wrapper(mesh->comm(), "mesh.bp", "mesh-write", "BP5", "write"); - auto io = adios.io(); + // auto io = adios.io(); + + // Set up ADIOS2 IO and Engine + adios2::ADIOS adios(mesh->comm()); + adios2::IO io = adios.DeclareIO("mesh-write"); + io.SetEngine("BP5"); + adios2::Engine engine = io.Open("mesh.bp", adios2::Mode::Write); // TODO: Need to move this inside the checkpointing module - io->DefineAttribute("version", DOLFINX_VERSION_STRING); - io->DefineAttribute("git_hash", DOLFINX_VERSION_GIT); + // io.DefineAttribute("version", DOLFINX_VERSION_STRING); + // io.DefineAttribute("git_hash", DOLFINX_VERSION_GIT); + + io::native::write_mesh(io, engine, *mesh); - // io::checkpointing::write_mesh(io, engine, *mesh); - io::checkpointing::write_mesh(adios, *mesh); + engine.Close(); - adios.close(); + // // io::native::write_mesh(io, engine, *mesh); + // io::native::write_mesh(adios, *mesh); + + // adios.close(); + + // Set up ADIOS2 IO and Engine + // adios2::ADIOS adios_read(mesh->comm()); + // adios2::IO io_read = adios_read.DeclareIO("mesh-read"); + // io_read.SetEngine("BP5"); + // adios2::Engine engine_read = io_read.Open("mesh.bp", adios2::Mode::Read); + + // auto mesh_read_variant + // = io::native::read_mesh_variant(io_read, engine_read, MPI_COMM_WORLD); + + // engine_read.Close(); // ---------------------------------------------------------------------- - auto adios_query - = ADIOS2Wrapper(mesh->comm(), "mesh.bp", "mesh-read", "BP5", "read"); + // auto adios_query + // = ADIOS2Wrapper(MPI_COMM_WORLD, "mesh.bp", "mesh-read", "BP5", "read"); - auto adios_read - = ADIOS2Wrapper(mesh->comm(), "mesh.bp", "mesh-read", "BP5", "read"); + // auto adios_read + // = ADIOS2Wrapper(MPI_COMM_WORLD, "mesh.bp", "mesh-read", "BP5", "read"); // Can't use the same engine to query as well as read // since, in that case BeginStep and EndStep will be called twice // on a dataset written with a single Step - auto mesh_read_variant = io::checkpointing::read_mesh_variant( - adios_query, adios_read, mesh->comm()); - // We cannot resolve easily the variant - // Hence the user can query the type and - // call the correct read_mesh - auto mesh_read = std::get<0>(mesh_read_variant); - // auto mesh_read = std::get>(mesh_read_variant); + // auto reader = adios_read.engine(); + // reader->BeginStep(); + // auto mesh_read_variant = io::native::read_mesh_variant( + // adios_query, adios_read, mesh->comm()); + + // reader->EndStep(); + + // // We cannot resolve easily the variant + // // Hence the user can query the type and + // // call the correct read_mesh + // auto mesh_read = std::get<0>(mesh_read_variant); + // // auto mesh_read = std::get>(mesh_read_variant); - auto adios_write - = ADIOS2Wrapper(mesh->comm(), "mesh2.bp", "mesh-rewrite", "BP5", "write"); + // auto adios_write + // = ADIOS2Wrapper(mesh->comm(), "mesh2.bp", "mesh-rewrite", "BP5", + // "write"); - io::checkpointing::write_mesh(adios_write, mesh_read); + // io::native::write_mesh(adios_write, mesh_read); // auto io_query = adios_query.io(); // auto engine_query = adios_query.engine(); @@ -81,7 +114,7 @@ int main(int argc, char* argv[]) // engine_query->EndStep(); // // std::string floating_point = - // dolfinx::io::checkpointing::query_type(adios_read); std::cout << + // dolfinx::io::native::query_type(adios_read); std::cout << // floating_point; // // TODO: move type deduction inside checkpointing @@ -89,7 +122,7 @@ int main(int argc, char* argv[]) // { // using T = float; // mesh::Mesh mesh_read - // = io::checkpointing::read_mesh(adios_read, mesh->comm()); + // = io::native::read_mesh(adios_read, mesh->comm()); // adios_read.close(); // auto adios_write = ADIOS2Wrapper(mesh->comm(), "mesh2.bp", @@ -98,14 +131,14 @@ int main(int argc, char* argv[]) // auto io_write = adios_write.io(); - // io::checkpointing::write_mesh(adios_write, mesh_read); + // io::native::write_mesh(adios_write, mesh_read); // adios_write.close(); // } // else if ("double" == floating_point) // { // using T = double; // mesh::Mesh mesh_read - // = io::checkpointing::read_mesh(adios_read, mesh->comm()); + // = io::native::read_mesh(adios_read, mesh->comm()); // adios_read.close(); // auto adios_write = ADIOS2Wrapper(mesh->comm(), "mesh2.bp", @@ -114,7 +147,7 @@ int main(int argc, char* argv[]) // auto io_write = adios_write.io(); - // io::checkpointing::write_mesh(adios_write, mesh_read); + // io::native::write_mesh(adios_write, mesh_read); // adios_write.close(); // } diff --git a/cpp/dolfinx/io/ADIOS2_utils.cpp b/cpp/dolfinx/io/ADIOS2_utils.cpp index 428dac770f9..22fc1d6cb48 100644 --- a/cpp/dolfinx/io/ADIOS2_utils.cpp +++ b/cpp/dolfinx/io/ADIOS2_utils.cpp @@ -25,11 +25,11 @@ ADIOS2Wrapper::ADIOS2Wrapper(MPI_Comm comm, std::string filename, std::string tag, std::string engine_type, std::string mode) - : _adios(std::make_unique(comm)), - _io(std::make_unique(_adios->DeclareIO(tag))) + : _adios(std::make_shared(comm)), + _io(std::make_shared(_adios->DeclareIO(tag))) { _io->SetEngine(engine_type); - _engine = std::make_unique( + _engine = std::make_shared( _io->Open(filename, string_to_mode[mode])); } //----------------------------------------------------------------------------- diff --git a/cpp/dolfinx/io/ADIOS2_utils.h b/cpp/dolfinx/io/ADIOS2_utils.h index 0ef73a1562f..0af93e4a048 100644 --- a/cpp/dolfinx/io/ADIOS2_utils.h +++ b/cpp/dolfinx/io/ADIOS2_utils.h @@ -57,7 +57,7 @@ class ADIOS2Wrapper /// @brief Close the Engine object std::shared_ptr engine() { return _engine; } -protected: +public: std::shared_ptr _adios; std::shared_ptr _io; std::shared_ptr _engine; diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 185f87c895e..e472d54ac5b 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -9,6 +9,7 @@ #include "checkpointing.h" #include #include +#include #include #include #include @@ -32,11 +33,12 @@ std::map string_to_variant{ } // namespace -namespace dolfinx::io::checkpointing +namespace dolfinx::io::native { //----------------------------------------------------------------------------- template -void write_mesh(ADIOS2Wrapper& ADIOS2, dolfinx::mesh::Mesh& mesh) +void write_mesh(adios2::IO& io, adios2::Engine& engine, + dolfinx::mesh::Mesh& mesh) { const mesh::Geometry& geometry = mesh.geometry(); @@ -108,77 +110,74 @@ void write_mesh(ADIOS2Wrapper& ADIOS2, dolfinx::mesh::Mesh& mesh) // ADIOS2 write attributes and variables { - auto io = ADIOS2.io(); - auto engine = ADIOS2.engine(); - - io->DefineAttribute("name", mesh.name); - io->DefineAttribute("dim", dim); - io->DefineAttribute("cell_type", cell_type); - io->DefineAttribute("degree", degree); - io->DefineAttribute("lagrange_variant", lagrange_variant); + io.DefineAttribute("version", dolfinx::version()); + io.DefineAttribute("git_hash", dolfinx::git_commit_hash()); + io.DefineAttribute("name", mesh.name); + io.DefineAttribute("dim", dim); + io.DefineAttribute("cell_type", cell_type); + io.DefineAttribute("degree", degree); + io.DefineAttribute("lagrange_variant", lagrange_variant); adios2::Variable var_num_vertices - = io->DefineVariable("num_vertices"); + = io.DefineVariable("num_vertices"); adios2::Variable var_num_cells - = io->DefineVariable("num_cells"); + = io.DefineVariable("num_cells"); adios2::Variable var_num_dofs_per_cell - = io->DefineVariable("num_dofs_per_cell"); + = io.DefineVariable("num_dofs_per_cell"); adios2::Variable var_input_global_indices - = io->DefineVariable( + = io.DefineVariable( "input_global_indices", {num_vertices_global}, {offset}, {num_vertices_local}, adios2::ConstantDims); adios2::Variable var_x - = io->DefineVariable("x", {num_vertices_global, 3}, {offset, 0}, - {num_vertices_local, 3}, adios2::ConstantDims); + = io.DefineVariable("x", {num_vertices_global, 3}, {offset, 0}, + {num_vertices_local, 3}, adios2::ConstantDims); adios2::Variable var_topology_array - = io->DefineVariable( + = io.DefineVariable( "topology_array", {num_cells_global * num_dofs_per_cell}, {cell_offset * num_dofs_per_cell}, {num_cells_local * num_dofs_per_cell}, adios2::ConstantDims); adios2::Variable var_topology_offsets - = io->DefineVariable( + = io.DefineVariable( "topology_offsets", {num_cells_global + 1}, {cell_offset}, {num_cells_local + 1}, adios2::ConstantDims); - engine->BeginStep(); - engine->Put(var_num_vertices, num_vertices_global); - engine->Put(var_num_cells, num_cells_global); - engine->Put(var_num_dofs_per_cell, num_dofs_per_cell); - engine->Put(var_input_global_indices, input_global_indices_span.data()); - engine->Put(var_x, mesh_x.subspan(0, num_vertices_local * 3).data()); - engine->Put(var_topology_array, array_global.data()); - engine->Put(var_topology_offsets, offsets_global.data()); - engine->EndStep(); + engine.BeginStep(); + engine.Put(var_num_vertices, num_vertices_global); + engine.Put(var_num_cells, num_cells_global); + engine.Put(var_num_dofs_per_cell, num_dofs_per_cell); + engine.Put(var_input_global_indices, input_global_indices_span.data()); + engine.Put(var_x, mesh_x.subspan(0, num_vertices_local * 3).data()); + engine.Put(var_topology_array, array_global.data()); + engine.Put(var_topology_offsets, offsets_global.data()); + engine.EndStep(); } } //----------------------------------------------------------------------------- /// @cond -template void write_mesh(ADIOS2Wrapper& ADIOS2, +template void write_mesh(adios2::IO& io, adios2::Engine& engine, dolfinx::mesh::Mesh& mesh); -template void write_mesh(ADIOS2Wrapper& ADIOS2, +template void write_mesh(adios2::IO& io, adios2::Engine& engine, dolfinx::mesh::Mesh& mesh); /// @endcond //----------------------------------------------------------------------------- template -dolfinx::mesh::Mesh read_mesh(ADIOS2Wrapper& ADIOS2, MPI_Comm comm) +dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, + MPI_Comm comm) { int rank, size; MPI_Comm_rank(comm, &rank); MPI_Comm_size(comm, &size); - auto io = ADIOS2.io(); - auto engine = ADIOS2.engine(); - - engine->BeginStep(); + // engine.BeginStep(); // Attributes std::string name; @@ -189,15 +188,15 @@ dolfinx::mesh::Mesh read_mesh(ADIOS2Wrapper& ADIOS2, MPI_Comm comm) // Read attributes { adios2::Attribute var_name - = io->InquireAttribute("name"); + = io.InquireAttribute("name"); adios2::Attribute var_dim - = io->InquireAttribute("dim"); + = io.InquireAttribute("dim"); adios2::Attribute var_cell_type - = io->InquireAttribute("cell_type"); + = io.InquireAttribute("cell_type"); adios2::Attribute var_degree - = io->InquireAttribute("degree"); + = io.InquireAttribute("degree"); adios2::Attribute var_variant - = io->InquireAttribute("lagrange_variant"); + = io.InquireAttribute("lagrange_variant"); name = var_name.Data()[0]; dim = var_dim.Data()[0]; @@ -214,15 +213,17 @@ dolfinx::mesh::Mesh read_mesh(ADIOS2Wrapper& ADIOS2, MPI_Comm comm) // Read scalar variables { adios2::Variable var_num_vertices - = io->InquireVariable("num_vertices"); + = io.InquireVariable("num_vertices"); adios2::Variable var_num_cells - = io->InquireVariable("num_cells"); + = io.InquireVariable("num_cells"); adios2::Variable var_num_dofs_per_cell - = io->InquireVariable("num_dofs_per_cell"); + = io.InquireVariable("num_dofs_per_cell"); - engine->Get(var_num_vertices, num_vertices_global); - engine->Get(var_num_cells, num_cells_global); - engine->Get(var_num_dofs_per_cell, num_dofs_per_cell); + engine.Get(var_num_vertices, num_vertices_global); + engine.Get(var_num_cells, num_cells_global); + engine.Get(var_num_dofs_per_cell, num_dofs_per_cell); + + std::cout << num_vertices_global; } // Compute local sizes, offsets @@ -247,50 +248,50 @@ dolfinx::mesh::Mesh read_mesh(ADIOS2Wrapper& ADIOS2, MPI_Comm comm) { adios2::Variable var_input_global_indices - = io->InquireVariable("input_global_indices"); + = io.InquireVariable("input_global_indices"); - adios2::Variable var_x = io->InquireVariable("x"); + adios2::Variable var_x = io.InquireVariable("x"); adios2::Variable var_topology_array - = io->InquireVariable("topology_array"); + = io.InquireVariable("topology_array"); adios2::Variable var_topology_offsets - = io->InquireVariable("topology_offsets"); + = io.InquireVariable("topology_offsets"); if (var_input_global_indices) { var_input_global_indices.SetSelection( {{local_range[0]}, {num_vertices_local}}); - engine->Get(var_input_global_indices, input_global_indices.data(), - adios2::Mode::Deferred); + engine.Get(var_input_global_indices, input_global_indices.data(), + adios2::Mode::Deferred); } if (var_x) { var_x.SetSelection({{local_range[0], 0}, {num_vertices_local, 3}}); - engine->Get(var_x, x.data(), adios2::Mode::Deferred); + engine.Get(var_x, x.data(), adios2::Mode::Deferred); } if (var_topology_array) { var_topology_array.SetSelection({{cell_range[0] * num_dofs_per_cell}, {cell_range[1] * num_dofs_per_cell}}); - engine->Get(var_topology_array, array.data(), adios2::Mode::Deferred); + engine.Get(var_topology_array, array.data(), adios2::Mode::Deferred); } if (var_topology_offsets) { var_topology_offsets.SetSelection({{cell_range[0]}, {cell_range[1] + 1}}); - engine->Get(var_topology_offsets, offsets.data(), adios2::Mode::Deferred); + engine.Get(var_topology_offsets, offsets.data(), adios2::Mode::Deferred); } - engine->EndStep(); - std::int32_t cell_offset = offsets[0]; for (auto offset = offsets.begin(); offset != offsets.end(); ++offset) *offset -= cell_offset; } + // engine.EndStep(); + std::vector x_reduced(num_vertices_local * dim); for (std::uint32_t i = 0; i < num_vertices_local; ++i) { @@ -311,44 +312,47 @@ dolfinx::mesh::Mesh read_mesh(ADIOS2Wrapper& ADIOS2, MPI_Comm comm) //----------------------------------------------------------------------------- /// @cond -template dolfinx::mesh::Mesh read_mesh(ADIOS2Wrapper& ADIOS2, - MPI_Comm comm); +template dolfinx::mesh::Mesh +read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm); -template dolfinx::mesh::Mesh read_mesh(ADIOS2Wrapper& ADIOS2, - MPI_Comm comm); +template dolfinx::mesh::Mesh +read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm); /// @endcond -//----------------------------------------------------------------------------- -std::string query_type(ADIOS2Wrapper& ADIOS2) -{ - auto io = ADIOS2.io(); - auto engine = ADIOS2.engine(); - engine->BeginStep(); - std::string floating_point = io->VariableType("x"); - engine->EndStep(); +// //----------------------------------------------------------------------------- +// std::string query_type(ADIOS2Wrapper& ADIOS2) +// { +// auto io = ADIOS2.io(); +// auto engine = ADIOS2.engine(); +// engine.BeginStep(); +// std::string floating_point = io.VariableType("x"); +// engine.EndStep(); - return floating_point; -} +// return floating_point; +// } //----------------------------------------------------------------------------- std::variant, dolfinx::mesh::Mesh> -read_mesh_variant(ADIOS2Wrapper& _ADIOS2, ADIOS2Wrapper& ADIOS2, MPI_Comm comm) +read_mesh_variant(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm) { - std::string floating_point = query_type(_ADIOS2); + engine.BeginStep(); + std::string floating_point = io.VariableType("x"); if (floating_point == "float") { - dolfinx::mesh::Mesh mesh = read_mesh(ADIOS2, comm); + dolfinx::mesh::Mesh mesh = read_mesh(io, engine, comm); + engine.EndStep(); return mesh; } else // floating_point == "double" { - dolfinx::mesh::Mesh mesh = read_mesh(ADIOS2, comm); + dolfinx::mesh::Mesh mesh = read_mesh(io, engine, comm); + engine.EndStep(); return mesh; } } -} // namespace dolfinx::io::checkpointing +} // namespace dolfinx::io::native #endif diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index ed785b33ef3..c071697f017 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -17,41 +17,44 @@ /// @file checkpointing.h /// @brief ADIOS2 based checkpointing -namespace dolfinx::io::checkpointing +namespace dolfinx::io::native { /// @brief Write mesh to a file. /// -/// @param[in] ADIOS2 ADIOS2Wrapper +/// @param[in] io ADIOS2 IO +/// @param[in] engine ADIOS2 Engine /// @param[in] mesh Mesh of type float or double to write to the file template -void write_mesh(ADIOS2Wrapper& ADIOS2, dolfinx::mesh::Mesh& mesh); +void write_mesh(adios2::IO& io, adios2::Engine& engine, + dolfinx::mesh::Mesh& mesh); /// @brief Read mesh from a file. /// -/// @param[in] ADIOS2 ADIOS2Wrapper +/// @param[in] io ADIOS2 IO +/// @param[in] engine ADIOS2 Engine /// @param[in] comm comm /// @return mesh reconstructed from the data template -dolfinx::mesh::Mesh read_mesh(ADIOS2Wrapper& ADIOS2, +dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm = MPI_COMM_WORLD); /// @brief Read mesh from a file. /// -/// @param[in] _ADIOS2 ADIOS2Wrapper -/// @param[in] ADIOS2 ADIOS2Wrapper +/// @param[in] io ADIOS2 IO +/// @param[in] engine ADIOS2 Engine /// @param[in] comm comm /// @return mesh reconstructed from the data std::variant, dolfinx::mesh::Mesh> -read_mesh_variant(ADIOS2Wrapper& _ADIOS2, ADIOS2Wrapper& ADIOS2, +read_mesh_variant(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm = MPI_COMM_WORLD); -/// @brief Query floating_point type stored in the file -/// -/// @param[in] ADIOS2 ADIOS2Wrapper -/// @return type float or double as a string -std::string query_type(ADIOS2Wrapper& ADIOS2); +// /// @brief Query floating_point type stored in the file +// /// +// /// @param[in] ADIOS2 ADIOS2Wrapper +// /// @return type float or double as a string +// std::string query_type(ADIOS2Wrapper& ADIOS2); -} // namespace dolfinx::io::checkpointing +} // namespace dolfinx::io::native #endif diff --git a/python/demo/demo_checkpointing.py b/python/demo/demo_checkpointing.py index 835fd51985e..ff3b52e236e 100644 --- a/python/demo/demo_checkpointing.py +++ b/python/demo/demo_checkpointing.py @@ -44,18 +44,13 @@ msh = mesh.create_rectangle( comm=MPI.COMM_WORLD, points=((0.0, 0.0), (1.0, 1.0)), - n=(4, 4), + n=(8, 8), cell_type=mesh.CellType.triangle, ) # - # + -filename = "mesh.bp" -engine_type = "BP5" -tag = "mesh-write" -mode = "write" - -adios2 = io.ADIOS2(msh.comm, filename, tag, engine_type, mode) +adios2 = io.ADIOS2(msh.comm, filename="mesh.bp", tag="mesh-write", engine_type="BP5", mode="write") # - # + @@ -63,22 +58,22 @@ adios2.close() # - -filename = "mesh.bp" -engine_type = "BP5" -tag = "mesh-read" -mode = "read" - -adios2_query = io.ADIOS2(msh.comm, filename, tag, engine_type, mode) -adios2_read = io.ADIOS2(msh.comm, filename, tag, engine_type, mode) +# + +adios2_query = io.ADIOS2( + msh.comm, filename="mesh.bp", tag="mesh-read", engine_type="BP5", mode="read" +) +adios2_read = io.ADIOS2( + msh.comm, filename="mesh.bp", tag="mesh-read", engine_type="BP5", mode="read" +) # - # + msh_read = io.read_mesh(adios2_query, adios2_read, msh.comm) # - -# + -print(type(msh_read)) -print(msh_read.name) -print(msh_read.geometry.x.dtype) -print(msh_read.geometry.x) -# - +# # + +# print(type(msh_read)) +# print(msh_read.name) +# print(msh_read.geometry.x.dtype) +# print(msh_read.geometry.x) +# # - From ff0fb6282bf9ce984e8195c4e69f0851a1d533e1 Mon Sep 17 00:00:00 2001 From: ampdes Date: Mon, 12 Aug 2024 11:33:37 +0200 Subject: [PATCH 47/60] Update from review * Update copyright headers * Remove ADIOS_utils.cpp * Correct the global topology computation * Rename vertices to nodes --- cpp/demo/checkpointing/checkpointing.yml | 10 ++ cpp/demo/checkpointing/main.cpp | 39 +++---- cpp/dolfinx/io/ADIOS2_utils.cpp | 45 -------- cpp/dolfinx/io/ADIOS2_utils.h | 32 +++++- cpp/dolfinx/io/CMakeLists.txt | 3 +- cpp/dolfinx/io/checkpointing.cpp | 135 +++++++++++------------ cpp/dolfinx/io/checkpointing.h | 16 ++- python/demo/checkpointing.yml | 10 ++ python/demo/demo_checkpointing.py | 5 +- python/dolfinx/io/utils.py | 4 +- python/dolfinx/wrappers/io.cpp | 28 +++-- 11 files changed, 156 insertions(+), 171 deletions(-) create mode 100644 cpp/demo/checkpointing/checkpointing.yml delete mode 100644 cpp/dolfinx/io/ADIOS2_utils.cpp create mode 100644 python/demo/checkpointing.yml diff --git a/cpp/demo/checkpointing/checkpointing.yml b/cpp/demo/checkpointing/checkpointing.yml new file mode 100644 index 00000000000..3ca79ee71d2 --- /dev/null +++ b/cpp/demo/checkpointing/checkpointing.yml @@ -0,0 +1,10 @@ +--- +# adios2 config.yaml +# IO YAML Sequence (-) Nodes to allow for multiple IO nodes +# IO name referred in code with DeclareIO is mandatory + +- IO: "mesh-write" + + Engine: + # If Type is missing or commented out, default Engine is picked up + Type: "BP5" diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 62165787ec7..83980e030d0 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -1,5 +1,5 @@ // ```text -// Copyright (C) 2024 Abdullah Mujahid +// Copyright (C) 2024 Abdullah Mujahid, Jørgen S. Dokken, Jack S. Hale // This file is part of DOLFINx (https://www.fenicsproject.org) // SPDX-License-Identifier: LGPL-3.0-or-later // ``` @@ -36,30 +36,31 @@ int main(int argc, char* argv[]) MPI_COMM_WORLD, {{{0.0, 0.0}, {1.0, 1.0}}}, {4, 4}, mesh::CellType::quadrilateral, part)); - // // Set up ADIOS2 IO and Engine - // auto adios - // = ADIOS2Wrapper(mesh->comm(), "mesh.bp", "mesh-write", "BP5", "write"); - // auto io = adios.io(); + try + { + // Set up ADIOS2 IO and Engine + adios2::ADIOS adios("checkpointing.yml", mesh->comm()); - // Set up ADIOS2 IO and Engine - adios2::ADIOS adios(mesh->comm()); - adios2::IO io = adios.DeclareIO("mesh-write"); - io.SetEngine("BP5"); - adios2::Engine engine = io.Open("mesh.bp", adios2::Mode::Write); - - // TODO: Need to move this inside the checkpointing module - // io.DefineAttribute("version", DOLFINX_VERSION_STRING); - // io.DefineAttribute("git_hash", DOLFINX_VERSION_GIT); + adios2::IO io = adios.DeclareIO("mesh-write"); + io.SetEngine("BP5"); + adios2::Engine engine = io.Open("mesh.bp", adios2::Mode::Write); - io::native::write_mesh(io, engine, *mesh); + std::vector mytags = {"one"}; + io.DefineAttribute("tags", mytags.data(), mytags.size(), "", "", true); - engine.Close(); + std::vector mytags2 = {"one", "two"}; + io.DefineAttribute("tags", mytags2.data(), mytags2.size()); - // // io::native::write_mesh(io, engine, *mesh); - // io::native::write_mesh(adios, *mesh); + io::native::write_mesh(io, engine, *mesh); - // adios.close(); + engine.Close(); + } + catch (std::exception &e) + { + std::cout << "ERROR: ADIOS2 exception: " << e.what() << "\n"; + MPI_Abort(MPI_COMM_WORLD, -1); + } // Set up ADIOS2 IO and Engine // adios2::ADIOS adios_read(mesh->comm()); diff --git a/cpp/dolfinx/io/ADIOS2_utils.cpp b/cpp/dolfinx/io/ADIOS2_utils.cpp deleted file mode 100644 index 22fc1d6cb48..00000000000 --- a/cpp/dolfinx/io/ADIOS2_utils.cpp +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (C) 2024 Abdullah Mujahid, Jørgen S. Dokken and Garth N. Wells -// -// This file is part of DOLFINX (https://www.fenicsproject.org) -// -// SPDX-License-Identifier: LGPL-3.0-or-later - -#ifdef HAS_ADIOS2 - -#include "ADIOS2_utils.h" -#include -#include - -namespace -{ -std::map string_to_mode{ - {"write", adios2::Mode::Write}, - {"read", adios2::Mode::Read}, -}; -} // namespace - -using namespace dolfinx::io; - -//----------------------------------------------------------------------------- -ADIOS2Wrapper::ADIOS2Wrapper(MPI_Comm comm, std::string filename, - std::string tag, std::string engine_type, - std::string mode) - - : _adios(std::make_shared(comm)), - _io(std::make_shared(_adios->DeclareIO(tag))) -{ - _io->SetEngine(engine_type); - _engine = std::make_shared( - _io->Open(filename, string_to_mode[mode])); -} -//----------------------------------------------------------------------------- -ADIOS2Wrapper::~ADIOS2Wrapper() { close(); } -//----------------------------------------------------------------------------- -void ADIOS2Wrapper::close() -{ - assert(_engine); - if (*_engine) - _engine->Close(); -} - -#endif \ No newline at end of file diff --git a/cpp/dolfinx/io/ADIOS2_utils.h b/cpp/dolfinx/io/ADIOS2_utils.h index 0af93e4a048..41a21a2ef69 100644 --- a/cpp/dolfinx/io/ADIOS2_utils.h +++ b/cpp/dolfinx/io/ADIOS2_utils.h @@ -16,6 +16,14 @@ /// @file ADIOS2_utils.h /// @brief Utils for ADIOS2 +namespace +{ +std::map string_to_mode{ + {"write", adios2::Mode::Write}, + {"read", adios2::Mode::Read}, +}; +} // namespace + namespace dolfinx::io { @@ -26,12 +34,19 @@ class ADIOS2Wrapper /// @brief Create an ADIOS2-based engine writer/reader /// @param[in] comm The MPI communicator /// @param[in] filename Name of output file - /// @param[in] tag The ADIOS2 object name + /// @param[in] tag The ADIOS2 IO name /// @param[in] engine_type ADIOS2 engine type. See /// https://adios2.readthedocs.io/en/latest/engines/engines.html. /// @param[in] mode ADIOS2 mode, default is Write or Read ADIOS2Wrapper(MPI_Comm comm, std::string filename, std::string tag, - std::string engine_type = "BP5", std::string mode = "write"); + std::string engine_type = "BP5", std::string mode = "write") + : _adios(std::make_shared(comm)), + _io(std::make_shared(_adios->DeclareIO(tag))) + { + _io->SetEngine(engine_type); + _engine = std::make_shared( + _io->Open(filename, string_to_mode[mode])); + } /// @brief Move constructor ADIOS2Wrapper(ADIOS2Wrapper&& ADIOS2) = default; @@ -40,7 +55,7 @@ class ADIOS2Wrapper ADIOS2Wrapper(const ADIOS2Wrapper&) = delete; /// @brief Destructor - ~ADIOS2Wrapper(); + ~ADIOS2Wrapper() { close(); } /// @brief Move assignment ADIOS2Wrapper& operator=(ADIOS2Wrapper&& ADIOS2) = default; @@ -49,15 +64,20 @@ class ADIOS2Wrapper ADIOS2Wrapper& operator=(const ADIOS2Wrapper&) = delete; /// @brief Close the file - void close(); + void close() + { + assert(_engine); + if (*_engine) + _engine->Close(); + } /// @brief Get the IO object std::shared_ptr io() { return _io; } - /// @brief Close the Engine object + /// @brief Get the Engine object std::shared_ptr engine() { return _engine; } -public: +private: std::shared_ptr _adios; std::shared_ptr _io; std::shared_ptr _engine; diff --git a/cpp/dolfinx/io/CMakeLists.txt b/cpp/dolfinx/io/CMakeLists.txt index 6e8c6fe59d0..c03d18de19f 100644 --- a/cpp/dolfinx/io/CMakeLists.txt +++ b/cpp/dolfinx/io/CMakeLists.txt @@ -16,8 +16,7 @@ set(HEADERS_io target_sources( dolfinx - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2_utils.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.cpp + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/cells.cpp ${CMAKE_CURRENT_SOURCE_DIR}/checkpointing.cpp ${CMAKE_CURRENT_SOURCE_DIR}/HDF5Interface.cpp diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index e472d54ac5b..46f3a02aae6 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2024 Abdullah Mujahid +// Copyright (C) 2024 Abdullah Mujahid, Jørgen S. Dokken, Jack S. Hale // // This file is part of DOLFINX (https://www.fenicsproject.org) // @@ -38,34 +38,38 @@ namespace dolfinx::io::native //----------------------------------------------------------------------------- template void write_mesh(adios2::IO& io, adios2::Engine& engine, - dolfinx::mesh::Mesh& mesh) + const dolfinx::mesh::Mesh& mesh) { const mesh::Geometry& geometry = mesh.geometry(); std::shared_ptr topology = mesh.topology(); + assert(topology); // Variables/attributes to save std::int32_t dim = geometry.dim(); - std::uint64_t num_vertices_global, offset, num_cells_global, cell_offset; - std::uint32_t num_vertices_local, num_cells_local, num_dofs_per_cell, degree; + std::int32_t tdim = topology->dim(); + std::uint64_t num_nodes_global, offset, num_cells_global, cell_offset; + std::uint32_t num_nodes_local, num_cells_local, num_dofs_per_cell, degree; std::string cell_type, lagrange_variant; std::vector array_global; std::vector offsets_global; std::shared_ptr geom_imap; - // Vertices information + // Nodes information { - geom_imap = mesh.geometry().index_map(); - num_vertices_global = geom_imap->size_global(); - num_vertices_local = geom_imap->size_local(); + geom_imap = geometry.index_map(); + num_nodes_global = geom_imap->size_global(); + num_nodes_local = geom_imap->size_local(); offset = geom_imap->local_range()[0]; } // Cells information { const std::shared_ptr topo_imap - = topology->index_map(dim); + = topology->index_map(tdim); + assert(topo_imap); + num_cells_global = topo_imap->size_global(); num_cells_local = topo_imap->size_local(); cell_offset = topo_imap->local_range()[0]; @@ -73,39 +77,31 @@ void write_mesh(adios2::IO& io, adios2::Engine& engine, // Coordinate element information { - const fem::CoordinateElement& cmap = mesh.geometry().cmap(); - fem::ElementDofLayout geom_layout = cmap.create_dof_layout(); - num_dofs_per_cell = geom_layout.num_entity_closure_dofs(dim); + const fem::CoordinateElement& cmap = geometry.cmap(); cell_type = mesh::to_string(cmap.cell_shape()); degree = cmap.degree(); lagrange_variant = variant_to_string[cmap.variant()]; } - const std::vector input_global_indices - = geometry.input_global_indices(); - const std::span input_global_indices_span( - input_global_indices.begin(), num_vertices_local); const std::span mesh_x = geometry.x(); // Connectivity { - std::shared_ptr> connectivity - = topology->connectivity(dim, 0); - const std::vector& array = connectivity->array(); - const std::vector& offsets = connectivity->offsets(); - - const std::span array_span(array.begin(), - offsets[num_cells_local]); - - array_global.resize(offsets[num_cells_local]); - offsets_global.resize(num_cells_local + 1); - + auto dofmap = geometry.dofmap(); + num_dofs_per_cell = dofmap.extent(1); + std::vector connectivity; + connectivity.reserve(num_cells_local * num_dofs_per_cell); + for (std::size_t i = 0; i < num_cells_local; ++i) + for (std::size_t j = 0; j < num_dofs_per_cell; ++j) + connectivity.push_back(dofmap(i, j)); + + array_global.resize(num_cells_local * num_dofs_per_cell); std::iota(array_global.begin(), array_global.end(), 0); - geom_imap->local_to_global(array_span, array_global); - + geom_imap->local_to_global(connectivity, array_global); + offsets_global.resize(num_cells_local + 1); for (std::size_t i = 0; i < num_cells_local + 1; ++i) - offsets_global[i] = offsets[i] + cell_offset * num_dofs_per_cell; + offsets_global[i] = (i + cell_offset) * num_dofs_per_cell; } // ADIOS2 write attributes and variables @@ -114,25 +110,21 @@ void write_mesh(adios2::IO& io, adios2::Engine& engine, io.DefineAttribute("git_hash", dolfinx::git_commit_hash()); io.DefineAttribute("name", mesh.name); io.DefineAttribute("dim", dim); + io.DefineAttribute("tdim", tdim); io.DefineAttribute("cell_type", cell_type); io.DefineAttribute("degree", degree); io.DefineAttribute("lagrange_variant", lagrange_variant); - adios2::Variable var_num_vertices - = io.DefineVariable("num_vertices"); + adios2::Variable var_num_nodes + = io.DefineVariable("num_nodes"); adios2::Variable var_num_cells = io.DefineVariable("num_cells"); adios2::Variable var_num_dofs_per_cell = io.DefineVariable("num_dofs_per_cell"); - adios2::Variable var_input_global_indices - = io.DefineVariable( - "input_global_indices", {num_vertices_global}, {offset}, - {num_vertices_local}, adios2::ConstantDims); - adios2::Variable var_x - = io.DefineVariable("x", {num_vertices_global, 3}, {offset, 0}, - {num_vertices_local, 3}, adios2::ConstantDims); + = io.DefineVariable("x", {num_nodes_global, 3}, {offset, 0}, + {num_nodes_local, 3}, adios2::ConstantDims); adios2::Variable var_topology_array = io.DefineVariable( @@ -146,11 +138,10 @@ void write_mesh(adios2::IO& io, adios2::Engine& engine, {num_cells_local + 1}, adios2::ConstantDims); engine.BeginStep(); - engine.Put(var_num_vertices, num_vertices_global); + engine.Put(var_num_nodes, num_nodes_global); engine.Put(var_num_cells, num_cells_global); engine.Put(var_num_dofs_per_cell, num_dofs_per_cell); - engine.Put(var_input_global_indices, input_global_indices_span.data()); - engine.Put(var_x, mesh_x.subspan(0, num_vertices_local * 3).data()); + engine.Put(var_x, mesh_x.subspan(0, num_nodes_local * 3).data()); engine.Put(var_topology_array, array_global.data()); engine.Put(var_topology_offsets, offsets_global.data()); engine.EndStep(); @@ -160,10 +151,10 @@ void write_mesh(adios2::IO& io, adios2::Engine& engine, //----------------------------------------------------------------------------- /// @cond template void write_mesh(adios2::IO& io, adios2::Engine& engine, - dolfinx::mesh::Mesh& mesh); + const dolfinx::mesh::Mesh& mesh); template void write_mesh(adios2::IO& io, adios2::Engine& engine, - dolfinx::mesh::Mesh& mesh); + const dolfinx::mesh::Mesh& mesh); /// @endcond @@ -206,33 +197,33 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, } // Scalar variables - std::uint64_t num_vertices_global; + std::uint64_t num_nodes_global; std::uint64_t num_cells_global; std::uint32_t num_dofs_per_cell; // Read scalar variables { - adios2::Variable var_num_vertices - = io.InquireVariable("num_vertices"); + adios2::Variable var_num_nodes + = io.InquireVariable("num_nodes"); adios2::Variable var_num_cells = io.InquireVariable("num_cells"); adios2::Variable var_num_dofs_per_cell = io.InquireVariable("num_dofs_per_cell"); - engine.Get(var_num_vertices, num_vertices_global); + engine.Get(var_num_nodes, num_nodes_global); engine.Get(var_num_cells, num_cells_global); engine.Get(var_num_dofs_per_cell, num_dofs_per_cell); - std::cout << num_vertices_global; + std::cout << num_nodes_global; } // Compute local sizes, offsets std::array _local_range - = dolfinx::MPI::local_range(rank, num_vertices_global, size); + = dolfinx::MPI::local_range(rank, num_nodes_global, size); std::array local_range{(std::uint64_t)_local_range[0], (std::uint64_t)_local_range[1]}; - std::uint64_t num_vertices_local = local_range[1] - local_range[0]; + std::uint64_t num_nodes_local = local_range[1] - local_range[0]; std::array _cell_range = dolfinx::MPI::local_range(rank, num_cells_global, size); @@ -241,8 +232,8 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, (std::uint64_t)_cell_range[1]}; std::uint64_t num_cells_local = cell_range[1] - cell_range[0]; - std::vector input_global_indices(num_vertices_local); - std::vector x(num_vertices_local * 3); + std::vector input_global_indices(num_nodes_local); + std::vector x(num_nodes_local * 3); std::vector array(num_cells_local * num_dofs_per_cell); std::vector offsets(num_cells_local + 1); @@ -261,14 +252,14 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, if (var_input_global_indices) { var_input_global_indices.SetSelection( - {{local_range[0]}, {num_vertices_local}}); + {{local_range[0]}, {num_nodes_local}}); engine.Get(var_input_global_indices, input_global_indices.data(), adios2::Mode::Deferred); } if (var_x) { - var_x.SetSelection({{local_range[0], 0}, {num_vertices_local, 3}}); + var_x.SetSelection({{local_range[0], 0}, {num_nodes_local, 3}}); engine.Get(var_x, x.data(), adios2::Mode::Deferred); } @@ -292,8 +283,8 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, // engine.EndStep(); - std::vector x_reduced(num_vertices_local * dim); - for (std::uint32_t i = 0; i < num_vertices_local; ++i) + std::vector x_reduced(num_nodes_local * dim); + for (std::uint32_t i = 0; i < num_nodes_local; ++i) { for (std::uint32_t j = 0; j < (std::uint32_t)dim; ++j) x_reduced[i * dim + j] = x[i * 3 + j]; @@ -302,7 +293,7 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, fem::CoordinateElement element = fem::CoordinateElement(cell_type, degree, lagrange_variant); - std::array xshape = {num_vertices_local, (std::uint32_t)dim}; + std::array xshape = {num_nodes_local, (std::uint32_t)dim}; auto part = mesh::create_cell_partitioner(mesh::GhostMode::shared_facet); mesh::Mesh mesh = mesh::create_mesh(comm, comm, array, element, comm, @@ -320,18 +311,10 @@ read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm); /// @endcond -// //----------------------------------------------------------------------------- -// std::string query_type(ADIOS2Wrapper& ADIOS2) -// { -// auto io = ADIOS2.io(); -// auto engine = ADIOS2.engine(); -// engine.BeginStep(); -// std::string floating_point = io.VariableType("x"); -// engine.EndStep(); - -// return floating_point; -// } +} // namespace dolfinx::io::native +namespace dolfinx::io::impl_native +{ //----------------------------------------------------------------------------- std::variant, dolfinx::mesh::Mesh> read_mesh_variant(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm) @@ -341,18 +324,24 @@ read_mesh_variant(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm) if (floating_point == "float") { - dolfinx::mesh::Mesh mesh = read_mesh(io, engine, comm); + dolfinx::mesh::Mesh mesh + = dolfinx::io::native::read_mesh(io, engine, comm); engine.EndStep(); return mesh; } - else // floating_point == "double" + else if (floating_point == "double") { - dolfinx::mesh::Mesh mesh = read_mesh(io, engine, comm); + dolfinx::mesh::Mesh mesh + = dolfinx::io::native::read_mesh(io, engine, comm); engine.EndStep(); return mesh; } + else + { + throw std::runtime_error("Floating point type is neither float or double"); + } } -} // namespace dolfinx::io::native +} // namespace dolfinx::io::impl_native #endif diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index c071697f017..25cc5faa92f 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -1,4 +1,4 @@ -// Copyright (C) 2024 Abdullah Mujahid +// Copyright (C) 2024 Abdullah Mujahid, Jørgen S. Dokken, Jack S. Hale // // This file is part of DOLFINX (https://www.fenicsproject.org) // @@ -27,7 +27,7 @@ namespace dolfinx::io::native /// @param[in] mesh Mesh of type float or double to write to the file template void write_mesh(adios2::IO& io, adios2::Engine& engine, - dolfinx::mesh::Mesh& mesh); + const dolfinx::mesh::Mesh& mesh); /// @brief Read mesh from a file. /// @@ -39,6 +39,10 @@ template dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm = MPI_COMM_WORLD); +} // namespace dolfinx::io::native + +namespace dolfinx::io::impl_native +{ /// @brief Read mesh from a file. /// /// @param[in] io ADIOS2 IO @@ -49,12 +53,6 @@ std::variant, dolfinx::mesh::Mesh> read_mesh_variant(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm = MPI_COMM_WORLD); -// /// @brief Query floating_point type stored in the file -// /// -// /// @param[in] ADIOS2 ADIOS2Wrapper -// /// @return type float or double as a string -// std::string query_type(ADIOS2Wrapper& ADIOS2); - -} // namespace dolfinx::io::native +} // namespace dolfinx::io::impl_native #endif diff --git a/python/demo/checkpointing.yml b/python/demo/checkpointing.yml new file mode 100644 index 00000000000..3ca79ee71d2 --- /dev/null +++ b/python/demo/checkpointing.yml @@ -0,0 +1,10 @@ +--- +# adios2 config.yaml +# IO YAML Sequence (-) Nodes to allow for multiple IO nodes +# IO name referred in code with DeclareIO is mandatory + +- IO: "mesh-write" + + Engine: + # If Type is missing or commented out, default Engine is picked up + Type: "BP5" diff --git a/python/demo/demo_checkpointing.py b/python/demo/demo_checkpointing.py index ff3b52e236e..834a295d75f 100644 --- a/python/demo/demo_checkpointing.py +++ b/python/demo/demo_checkpointing.py @@ -59,16 +59,13 @@ # - # + -adios2_query = io.ADIOS2( - msh.comm, filename="mesh.bp", tag="mesh-read", engine_type="BP5", mode="read" -) adios2_read = io.ADIOS2( msh.comm, filename="mesh.bp", tag="mesh-read", engine_type="BP5", mode="read" ) # - # + -msh_read = io.read_mesh(adios2_query, adios2_read, msh.comm) +msh_read = io.read_mesh(adios2_read, msh.comm) # - # # + diff --git a/python/dolfinx/io/utils.py b/python/dolfinx/io/utils.py index bc762d6222c..919006da777 100644 --- a/python/dolfinx/io/utils.py +++ b/python/dolfinx/io/utils.py @@ -218,9 +218,9 @@ def write_mesh(ADIOS2: ADIOS2, mesh: Mesh) -> None: return _writer(ADIOS2, mesh._cpp_object) - def read_mesh(_ADIOS2: ADIOS2, ADIOS2: ADIOS2, comm: _MPI.Comm) -> Mesh: + def read_mesh(ADIOS2: ADIOS2, comm: _MPI.Comm) -> Mesh: """Read mesh from a file using ADIOS2""" - msh = _cpp.io.read_mesh(_ADIOS2, ADIOS2, comm) + msh = _cpp.io.read_mesh(ADIOS2, comm) return msh diff --git a/python/dolfinx/wrappers/io.cpp b/python/dolfinx/wrappers/io.cpp index 6af675afbe8..9bb93ead3a9 100644 --- a/python/dolfinx/wrappers/io.cpp +++ b/python/dolfinx/wrappers/io.cpp @@ -238,11 +238,17 @@ void declare_data_types(nb::module_& m) template void declare_write_mesh(nb::module_& m, std::string type) { - // dolfinx::io::checkpointing::write_mesh + // dolfinx::io::native::write_mesh std::string pyfunction_write_mesh_name = std::string("write_mesh_") + type; - m.def(pyfunction_write_mesh_name.c_str(), - &dolfinx::io::checkpointing::write_mesh, nb::arg("adios2"), - nb::arg("mesh"), "Write mesh to file using ADIOS2"); + m.def( + pyfunction_write_mesh_name.c_str(), + [](dolfinx::io::ADIOS2Wrapper& ADIOS2, dolfinx::mesh::Mesh& mesh) + { + auto io = ADIOS2.io(); + auto engine = ADIOS2.engine(); + return dolfinx::io::native::write_mesh(*io, *engine, mesh); + }, + nb::arg("adios2"), nb::arg("mesh"), "Write mesh to file using ADIOS2"); } #endif @@ -269,17 +275,17 @@ void io(nb::module_& m) nb::arg("engine_type"), nb::arg("mode")) .def("close", &dolfinx::io::ADIOS2Wrapper::close); - // dolfinx::io::checkpointing::read_mesh_variant + // dolfinx::io::impl_native::read_mesh_variant m.def( "read_mesh", - [](dolfinx::io::ADIOS2Wrapper& _ADIOS2, - dolfinx::io::ADIOS2Wrapper& ADIOS2, MPICommWrapper comm) + [](dolfinx::io::ADIOS2Wrapper& ADIOS2, MPICommWrapper comm) { - return dolfinx::io::checkpointing::read_mesh_variant(_ADIOS2, ADIOS2, - comm.get()); + auto io = ADIOS2.io(); + auto engine = ADIOS2.engine(); + return dolfinx::io::impl_native::read_mesh_variant(*io, *engine, + comm.get()); }, - nb::arg("_adios2"), nb::arg("adios2"), nb::arg("comm"), - "Read mesh from file using ADIOS2"); + nb::arg("adios2"), nb::arg("comm"), "Read mesh from file using ADIOS2"); declare_write_mesh(m, "float32"); declare_write_mesh(m, "float64"); From a3fe473ebec1932370ede3c230d551a9bc066a19 Mon Sep 17 00:00:00 2001 From: ampdes Date: Mon, 12 Aug 2024 11:59:39 +0200 Subject: [PATCH 48/60] Add config file input to ADIOS2 wrapper --- cpp/dolfinx/io/ADIOS2_utils.h | 15 +++++++++++++++ python/demo/demo_checkpointing.py | 6 ++++-- python/dolfinx/wrappers/io.cpp | 11 +++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/cpp/dolfinx/io/ADIOS2_utils.h b/cpp/dolfinx/io/ADIOS2_utils.h index 41a21a2ef69..b8613456508 100644 --- a/cpp/dolfinx/io/ADIOS2_utils.h +++ b/cpp/dolfinx/io/ADIOS2_utils.h @@ -48,6 +48,21 @@ class ADIOS2Wrapper _io->Open(filename, string_to_mode[mode])); } + /// @brief Create an ADIOS2-based engine writer/reader + /// @param[in] config_file Path to config file to set up ADIOS2 engine from + /// @param[in] comm The MPI communicator + /// @param[in] filename Name of output file + /// @param[in] tag The ADIOS2 IO name + /// @param[in] mode ADIOS2 mode, default is Write or Read + ADIOS2Wrapper(std::string config_file, MPI_Comm comm, std::string filename, + std::string tag, std::string mode = "write") + { + _adios = std::make_shared(config_file, comm); + _io = std::make_shared(_adios->DeclareIO(tag)); + _engine = std::make_shared( + _io->Open(filename, string_to_mode[mode])); + } + /// @brief Move constructor ADIOS2Wrapper(ADIOS2Wrapper&& ADIOS2) = default; diff --git a/python/demo/demo_checkpointing.py b/python/demo/demo_checkpointing.py index 834a295d75f..8720f37ae22 100644 --- a/python/demo/demo_checkpointing.py +++ b/python/demo/demo_checkpointing.py @@ -16,6 +16,7 @@ # + import importlib.util +import os if importlib.util.find_spec("adios2") is not None: import dolfinx @@ -50,7 +51,8 @@ # - # + -adios2 = io.ADIOS2(msh.comm, filename="mesh.bp", tag="mesh-write", engine_type="BP5", mode="write") +config_path = os.getcwd() + "/checkpointing.yml" +adios2 = io.ADIOS2(config_path, msh.comm, filename="mesh.bp", tag="mesh-write", mode="write") # - # + @@ -65,7 +67,7 @@ # - # + -msh_read = io.read_mesh(adios2_read, msh.comm) +# msh_read = io.read_mesh(adios2_read, msh.comm) # - # # + diff --git a/python/dolfinx/wrappers/io.cpp b/python/dolfinx/wrappers/io.cpp index 9bb93ead3a9..df656ccbd8b 100644 --- a/python/dolfinx/wrappers/io.cpp +++ b/python/dolfinx/wrappers/io.cpp @@ -273,6 +273,17 @@ void io(nb::module_& m) }, nb::arg("comm"), nb::arg("filename"), nb::arg("tag"), nb::arg("engine_type"), nb::arg("mode")) + .def( + "__init__", + [](dolfinx::io::ADIOS2Wrapper* v, std::string config, + MPICommWrapper comm, const std::string filename, std::string tag, + std::string mode = "write") + { + new (v) dolfinx::io::ADIOS2Wrapper(config, comm.get(), filename, + tag, mode); + }, + nb::arg("config"), nb::arg("comm"), nb::arg("filename"), + nb::arg("tag"), nb::arg("mode")) .def("close", &dolfinx::io::ADIOS2Wrapper::close); // dolfinx::io::impl_native::read_mesh_variant From 636f7427ca5a8fcdd36f2662604f3da66df531c7 Mon Sep 17 00:00:00 2001 From: ampdes Date: Wed, 14 Aug 2024 07:28:15 +0200 Subject: [PATCH 49/60] Refactor and clean --- cpp/demo/checkpointing/main.cpp | 137 +++++-------------- cpp/dolfinx/io/checkpointing.cpp | 211 ++++++++++++++++++------------ cpp/dolfinx/io/checkpointing.h | 36 +++++ python/demo/demo_checkpointing.py | 14 +- 4 files changed, 202 insertions(+), 196 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 83980e030d0..0e1d082cb86 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -24,133 +24,60 @@ int main(int argc, char* argv[]) dolfinx::init_logging(argc, argv); MPI_Init(&argc, &argv); - // { - // int i=0; - // while (i == 0) - // sleep(5); - // } - // Create mesh and function space auto part = mesh::create_cell_partitioner(mesh::GhostMode::shared_facet); auto mesh = std::make_shared>(mesh::create_rectangle( MPI_COMM_WORLD, {{{0.0, 0.0}, {1.0, 1.0}}}, {4, 4}, mesh::CellType::quadrilateral, part)); - try { // Set up ADIOS2 IO and Engine - adios2::ADIOS adios("checkpointing.yml", mesh->comm()); + adios2::ADIOS adios(mesh->comm()); adios2::IO io = adios.DeclareIO("mesh-write"); io.SetEngine("BP5"); adios2::Engine engine = io.Open("mesh.bp", adios2::Mode::Write); - std::vector mytags = {"one"}; - io.DefineAttribute("tags", mytags.data(), mytags.size(), "", "", true); - - std::vector mytags2 = {"one", "two"}; - io.DefineAttribute("tags", mytags2.data(), mytags2.size()); - io::native::write_mesh(io, engine, *mesh); engine.Close(); } catch (std::exception &e) { - std::cout << "ERROR: ADIOS2 exception: " << e.what() << "\n"; - MPI_Abort(MPI_COMM_WORLD, -1); + std::cout << "ERROR: ADIOS2 exception: " << e.what() << "\n"; + MPI_Abort(MPI_COMM_WORLD, -1); } - // Set up ADIOS2 IO and Engine - // adios2::ADIOS adios_read(mesh->comm()); - // adios2::IO io_read = adios_read.DeclareIO("mesh-read"); - // io_read.SetEngine("BP5"); - // adios2::Engine engine_read = io_read.Open("mesh.bp", adios2::Mode::Read); - - // auto mesh_read_variant - // = io::native::read_mesh_variant(io_read, engine_read, MPI_COMM_WORLD); - - // engine_read.Close(); - - // ---------------------------------------------------------------------- - // auto adios_query - // = ADIOS2Wrapper(MPI_COMM_WORLD, "mesh.bp", "mesh-read", "BP5", "read"); - - // auto adios_read - // = ADIOS2Wrapper(MPI_COMM_WORLD, "mesh.bp", "mesh-read", "BP5", "read"); - - // Can't use the same engine to query as well as read - // since, in that case BeginStep and EndStep will be called twice - // on a dataset written with a single Step - - // auto reader = adios_read.engine(); - // reader->BeginStep(); - // auto mesh_read_variant = io::native::read_mesh_variant( - // adios_query, adios_read, mesh->comm()); - - // reader->EndStep(); - - // // We cannot resolve easily the variant - // // Hence the user can query the type and - // // call the correct read_mesh - // auto mesh_read = std::get<0>(mesh_read_variant); - // // auto mesh_read = std::get>(mesh_read_variant); - - // auto adios_write - // = ADIOS2Wrapper(mesh->comm(), "mesh2.bp", "mesh-rewrite", "BP5", - // "write"); - - // io::native::write_mesh(adios_write, mesh_read); - - // auto io_query = adios_query.io(); - // auto engine_query = adios_query.engine(); - - // Following throws an error. engine_read->BeginStep() is needed to - // read the VariableType, but then EndStep() fails with message - // EndStep() called without a successful BeginStep() - - // engine_query->BeginStep(); - // std::string floating_point = io_query->VariableType("x"); - // engine_query->EndStep(); - - // // std::string floating_point = - // dolfinx::io::native::query_type(adios_read); std::cout << - // floating_point; - - // // TODO: move type deduction inside checkpointing - // if ("float" == floating_point) - // { - // using T = float; - // mesh::Mesh mesh_read - // = io::native::read_mesh(adios_read, mesh->comm()); - // adios_read.close(); - - // auto adios_write = ADIOS2Wrapper(mesh->comm(), "mesh2.bp", - // "mesh-rewrite", - // "BP5", "write"); - - // auto io_write = adios_write.io(); - - // io::native::write_mesh(adios_write, mesh_read); - // adios_write.close(); - // } - // else if ("double" == floating_point) - // { - // using T = double; - // mesh::Mesh mesh_read - // = io::native::read_mesh(adios_read, mesh->comm()); - // adios_read.close(); - - // auto adios_write = ADIOS2Wrapper(mesh->comm(), "mesh2.bp", - // "mesh-rewrite", - // "BP5", "write"); - - // auto io_write = adios_write.io(); - - // io::native::write_mesh(adios_write, mesh_read); - // adios_write.close(); - // } + try + { + // Set up ADIOS2 IO and Engine + adios2::ADIOS adios_read(MPI_COMM_WORLD); + adios2::IO io_read = adios_read.DeclareIO("mesh-read"); + io_read.SetEngine("BP5"); + adios2::Engine engine_read = io_read.Open("mesh.bp", adios2::Mode::Read); + + engine_read.BeginStep(); + auto mesh_read = io::native::read_mesh(io_read, engine_read, MPI_COMM_WORLD); + if (engine_read.BetweenStepPairs()) + { + engine_read.EndStep(); + } + + engine_read.Close(); + + adios2::IO io_write = adios_read.DeclareIO("mesh-write"); + io_write.SetEngine("BP5"); + adios2::Engine engine_write = io_write.Open("mesh2.bp", adios2::Mode::Write); + + io::native::write_mesh(io_write, engine_write, mesh_read); + engine_write.Close(); + } + catch (std::exception &e) + { + std::cout << "ERROR: ADIOS2 exception: " << e.what() << "\n"; + MPI_Abort(MPI_COMM_WORLD, -1); + } MPI_Finalize(); return 0; diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 46f3a02aae6..66565288757 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -49,7 +49,7 @@ void write_mesh(adios2::IO& io, adios2::Engine& engine, std::int32_t dim = geometry.dim(); std::int32_t tdim = topology->dim(); std::uint64_t num_nodes_global, offset, num_cells_global, cell_offset; - std::uint32_t num_nodes_local, num_cells_local, num_dofs_per_cell, degree; + std::uint32_t num_nodes_local, num_cells_local, degree; std::string cell_type, lagrange_variant; std::vector array_global; std::vector offsets_global; @@ -85,6 +85,7 @@ void write_mesh(adios2::IO& io, adios2::Engine& engine, const std::span mesh_x = geometry.x(); + std::uint32_t num_dofs_per_cell; // Connectivity { auto dofmap = geometry.dofmap(); @@ -100,6 +101,8 @@ void write_mesh(adios2::IO& io, adios2::Engine& engine, geom_imap->local_to_global(connectivity, array_global); offsets_global.resize(num_cells_local + 1); + + // FIXME: use std::ranges::transform for (std::size_t i = 0; i < num_cells_local + 1; ++i) offsets_global[i] = (i + cell_offset) * num_dofs_per_cell; } @@ -119,8 +122,6 @@ void write_mesh(adios2::IO& io, adios2::Engine& engine, = io.DefineVariable("num_nodes"); adios2::Variable var_num_cells = io.DefineVariable("num_cells"); - adios2::Variable var_num_dofs_per_cell - = io.DefineVariable("num_dofs_per_cell"); adios2::Variable var_x = io.DefineVariable("x", {num_nodes_global, 3}, {offset, 0}, @@ -137,14 +138,16 @@ void write_mesh(adios2::IO& io, adios2::Engine& engine, "topology_offsets", {num_cells_global + 1}, {cell_offset}, {num_cells_local + 1}, adios2::ConstantDims); + assert(!engine.BetweenStepPairs()); engine.BeginStep(); engine.Put(var_num_nodes, num_nodes_global); engine.Put(var_num_cells, num_cells_global); - engine.Put(var_num_dofs_per_cell, num_dofs_per_cell); engine.Put(var_x, mesh_x.subspan(0, num_nodes_local * 3).data()); engine.Put(var_topology_array, array_global.data()); engine.Put(var_topology_offsets, offsets_global.data()); engine.EndStep(); + + spdlog::info("Mesh written"); } } @@ -168,11 +171,14 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm_rank(comm, &rank); MPI_Comm_size(comm, &size); - // engine.BeginStep(); + if (!engine.BetweenStepPairs()) + { + engine.BeginStep(); + } // Attributes std::string name; - std::int32_t dim, degree; + std::int32_t dim, tdim, degree; mesh::CellType cell_type; basix::element::lagrange_variant lagrange_variant; @@ -182,6 +188,8 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, = io.InquireAttribute("name"); adios2::Attribute var_dim = io.InquireAttribute("dim"); + adios2::Attribute var_tdim + = io.InquireAttribute("tdim"); adios2::Attribute var_cell_type = io.InquireAttribute("cell_type"); adios2::Attribute var_degree @@ -191,15 +199,18 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, name = var_name.Data()[0]; dim = var_dim.Data()[0]; + tdim = var_tdim.Data()[0]; cell_type = mesh::to_type(var_cell_type.Data()[0]); degree = var_degree.Data()[0]; lagrange_variant = string_to_variant[var_variant.Data()[0]]; } + spdlog::info( + "Reading mesh with geometric dimension: {} and topological dimension: {}", + dim, tdim); + // Scalar variables - std::uint64_t num_nodes_global; - std::uint64_t num_cells_global; - std::uint32_t num_dofs_per_cell; + std::uint64_t num_nodes_global, num_cells_global; // Read scalar variables { @@ -207,82 +218,85 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, = io.InquireVariable("num_nodes"); adios2::Variable var_num_cells = io.InquireVariable("num_cells"); - adios2::Variable var_num_dofs_per_cell - = io.InquireVariable("num_dofs_per_cell"); engine.Get(var_num_nodes, num_nodes_global); engine.Get(var_num_cells, num_cells_global); - engine.Get(var_num_dofs_per_cell, num_dofs_per_cell); - - std::cout << num_nodes_global; } - // Compute local sizes, offsets - std::array _local_range - = dolfinx::MPI::local_range(rank, num_nodes_global, size); + auto [x_reduced, x_shape] = dolfinx::io::impl_native::read_geometry_data( + io, engine, dim, num_nodes_global, rank, size); - std::array local_range{(std::uint64_t)_local_range[0], - (std::uint64_t)_local_range[1]}; - std::uint64_t num_nodes_local = local_range[1] - local_range[0]; + std::vector array = dolfinx::io::impl_native::read_topology_data( + io, engine, num_cells_global, rank, size); - std::array _cell_range - = dolfinx::MPI::local_range(rank, num_cells_global, size); + engine.EndStep(); - std::array cell_range{(std::uint64_t)_cell_range[0], - (std::uint64_t)_cell_range[1]}; - std::uint64_t num_cells_local = cell_range[1] - cell_range[0]; + fem::CoordinateElement element + = fem::CoordinateElement(cell_type, degree, lagrange_variant); - std::vector input_global_indices(num_nodes_local); - std::vector x(num_nodes_local * 3); - std::vector array(num_cells_local * num_dofs_per_cell); - std::vector offsets(num_cells_local + 1); + auto part = mesh::create_cell_partitioner(mesh::GhostMode::shared_facet); - { - adios2::Variable var_input_global_indices - = io.InquireVariable("input_global_indices"); + mesh::Mesh mesh = mesh::create_mesh(comm, comm, array, element, comm, + x_reduced, x_shape, part); - adios2::Variable var_x = io.InquireVariable("x"); + mesh.name = name; - adios2::Variable var_topology_array - = io.InquireVariable("topology_array"); + return mesh; +} - adios2::Variable var_topology_offsets - = io.InquireVariable("topology_offsets"); +//----------------------------------------------------------------------------- +/// @cond +template dolfinx::mesh::Mesh +read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm); - if (var_input_global_indices) - { - var_input_global_indices.SetSelection( - {{local_range[0]}, {num_nodes_local}}); - engine.Get(var_input_global_indices, input_global_indices.data(), - adios2::Mode::Deferred); - } +template dolfinx::mesh::Mesh +read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm); - if (var_x) - { - var_x.SetSelection({{local_range[0], 0}, {num_nodes_local, 3}}); - engine.Get(var_x, x.data(), adios2::Mode::Deferred); - } +/// @endcond - if (var_topology_array) - { - var_topology_array.SetSelection({{cell_range[0] * num_dofs_per_cell}, - {cell_range[1] * num_dofs_per_cell}}); - engine.Get(var_topology_array, array.data(), adios2::Mode::Deferred); - } +} // namespace dolfinx::io::native - if (var_topology_offsets) - { - var_topology_offsets.SetSelection({{cell_range[0]}, {cell_range[1] + 1}}); - engine.Get(var_topology_offsets, offsets.data(), adios2::Mode::Deferred); - } +namespace dolfinx::io::impl_native +{ +//----------------------------------------------------------------------------- +std::pair get_counters(int rank, std::uint64_t N, + int size) +{ + assert(rank >= 0); + assert(size > 0); - std::int32_t cell_offset = offsets[0]; - for (auto offset = offsets.begin(); offset != offsets.end(); ++offset) - *offset -= cell_offset; - } + // Compute number of items per rank and remainder + const std::uint64_t n = N / size; + const std::uint64_t r = N % size; - // engine.EndStep(); + // Compute local range + if (static_cast(rank) < r) + return {rank * (n + 1), n + 1}; + else + return {rank * n + r, n}; +} + +//----------------------------------------------------------------------------- +template +std::pair, std::array> +read_geometry_data(adios2::IO& io, adios2::Engine& engine, int dim, + std::uint64_t num_nodes_global, int rank, int size) +{ + auto [nodes_offset, num_nodes_local] + = dolfinx::io::impl_native::get_counters(rank, num_nodes_global, size); + std::vector x(num_nodes_local * 3); + adios2::Variable var_x = io.InquireVariable("x"); + if (var_x) + { + var_x.SetSelection({{nodes_offset, 0}, {num_nodes_local, 3}}); + engine.Get(var_x, x.data(), adios2::Mode::Sync); + } + else + { + throw std::runtime_error("Coordinates data not found"); + } + // FIXME: Use std::ranges::transform std::vector x_reduced(num_nodes_local * dim); for (std::uint32_t i = 0; i < num_nodes_local; ++i) { @@ -290,31 +304,54 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, x_reduced[i * dim + j] = x[i * 3 + j]; } - fem::CoordinateElement element - = fem::CoordinateElement(cell_type, degree, lagrange_variant); - std::array xshape = {num_nodes_local, (std::uint32_t)dim}; - auto part = mesh::create_cell_partitioner(mesh::GhostMode::shared_facet); - mesh::Mesh mesh = mesh::create_mesh(comm, comm, array, element, comm, - x_reduced, xshape, part); - return mesh; + return {std::move(x_reduced), xshape}; } //----------------------------------------------------------------------------- -/// @cond -template dolfinx::mesh::Mesh -read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm); +std::vector read_topology_data(adios2::IO& io, adios2::Engine& engine, + std::uint64_t num_cells_global, + int rank, int size) +{ + auto [cells_offset, num_cells_local] + = dolfinx::io::impl_native::get_counters(rank, num_cells_global, size); + std::vector offsets(num_cells_local + 1); -template dolfinx::mesh::Mesh -read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm); + adios2::Variable var_topology_array + = io.InquireVariable("topology_array"); -/// @endcond + adios2::Variable var_topology_offsets + = io.InquireVariable("topology_offsets"); -} // namespace dolfinx::io::native + if (var_topology_offsets) + { + var_topology_offsets.SetSelection({{cells_offset}, {num_cells_local + 1}}); + engine.Get(var_topology_offsets, offsets.data(), adios2::Mode::Sync); + } + else + { + throw std::runtime_error("Topology offsets not found"); + } + + std::uint64_t count + = static_cast(offsets[num_cells_local] - offsets[0]); + std::vector array(count); + + if (var_topology_array) + { + var_topology_array.SetSelection( + {{static_cast(offsets[0])}, {count}}); + engine.Get(var_topology_array, array.data(), adios2::Mode::Sync); + } + else + { + throw std::runtime_error("Topology array not found"); + } + + return array; +} -namespace dolfinx::io::impl_native -{ //----------------------------------------------------------------------------- std::variant, dolfinx::mesh::Mesh> read_mesh_variant(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm) @@ -326,19 +363,25 @@ read_mesh_variant(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm) { dolfinx::mesh::Mesh mesh = dolfinx::io::native::read_mesh(io, engine, comm); - engine.EndStep(); + if (engine.BetweenStepPairs()) + { + engine.EndStep(); + } return mesh; } else if (floating_point == "double") { dolfinx::mesh::Mesh mesh = dolfinx::io::native::read_mesh(io, engine, comm); - engine.EndStep(); + if (engine.BetweenStepPairs()) + { + engine.EndStep(); + } return mesh; } else { - throw std::runtime_error("Floating point type is neither float or double"); + throw std::runtime_error("Floating point type is neither float nor double"); } } diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 25cc5faa92f..3337f6b37b2 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -43,6 +43,42 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, namespace dolfinx::io::impl_native { +/// @brief Find offset and size. +/// +/// @param[in] rank MPI rank +/// @param[in] N size of data to distribute +/// @param[in] size MPI size +/// @return start and count +std::pair get_counters(int rank, std::uint64_t N, + int size); + +/// @brief Read geometry data +/// +/// @param[in] io ADIOS2 IO +/// @param[in] engine ADIOS2 Engine +/// @param[in] dim The geometric dimension (`0 < dim <= 3`). +/// @param[in] num_nodes_global size of the global array of nodes +/// @param[in] rank MPI rank +/// @param[in] size MPI size +/// @return The point coordinates of row-major storage and +/// itsshape `(num_nodes_local, dim)` +template +std::pair, std::array> +read_geometry_data(adios2::IO& io, adios2::Engine& engine, int dim, + std::uint64_t num_nodes_global, int rank, int size); + +/// @brief Read topology array +/// +/// @param[in] io ADIOS2 IO +/// @param[in] engine ADIOS2 Engine +/// @param[in] num_cells_global global number of cells +/// @param[in] rank MPI rank +/// @param[in] size MPI size +/// @return The cell-to-node connectivity in a flattened array +std::vector read_topology_data(adios2::IO& io, adios2::Engine& engine, + std::uint64_t num_cells_global, + int rank, int size); + /// @brief Read mesh from a file. /// /// @param[in] io ADIOS2 IO diff --git a/python/demo/demo_checkpointing.py b/python/demo/demo_checkpointing.py index 8720f37ae22..b4f1d6a9933 100644 --- a/python/demo/demo_checkpointing.py +++ b/python/demo/demo_checkpointing.py @@ -67,12 +67,12 @@ # - # + -# msh_read = io.read_mesh(adios2_read, msh.comm) +msh_read = io.read_mesh(adios2_read, msh.comm) # - -# # + -# print(type(msh_read)) -# print(msh_read.name) -# print(msh_read.geometry.x.dtype) -# print(msh_read.geometry.x) -# # - +# + +print(type(msh_read)) +print(msh_read.name) +print(msh_read.geometry.x.dtype) +print(msh_read.geometry.x) +# - From 77f059f60234cfdfd1cefc4353c661e557df9ecf Mon Sep 17 00:00:00 2001 From: ampdes Date: Wed, 14 Aug 2024 10:59:55 +0200 Subject: [PATCH 50/60] Parametrize ghost_mode; add cpp demo to doc --- cpp/demo/checkpointing/main.cpp | 82 ++++++++++++++++---------------- cpp/doc/source/demo.rst | 1 + cpp/dolfinx/io/checkpointing.cpp | 21 +++++--- cpp/dolfinx/io/checkpointing.h | 10 +++- python/dolfinx/io/utils.py | 6 ++- python/dolfinx/wrappers/io.cpp | 10 ++-- 6 files changed, 75 insertions(+), 55 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 0e1d082cb86..91f1b60d289 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -30,54 +30,56 @@ int main(int argc, char* argv[]) MPI_COMM_WORLD, {{{0.0, 0.0}, {1.0, 1.0}}}, {4, 4}, mesh::CellType::quadrilateral, part)); - try - { - // Set up ADIOS2 IO and Engine - adios2::ADIOS adios(mesh->comm()); + try + { + // Set up ADIOS2 IO and Engine + adios2::ADIOS adios(mesh->comm()); - adios2::IO io = adios.DeclareIO("mesh-write"); - io.SetEngine("BP5"); - adios2::Engine engine = io.Open("mesh.bp", adios2::Mode::Write); + adios2::IO io = adios.DeclareIO("mesh-write"); + io.SetEngine("BP5"); + adios2::Engine engine = io.Open("mesh.bp", adios2::Mode::Write); - io::native::write_mesh(io, engine, *mesh); + io::native::write_mesh(io, engine, *mesh); - engine.Close(); - } - catch (std::exception &e) - { - std::cout << "ERROR: ADIOS2 exception: " << e.what() << "\n"; - MPI_Abort(MPI_COMM_WORLD, -1); - } + engine.Close(); + } + catch (std::exception& e) + { + std::cout << "ERROR: ADIOS2 exception: " << e.what() << "\n"; + MPI_Abort(MPI_COMM_WORLD, -1); + } - try - { - // Set up ADIOS2 IO and Engine - adios2::ADIOS adios_read(MPI_COMM_WORLD); - adios2::IO io_read = adios_read.DeclareIO("mesh-read"); - io_read.SetEngine("BP5"); - adios2::Engine engine_read = io_read.Open("mesh.bp", adios2::Mode::Read); + try + { + // Set up ADIOS2 IO and Engine + adios2::ADIOS adios_read(MPI_COMM_WORLD); + adios2::IO io_read = adios_read.DeclareIO("mesh-read"); + io_read.SetEngine("BP5"); + adios2::Engine engine_read = io_read.Open("mesh.bp", adios2::Mode::Read); - engine_read.BeginStep(); - auto mesh_read = io::native::read_mesh(io_read, engine_read, MPI_COMM_WORLD); - if (engine_read.BetweenStepPairs()) - { - engine_read.EndStep(); - } + engine_read.BeginStep(); + auto mesh_read + = io::native::read_mesh(io_read, engine_read, MPI_COMM_WORLD); + if (engine_read.BetweenStepPairs()) + { + engine_read.EndStep(); + } - engine_read.Close(); + engine_read.Close(); - adios2::IO io_write = adios_read.DeclareIO("mesh-write"); - io_write.SetEngine("BP5"); - adios2::Engine engine_write = io_write.Open("mesh2.bp", adios2::Mode::Write); + adios2::IO io_write = adios_read.DeclareIO("mesh-write"); + io_write.SetEngine("BP5"); + adios2::Engine engine_write + = io_write.Open("mesh2.bp", adios2::Mode::Write); - io::native::write_mesh(io_write, engine_write, mesh_read); - engine_write.Close(); - } - catch (std::exception &e) - { - std::cout << "ERROR: ADIOS2 exception: " << e.what() << "\n"; - MPI_Abort(MPI_COMM_WORLD, -1); - } + io::native::write_mesh(io_write, engine_write, mesh_read); + engine_write.Close(); + } + catch (std::exception& e) + { + std::cout << "ERROR: ADIOS2 exception: " << e.what() << "\n"; + MPI_Abort(MPI_COMM_WORLD, -1); + } MPI_Finalize(); return 0; diff --git a/cpp/doc/source/demo.rst b/cpp/doc/source/demo.rst index 7ce9d8d80db..96b21104885 100644 --- a/cpp/doc/source/demo.rst +++ b/cpp/doc/source/demo.rst @@ -42,3 +42,4 @@ Experimental :maxdepth: 1 demos/demo_mixed_topology.md + demos/demo_checkpointing.md diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 66565288757..b7c924a5615 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -164,7 +164,8 @@ template void write_mesh(adios2::IO& io, adios2::Engine& engine, //----------------------------------------------------------------------------- template dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, - MPI_Comm comm) + MPI_Comm comm, + dolfinx::mesh::GhostMode ghost_mode) { int rank, size; @@ -234,7 +235,10 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, fem::CoordinateElement element = fem::CoordinateElement(cell_type, degree, lagrange_variant); - auto part = mesh::create_cell_partitioner(mesh::GhostMode::shared_facet); + auto part = mesh::create_cell_partitioner(ghost_mode); + + if (size == 1) + part = nullptr; mesh::Mesh mesh = mesh::create_mesh(comm, comm, array, element, comm, x_reduced, x_shape, part); @@ -247,10 +251,12 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, //----------------------------------------------------------------------------- /// @cond template dolfinx::mesh::Mesh -read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm); +read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm, + dolfinx::mesh::GhostMode ghost_mode); template dolfinx::mesh::Mesh -read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm); +read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm, + dolfinx::mesh::GhostMode ghost_mode); /// @endcond @@ -354,7 +360,8 @@ std::vector read_topology_data(adios2::IO& io, adios2::Engine& engine, //----------------------------------------------------------------------------- std::variant, dolfinx::mesh::Mesh> -read_mesh_variant(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm) +read_mesh_variant(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm, + dolfinx::mesh::GhostMode ghost_mode) { engine.BeginStep(); std::string floating_point = io.VariableType("x"); @@ -362,7 +369,7 @@ read_mesh_variant(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm) if (floating_point == "float") { dolfinx::mesh::Mesh mesh - = dolfinx::io::native::read_mesh(io, engine, comm); + = dolfinx::io::native::read_mesh(io, engine, comm, ghost_mode); if (engine.BetweenStepPairs()) { engine.EndStep(); @@ -372,7 +379,7 @@ read_mesh_variant(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm) else if (floating_point == "double") { dolfinx::mesh::Mesh mesh - = dolfinx::io::native::read_mesh(io, engine, comm); + = dolfinx::io::native::read_mesh(io, engine, comm, ghost_mode); if (engine.BetweenStepPairs()) { engine.EndStep(); diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 3337f6b37b2..7aa5a4e52f9 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -12,6 +12,7 @@ #include #include #include +#include #include /// @file checkpointing.h @@ -37,7 +38,9 @@ void write_mesh(adios2::IO& io, adios2::Engine& engine, /// @return mesh reconstructed from the data template dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, - MPI_Comm comm = MPI_COMM_WORLD); + MPI_Comm comm = MPI_COMM_WORLD, + dolfinx::mesh::GhostMode ghost_mode + = dolfinx::mesh::GhostMode::shared_facet); } // namespace dolfinx::io::native @@ -84,10 +87,13 @@ std::vector read_topology_data(adios2::IO& io, adios2::Engine& engine, /// @param[in] io ADIOS2 IO /// @param[in] engine ADIOS2 Engine /// @param[in] comm comm +/// @param[in] ghost_mode The requested type of cell ghosting/overlap /// @return mesh reconstructed from the data std::variant, dolfinx::mesh::Mesh> read_mesh_variant(adios2::IO& io, adios2::Engine& engine, - MPI_Comm comm = MPI_COMM_WORLD); + MPI_Comm comm = MPI_COMM_WORLD, + dolfinx::mesh::GhostMode ghost_mode + = dolfinx::mesh::GhostMode::shared_facet); } // namespace dolfinx::io::impl_native diff --git a/python/dolfinx/io/utils.py b/python/dolfinx/io/utils.py index 919006da777..50a563f61f7 100644 --- a/python/dolfinx/io/utils.py +++ b/python/dolfinx/io/utils.py @@ -218,9 +218,11 @@ def write_mesh(ADIOS2: ADIOS2, mesh: Mesh) -> None: return _writer(ADIOS2, mesh._cpp_object) - def read_mesh(ADIOS2: ADIOS2, comm: _MPI.Comm) -> Mesh: + def read_mesh( + ADIOS2: ADIOS2, comm: _MPI.Comm, ghost_mode: GhostMode = GhostMode.shared_facet + ) -> Mesh: """Read mesh from a file using ADIOS2""" - msh = _cpp.io.read_mesh(ADIOS2, comm) + msh = _cpp.io.read_mesh(ADIOS2, comm, ghost_mode) return msh diff --git a/python/dolfinx/wrappers/io.cpp b/python/dolfinx/wrappers/io.cpp index df656ccbd8b..62c7440d483 100644 --- a/python/dolfinx/wrappers/io.cpp +++ b/python/dolfinx/wrappers/io.cpp @@ -289,14 +289,16 @@ void io(nb::module_& m) // dolfinx::io::impl_native::read_mesh_variant m.def( "read_mesh", - [](dolfinx::io::ADIOS2Wrapper& ADIOS2, MPICommWrapper comm) + [](dolfinx::io::ADIOS2Wrapper& ADIOS2, MPICommWrapper comm, + dolfinx::mesh::GhostMode ghost_mode) { auto io = ADIOS2.io(); auto engine = ADIOS2.engine(); - return dolfinx::io::impl_native::read_mesh_variant(*io, *engine, - comm.get()); + return dolfinx::io::impl_native::read_mesh_variant( + *io, *engine, comm.get(), ghost_mode); }, - nb::arg("adios2"), nb::arg("comm"), "Read mesh from file using ADIOS2"); + nb::arg("adios2"), nb::arg("comm"), nb::arg("ghost_mode"), + "Read mesh from file using ADIOS2"); declare_write_mesh(m, "float32"); declare_write_mesh(m, "float64"); From 64480e211d6c49c824b0694138975419a35bf99b Mon Sep 17 00:00:00 2001 From: ampdes Date: Fri, 16 Aug 2024 07:46:20 +0200 Subject: [PATCH 51/60] Add test; add ufl_domain to the read_mesh --- cpp/dolfinx/io/checkpointing.h | 1 + python/demo/demo_checkpointing.py | 7 ---- python/dolfinx/io/utils.py | 12 +++++- python/test/unit/io/test_adios2.py | 61 +++++++++++++++++++++++++++++- 4 files changed, 71 insertions(+), 10 deletions(-) diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 7aa5a4e52f9..b006a926566 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -35,6 +35,7 @@ void write_mesh(adios2::IO& io, adios2::Engine& engine, /// @param[in] io ADIOS2 IO /// @param[in] engine ADIOS2 Engine /// @param[in] comm comm +/// @param[in] ghost_mode The requested type of cell ghosting/overlap /// @return mesh reconstructed from the data template dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, diff --git a/python/demo/demo_checkpointing.py b/python/demo/demo_checkpointing.py index b4f1d6a9933..08476fce89b 100644 --- a/python/demo/demo_checkpointing.py +++ b/python/demo/demo_checkpointing.py @@ -69,10 +69,3 @@ # + msh_read = io.read_mesh(adios2_read, msh.comm) # - - -# + -print(type(msh_read)) -print(msh_read.name) -print(msh_read.geometry.x.dtype) -print(msh_read.geometry.x) -# - diff --git a/python/dolfinx/io/utils.py b/python/dolfinx/io/utils.py index 50a563f61f7..bc2fa04add4 100644 --- a/python/dolfinx/io/utils.py +++ b/python/dolfinx/io/utils.py @@ -224,7 +224,17 @@ def read_mesh( """Read mesh from a file using ADIOS2""" msh = _cpp.io.read_mesh(ADIOS2, comm, ghost_mode) - return msh + element = basix.ufl.element( + basix.ElementFamily.P, + basix.CellType[msh.topology.cell_name()], + msh.geometry.cmap.degree, + basix.LagrangeVariant(int(msh.geometry.cmap.variant)), + shape=(msh.geometry.x.shape[1],), + dtype=msh.geometry.x.dtype, + ) + domain = ufl.Mesh(element) + + return Mesh(msh, domain) class VTKFile(_cpp.io.VTKFile): diff --git a/python/test/unit/io/test_adios2.py b/python/test/unit/io/test_adios2.py index 980558415dd..f82b3107957 100644 --- a/python/test/unit/io/test_adios2.py +++ b/python/test/unit/io/test_adios2.py @@ -14,9 +14,10 @@ import ufl from basix.ufl import element from dolfinx import default_real_type, default_scalar_type -from dolfinx.fem import Function, functionspace +from dolfinx.fem import Function, assemble_scalar, form, functionspace from dolfinx.graph import adjacencylist -from dolfinx.mesh import CellType, create_mesh, create_unit_cube, create_unit_square +from dolfinx.io import ADIOS2, read_mesh, write_mesh +from dolfinx.mesh import CellType, GhostMode, create_mesh, create_unit_cube, create_unit_square def generate_mesh(dim: int, simplex: bool, N: int = 5, dtype=None): @@ -38,6 +39,62 @@ def generate_mesh(dim: int, simplex: bool, N: int = 5, dtype=None): raise RuntimeError("Unsupported dimension") +# TODO: Fix problems with ("HDF5", ".h5") +@pytest.mark.adios2 +@pytest.mark.parametrize("encoder, suffix", [("BP4", ".bp"), ("BP5", ".bp")]) +@pytest.mark.parametrize("ghost_mode", [GhostMode.shared_facet, GhostMode.none]) +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +@pytest.mark.parametrize("dim", [2, 3]) +@pytest.mark.parametrize("simplex", [True, False]) +def test_mesh_read_write(encoder, suffix, ghost_mode, dtype, dim, simplex, tmp_path): + N = 5 + # Consistent tmp dir across processes + fname = MPI.COMM_WORLD.bcast(tmp_path, root=0) + file = fname / f"adios_mesh_{encoder}" + + mesh = generate_mesh(dim, simplex, N, dtype) + + adios = ADIOS2( + mesh.comm, + filename=str(file.with_suffix(suffix)), + tag="mesh-write", + engine_type=encoder, + mode="write", + ) + + write_mesh(adios, mesh) + + adios_read = ADIOS2( + MPI.COMM_WORLD, + filename=str(file.with_suffix(suffix)), + tag="mesh-read", + engine_type=encoder, + mode="read", + ) + + mesh_adios = read_mesh(adios_read, MPI.COMM_WORLD, ghost_mode=ghost_mode) + + mesh_adios.comm.Barrier() + mesh.comm.Barrier() + + for i in range(mesh.topology.dim + 1): + mesh.topology.create_entities(i) + mesh_adios.topology.create_entities(i) + assert ( + mesh.topology.index_map(i).size_global == mesh_adios.topology.index_map(i).size_global + ) + + # Check that integration over different entities are consistent + measures = [ufl.ds, ufl.dx] if ghost_mode is GhostMode.none else [ufl.ds, ufl.dS, ufl.dx] + for measure in measures: + c_adios = assemble_scalar(form(1 * measure(domain=mesh_adios), dtype=dtype)) + c_ref = assemble_scalar(form(1 * measure(domain=mesh), dtype=dtype)) + assert np.isclose( + mesh_adios.comm.allreduce(c_adios, MPI.SUM), + mesh.comm.allreduce(c_ref, MPI.SUM), + ) + + @pytest.mark.adios2 class TestFides: @pytest.mark.parametrize("dim", [2, 3]) From 61a8288d02c2c49a9c27519e0acafd074da5b1d5 Mon Sep 17 00:00:00 2001 From: ampdes Date: Fri, 16 Aug 2024 08:21:08 +0200 Subject: [PATCH 52/60] Fix import errors --- python/demo/demo_checkpointing.py | 11 +++-------- python/test/unit/io/test_adios2.py | 4 +++- python/test/unit/io/test_xdmf_meshtags.py | 6 +++--- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/python/demo/demo_checkpointing.py b/python/demo/demo_checkpointing.py index 08476fce89b..b78993994af 100644 --- a/python/demo/demo_checkpointing.py +++ b/python/demo/demo_checkpointing.py @@ -15,17 +15,12 @@ # # + -import importlib.util import os -if importlib.util.find_spec("adios2") is not None: - import dolfinx +import dolfinx - if not dolfinx.has_adios2: - print("This demo requires DOLFINx to be compiled with ADIOS2 enabled.") - exit(0) -else: - print("This demo requires ADIOS2.") +if not dolfinx.common.has_adios2: + print("This demo requires DOLFINx to be compiled with ADIOS2 enabled.") exit(0) from mpi4py import MPI diff --git a/python/test/unit/io/test_adios2.py b/python/test/unit/io/test_adios2.py index f82b3107957..1fa29ef2de7 100644 --- a/python/test/unit/io/test_adios2.py +++ b/python/test/unit/io/test_adios2.py @@ -16,7 +16,6 @@ from dolfinx import default_real_type, default_scalar_type from dolfinx.fem import Function, assemble_scalar, form, functionspace from dolfinx.graph import adjacencylist -from dolfinx.io import ADIOS2, read_mesh, write_mesh from dolfinx.mesh import CellType, GhostMode, create_mesh, create_unit_cube, create_unit_square @@ -47,6 +46,9 @@ def generate_mesh(dim: int, simplex: bool, N: int = 5, dtype=None): @pytest.mark.parametrize("dim", [2, 3]) @pytest.mark.parametrize("simplex", [True, False]) def test_mesh_read_write(encoder, suffix, ghost_mode, dtype, dim, simplex, tmp_path): + "Test writing of a mesh" + from dolfinx.io import ADIOS2, read_mesh, write_mesh + N = 5 # Consistent tmp dir across processes fname = MPI.COMM_WORLD.bcast(tmp_path, root=0) diff --git a/python/test/unit/io/test_xdmf_meshtags.py b/python/test/unit/io/test_xdmf_meshtags.py index fd777a75b46..d8d18f48da5 100644 --- a/python/test/unit/io/test_xdmf_meshtags.py +++ b/python/test/unit/io/test_xdmf_meshtags.py @@ -5,7 +5,7 @@ # SPDX-License-Identifier: LGPL-3.0-or-later from pathlib import Path -from xml.etree import ElementTree +from xml.etree import ElementTree as ET from mpi4py import MPI @@ -93,8 +93,8 @@ def test_3d(tempdir, cell_type, encoding): facets_local = comm.allreduce( (mt.indices < mesh.topology.index_map(2).size_local).sum(), op=MPI.SUM ) - parser = ElementTree.XMLParser() - tree = ElementTree.parse(Path(tempdir, "meshtags_3d_out.xdmf"), parser) + parser = ET.XMLParser() + tree = ET.parse(Path(tempdir, "meshtags_3d_out.xdmf"), parser) num_lines = int(tree.findall(".//Grid[@Name='lines']/Topology")[0].get("NumberOfElements")) num_facets = int(tree.findall(".//Grid[@Name='facets']/Topology")[0].get("NumberOfElements")) assert num_lines == lines_local From e988a0a11b6bfec962c00c9072995986d20a9fcd Mon Sep 17 00:00:00 2001 From: ampdes Date: Sat, 17 Aug 2024 15:42:23 +0200 Subject: [PATCH 53/60] Add version compatibility check Add Append adios mode Add complete set of basix enum-string maps --- cpp/demo/checkpointing/main.cpp | 2 +- cpp/dolfinx/io/ADIOS2_utils.h | 3 ++- cpp/dolfinx/io/checkpointing.cpp | 42 ++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 91f1b60d289..06b9ad00d1e 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -37,7 +37,7 @@ int main(int argc, char* argv[]) adios2::IO io = adios.DeclareIO("mesh-write"); io.SetEngine("BP5"); - adios2::Engine engine = io.Open("mesh.bp", adios2::Mode::Write); + adios2::Engine engine = io.Open("mesh.bp", adios2::Mode::Append); io::native::write_mesh(io, engine, *mesh); diff --git a/cpp/dolfinx/io/ADIOS2_utils.h b/cpp/dolfinx/io/ADIOS2_utils.h index b8613456508..297dd38ab73 100644 --- a/cpp/dolfinx/io/ADIOS2_utils.h +++ b/cpp/dolfinx/io/ADIOS2_utils.h @@ -21,6 +21,7 @@ namespace std::map string_to_mode{ {"write", adios2::Mode::Write}, {"read", adios2::Mode::Read}, + {"append", adios2::Mode::Append}, }; } // namespace @@ -55,7 +56,7 @@ class ADIOS2Wrapper /// @param[in] tag The ADIOS2 IO name /// @param[in] mode ADIOS2 mode, default is Write or Read ADIOS2Wrapper(std::string config_file, MPI_Comm comm, std::string filename, - std::string tag, std::string mode = "write") + std::string tag, std::string mode = "append") { _adios = std::make_shared(config_file, comm); _io = std::make_shared(_adios->DeclareIO(tag)); diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index b7c924a5615..8fbed23a51e 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -19,16 +19,36 @@ /// @brief ADIOS2 based checkpointing namespace { +/// basix enum to string std::map variant_to_string{ {basix::element::lagrange_variant::unset, "unset"}, {basix::element::lagrange_variant::equispaced, "equispaced"}, {basix::element::lagrange_variant::gll_warped, "gll_warped"}, + {basix::element::lagrange_variant::gll_isaac, "gll_isaac"}, + {basix::element::lagrange_variant::gll_centroid, "gll_centroid"}, + {basix::element::lagrange_variant::chebyshev_warped, "chebyshev_warped"}, + {basix::element::lagrange_variant::chebyshev_isaac, "chebyshev_isaac"}, + {basix::element::lagrange_variant::gl_warped, "gl_warped"}, + {basix::element::lagrange_variant::gl_isaac, "gl_isaac"}, + {basix::element::lagrange_variant::gl_centroid, "gl_centroid"}, + {basix::element::lagrange_variant::legendre, "legendre"}, + {basix::element::lagrange_variant::bernstein, "bernstein"}, }; +/// string to basix enum std::map string_to_variant{ {"unset", basix::element::lagrange_variant::unset}, {"equispaced", basix::element::lagrange_variant::equispaced}, {"gll_warped", basix::element::lagrange_variant::gll_warped}, + {"gll_isaac", basix::element::lagrange_variant::gll_isaac}, + {"gll_centroid", basix::element::lagrange_variant::gll_centroid}, + {"chebyshev_warped", basix::element::lagrange_variant::chebyshev_warped}, + {"chebyshev_isaac", basix::element::lagrange_variant::chebyshev_isaac}, + {"gl_warped", basix::element::lagrange_variant::gl_warped}, + {"gl_isaac", basix::element::lagrange_variant::gl_isaac}, + {"gl_centroid", basix::element::lagrange_variant::gl_centroid}, + {"legendre", basix::element::lagrange_variant::legendre}, + {"bernstein", basix::element::lagrange_variant::bernstein}, }; } // namespace @@ -177,6 +197,27 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, engine.BeginStep(); } + // Compatibility check for version and git commit hash + { + adios2::Attribute var_version + = io.InquireAttribute("version"); + adios2::Attribute var_hash + = io.InquireAttribute("git_hash"); + std::string version = var_version.Data()[0]; + if (version != dolfinx::version()) + { + throw std::runtime_error("Reading from version: " + dolfinx::version() + + " written with version: " + version); + } + + std::string git_hash = var_hash.Data()[0]; + if (git_hash != dolfinx::git_commit_hash()) + { + throw std::runtime_error("Reading from GIT_COMMIT_HASH: " + + dolfinx::git_commit_hash() + + " written with GIT_COMMIT_HASH: " + git_hash); + } + } // Attributes std::string name; std::int32_t dim, tdim, degree; @@ -230,6 +271,7 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, std::vector array = dolfinx::io::impl_native::read_topology_data( io, engine, num_cells_global, rank, size); + assert(engine.BetweenStepPairs()); engine.EndStep(); fem::CoordinateElement element From 2cfebf2edfcbf3caf0f6f2f385c64b972bb0ea54 Mon Sep 17 00:00:00 2001 From: ampdes Date: Mon, 19 Aug 2024 18:02:19 +0200 Subject: [PATCH 54/60] Reorganize namespaces --- cpp/dolfinx/io/checkpointing.h | 54 +++++++++++++++++----------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index b006a926566..4646e226128 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -18,33 +18,6 @@ /// @file checkpointing.h /// @brief ADIOS2 based checkpointing -namespace dolfinx::io::native -{ - -/// @brief Write mesh to a file. -/// -/// @param[in] io ADIOS2 IO -/// @param[in] engine ADIOS2 Engine -/// @param[in] mesh Mesh of type float or double to write to the file -template -void write_mesh(adios2::IO& io, adios2::Engine& engine, - const dolfinx::mesh::Mesh& mesh); - -/// @brief Read mesh from a file. -/// -/// @param[in] io ADIOS2 IO -/// @param[in] engine ADIOS2 Engine -/// @param[in] comm comm -/// @param[in] ghost_mode The requested type of cell ghosting/overlap -/// @return mesh reconstructed from the data -template -dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, - MPI_Comm comm = MPI_COMM_WORLD, - dolfinx::mesh::GhostMode ghost_mode - = dolfinx::mesh::GhostMode::shared_facet); - -} // namespace dolfinx::io::native - namespace dolfinx::io::impl_native { /// @brief Find offset and size. @@ -98,4 +71,31 @@ read_mesh_variant(adios2::IO& io, adios2::Engine& engine, } // namespace dolfinx::io::impl_native +namespace dolfinx::io::native +{ + +/// @brief Write mesh to a file. +/// +/// @param[in] io ADIOS2 IO +/// @param[in] engine ADIOS2 Engine +/// @param[in] mesh Mesh of type float or double to write to the file +template +void write_mesh(adios2::IO& io, adios2::Engine& engine, + const dolfinx::mesh::Mesh& mesh); + +/// @brief Read mesh from a file. +/// +/// @param[in] io ADIOS2 IO +/// @param[in] engine ADIOS2 Engine +/// @param[in] comm comm +/// @param[in] ghost_mode The requested type of cell ghosting/overlap +/// @return mesh reconstructed from the data +template +dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, + MPI_Comm comm = MPI_COMM_WORLD, + dolfinx::mesh::GhostMode ghost_mode + = dolfinx::mesh::GhostMode::shared_facet); + +} // namespace dolfinx::io::native + #endif From 2688e51ca4ec105cedee3baa53714b6f0e3e0343 Mon Sep 17 00:00:00 2001 From: ampdes Date: Thu, 22 Aug 2024 22:35:18 +0200 Subject: [PATCH 55/60] Redesign ADIOS2Wrapper to hold multiple IO and Engine --- cpp/dolfinx/io/ADIOS2_utils.h | 85 ++++++++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 15 deletions(-) diff --git a/cpp/dolfinx/io/ADIOS2_utils.h b/cpp/dolfinx/io/ADIOS2_utils.h index 297dd38ab73..264eb8022cd 100644 --- a/cpp/dolfinx/io/ADIOS2_utils.h +++ b/cpp/dolfinx/io/ADIOS2_utils.h @@ -22,6 +22,7 @@ std::map string_to_mode{ {"write", adios2::Mode::Write}, {"read", adios2::Mode::Read}, {"append", adios2::Mode::Append}, + {"readrandomaccess", adios2::Mode::ReadRandomAccess}, }; } // namespace @@ -32,6 +33,14 @@ namespace dolfinx::io class ADIOS2Wrapper { public: + /// @brief Create an ADIOS2-based engine writer/reader + /// @param[in] comm The MPI communicator + /// @param[in] filename Name of output file + ADIOS2Wrapper(MPI_Comm comm, std::string filename) + : _filename(filename), _adios(std::make_shared(comm)) + { + } + /// @brief Create an ADIOS2-based engine writer/reader /// @param[in] comm The MPI communicator /// @param[in] filename Name of output file @@ -41,22 +50,23 @@ class ADIOS2Wrapper /// @param[in] mode ADIOS2 mode, default is Write or Read ADIOS2Wrapper(MPI_Comm comm, std::string filename, std::string tag, std::string engine_type = "BP5", std::string mode = "write") - : _adios(std::make_shared(comm)), - _io(std::make_shared(_adios->DeclareIO(tag))) + : _filename(filename), _adios(std::make_shared(comm)) { - _io->SetEngine(engine_type); - _engine = std::make_shared( - _io->Open(filename, string_to_mode[mode])); + _ios.insert({tag, std::make_shared(_adios->DeclareIO(tag))}); + _ios[tag]->SetEngine(engine_type); + _engines.insert(tag, std::make_shared( + _ios[tag]->Open(filename, string_to_mode[mode]))); } /// @brief Create an ADIOS2-based engine writer/reader - /// @param[in] config_file Path to config file to set up ADIOS2 engine from + /// @param[in] config_file Path to config file to set up ADIOS2 engines from /// @param[in] comm The MPI communicator /// @param[in] filename Name of output file /// @param[in] tag The ADIOS2 IO name /// @param[in] mode ADIOS2 mode, default is Write or Read ADIOS2Wrapper(std::string config_file, MPI_Comm comm, std::string filename, std::string tag, std::string mode = "append") + : _filename(filename) { _adios = std::make_shared(config_file, comm); _io = std::make_shared(_adios->DeclareIO(tag)); @@ -79,24 +89,69 @@ class ADIOS2Wrapper // Copy assignment ADIOS2Wrapper& operator=(const ADIOS2Wrapper&) = delete; - /// @brief Close the file + /// @brief Add IO and an Engine with specified type and mode + /// @param[in] tag The ADIOS2 IO name + /// @param[in] engine_type ADIOS2 engine type. See + /// https://adios2.readthedocs.io/en/latest/engines/engines.html. + /// @param[in] mode ADIOS2 mode, available are: "write", "read", "append", + /// "readrandomaccess", and the default is "append" + void add_io(std::string tag, std::string engine_type = "BP5", + std::string mode = "append") + { + std::map>::iterator it_io + = _ios.end(); + _ios.insert(it_io, + std::pair>( + tag, std::make_shared(_adios->DeclareIO(tag)))); + + assert(_ios[tag]); + _ios[tag]->SetEngine(engine_type); + std::map>::iterator it_engine + = _engines.end(); + _engines.insert(it_engine, + std::pair>( + tag, std::make_shared(_ios[tag]->Open( + _filename, string_to_mode[mode])))); + } + + /// @brief Close engine associated with the IO with the given tag + /// @param[in] tag The ADIOS2 IO name whose associated engine needs to be + /// closed + void close(std::string tag) + { + assert(_engines[tag]); + if (*_engines[tag]) + _engines[tag]->Close(); + } + + /// @brief Close all Engines void close() { - assert(_engine); - if (*_engine) - _engine->Close(); + for (auto it = _engines.begin(); it != _engines.end(); ++it) + { + auto engine = it->second; + assert(engine); + if (*engine) + engine->Close(); + } } - /// @brief Get the IO object - std::shared_ptr io() { return _io; } + /// @brief Get the IO with the given tag + /// @param[in] tag The ADIOS2 IO name + std::shared_ptr io(std::string tag) { return _ios[tag]; } /// @brief Get the Engine object - std::shared_ptr engine() { return _engine; } + /// @param[in] tag The ADIOS2 IO name + std::shared_ptr engine(std::string tag) + { + return _engines[tag]; + } private: + std::string _filename; std::shared_ptr _adios; - std::shared_ptr _io; - std::shared_ptr _engine; + std::map> _ios; + std::map> _engines; }; } // namespace dolfinx::io From fb399167ac24419a930f26e36c8791e5847f1fa5 Mon Sep 17 00:00:00 2001 From: ampdes Date: Sun, 25 Aug 2024 11:45:06 +0200 Subject: [PATCH 56/60] Clean up and update for the new wrapper --- cpp/dolfinx/io/ADIOS2_utils.h | 47 ++++++------------------ python/demo/demo_checkpointing.py | 28 ++++++++++---- python/dolfinx/io/utils.py | 47 +++++++++++++++++++++--- python/dolfinx/wrappers/io.cpp | 59 ++++++++++++++++-------------- python/test/unit/io/test_adios2.py | 24 +++++------- 5 files changed, 114 insertions(+), 91 deletions(-) diff --git a/cpp/dolfinx/io/ADIOS2_utils.h b/cpp/dolfinx/io/ADIOS2_utils.h index 264eb8022cd..c5a0fb27c37 100644 --- a/cpp/dolfinx/io/ADIOS2_utils.h +++ b/cpp/dolfinx/io/ADIOS2_utils.h @@ -35,43 +35,17 @@ class ADIOS2Wrapper public: /// @brief Create an ADIOS2-based engine writer/reader /// @param[in] comm The MPI communicator - /// @param[in] filename Name of output file - ADIOS2Wrapper(MPI_Comm comm, std::string filename) - : _filename(filename), _adios(std::make_shared(comm)) + ADIOS2Wrapper(MPI_Comm comm) { - } - - /// @brief Create an ADIOS2-based engine writer/reader - /// @param[in] comm The MPI communicator - /// @param[in] filename Name of output file - /// @param[in] tag The ADIOS2 IO name - /// @param[in] engine_type ADIOS2 engine type. See - /// https://adios2.readthedocs.io/en/latest/engines/engines.html. - /// @param[in] mode ADIOS2 mode, default is Write or Read - ADIOS2Wrapper(MPI_Comm comm, std::string filename, std::string tag, - std::string engine_type = "BP5", std::string mode = "write") - : _filename(filename), _adios(std::make_shared(comm)) - { - _ios.insert({tag, std::make_shared(_adios->DeclareIO(tag))}); - _ios[tag]->SetEngine(engine_type); - _engines.insert(tag, std::make_shared( - _ios[tag]->Open(filename, string_to_mode[mode]))); + _adios = std::make_shared(comm); } /// @brief Create an ADIOS2-based engine writer/reader /// @param[in] config_file Path to config file to set up ADIOS2 engines from /// @param[in] comm The MPI communicator - /// @param[in] filename Name of output file - /// @param[in] tag The ADIOS2 IO name - /// @param[in] mode ADIOS2 mode, default is Write or Read - ADIOS2Wrapper(std::string config_file, MPI_Comm comm, std::string filename, - std::string tag, std::string mode = "append") - : _filename(filename) + ADIOS2Wrapper(std::string config_file, MPI_Comm comm) { _adios = std::make_shared(config_file, comm); - _io = std::make_shared(_adios->DeclareIO(tag)); - _engine = std::make_shared( - _io->Open(filename, string_to_mode[mode])); } /// @brief Move constructor @@ -90,13 +64,14 @@ class ADIOS2Wrapper ADIOS2Wrapper& operator=(const ADIOS2Wrapper&) = delete; /// @brief Add IO and an Engine with specified type and mode + /// @param[in] filename Name of input/output file /// @param[in] tag The ADIOS2 IO name /// @param[in] engine_type ADIOS2 engine type. See /// https://adios2.readthedocs.io/en/latest/engines/engines.html. /// @param[in] mode ADIOS2 mode, available are: "write", "read", "append", /// "readrandomaccess", and the default is "append" - void add_io(std::string tag, std::string engine_type = "BP5", - std::string mode = "append") + void add_io(const std::string filename, std::string tag, + std::string engine_type = "BP5", std::string mode = "append") { std::map>::iterator it_io = _ios.end(); @@ -105,13 +80,16 @@ class ADIOS2Wrapper tag, std::make_shared(_adios->DeclareIO(tag)))); assert(_ios[tag]); - _ios[tag]->SetEngine(engine_type); + if (!_ios[tag]->InConfigFile()) + { + _ios[tag]->SetEngine(engine_type); + } std::map>::iterator it_engine = _engines.end(); _engines.insert(it_engine, std::pair>( tag, std::make_shared(_ios[tag]->Open( - _filename, string_to_mode[mode])))); + filename, string_to_mode[mode])))); } /// @brief Close engine associated with the IO with the given tag @@ -148,10 +126,9 @@ class ADIOS2Wrapper } private: - std::string _filename; std::shared_ptr _adios; std::map> _ios; - std::map> _engines; + std::map> _engines; }; } // namespace dolfinx::io diff --git a/python/demo/demo_checkpointing.py b/python/demo/demo_checkpointing.py index b78993994af..1d8af76622c 100644 --- a/python/demo/demo_checkpointing.py +++ b/python/demo/demo_checkpointing.py @@ -47,20 +47,34 @@ # + config_path = os.getcwd() + "/checkpointing.yml" -adios2 = io.ADIOS2(config_path, msh.comm, filename="mesh.bp", tag="mesh-write", mode="write") +adios2 = io.ADIOS2(config_path, msh.comm) +tag = "mesh-write" +adios2.add_io(filename="mesh.bp", tag=tag, mode="write") # - # + -io.write_mesh(adios2, msh) -adios2.close() +io.write_mesh(adios2, tag, msh) +adios2.close(tag) # - +# # + +# adios2_read = io.ADIOS2(msh.comm) +# tag = "mesh-read" +# adios2_read.add_io(filename="mesh.bp", tag=tag, engine_type="BP5", mode="read") +# # - + +# # + +# msh_read = io.read_mesh(adios2_read, tag, msh.comm) +# adios2_read.close(tag) +# # - + # + -adios2_read = io.ADIOS2( - msh.comm, filename="mesh.bp", tag="mesh-read", engine_type="BP5", mode="read" -) +# adios2_read = io.ADIOS2(msh.comm) +tag = "mesh-read" +adios2.add_io(filename="mesh.bp", tag=tag, engine_type="BP5", mode="read") # - # + -msh_read = io.read_mesh(adios2_read, msh.comm) +msh_read = io.read_mesh(adios2, tag, msh.comm) +adios2.close(tag) # - diff --git a/python/dolfinx/io/utils.py b/python/dolfinx/io/utils.py index bc2fa04add4..e63c167bdd5 100644 --- a/python/dolfinx/io/utils.py +++ b/python/dolfinx/io/utils.py @@ -202,27 +202,62 @@ def close(self): self._cpp_object.close() class ADIOS2(_cpp.io.ADIOS2): + """Wrapper around ADIOS2. + Supports creation of arbitrary number of IOs + """ + def __enter__(self): return self def __exit__(self, exception_type, exception_value, traceback): self.close() - def write_mesh(ADIOS2: ADIOS2, mesh: Mesh) -> None: - """Write mesh to a file using ADIOS2""" + def add_io(self, filename: str, tag: str, engine_type: str = "BP5", mode: str = "append"): + """Add IO and Engine + + Args: + filename: filename to associate Engine with + tag: Name of the IO to use + engine_type: ADIOS2 Engine type, default is "BP5" + mode: ADIOS2 mode, default is "append" + """ + super().add_io(filename, tag, engine_type, mode) + + def close(self, tag: str): + """Close IO and Engine associated with the given tag""" + super().close(tag) + + def write_mesh(ADIOS2: ADIOS2, tag: str, mesh: Mesh) -> None: + """Write mesh to a file using ADIOS2 + + Args: + ADIOS2: Wrapper around ADIOS2. + tag: Name of the IO to use. + mesh: Mesh to write to the file. + """ + dtype = mesh.geometry.x.dtype # type: ignore if np.issubdtype(dtype, np.float32): _writer = _cpp.io.write_mesh_float32 elif np.issubdtype(dtype, np.float64): _writer = _cpp.io.write_mesh_float64 - return _writer(ADIOS2, mesh._cpp_object) + return _writer(ADIOS2, tag, mesh._cpp_object) def read_mesh( - ADIOS2: ADIOS2, comm: _MPI.Comm, ghost_mode: GhostMode = GhostMode.shared_facet + ADIOS2: ADIOS2, tag: str, comm: _MPI.Comm, ghost_mode: GhostMode = GhostMode.shared_facet ) -> Mesh: - """Read mesh from a file using ADIOS2""" - msh = _cpp.io.read_mesh(ADIOS2, comm, ghost_mode) + """Read mesh from a file using ADIOS2 + + Args: + ADIOS2: Wrapper around ADIOS2 + tag: Name of the IO to use + comm: The MPI communicator + ghost_mode: GhostMode to use on mesh partitioning + Returns: + Mesh + """ + msh = _cpp.io.read_mesh(ADIOS2, tag, comm, ghost_mode) element = basix.ufl.element( basix.ElementFamily.P, diff --git a/python/dolfinx/wrappers/io.cpp b/python/dolfinx/wrappers/io.cpp index 62c7440d483..c668ac839bb 100644 --- a/python/dolfinx/wrappers/io.cpp +++ b/python/dolfinx/wrappers/io.cpp @@ -242,13 +242,15 @@ void declare_write_mesh(nb::module_& m, std::string type) std::string pyfunction_write_mesh_name = std::string("write_mesh_") + type; m.def( pyfunction_write_mesh_name.c_str(), - [](dolfinx::io::ADIOS2Wrapper& ADIOS2, dolfinx::mesh::Mesh& mesh) + [](dolfinx::io::ADIOS2Wrapper& ADIOS2, std::string tag, + dolfinx::mesh::Mesh& mesh) { - auto io = ADIOS2.io(); - auto engine = ADIOS2.engine(); + auto io = ADIOS2.io(tag); + auto engine = ADIOS2.engine(tag); return dolfinx::io::native::write_mesh(*io, *engine, mesh); }, - nb::arg("adios2"), nb::arg("mesh"), "Write mesh to file using ADIOS2"); + nb::arg("adios2"), nb::arg("tag"), nb::arg("mesh"), + "Write mesh to file using ADIOS2"); } #endif @@ -263,41 +265,42 @@ void io(nb::module_& m) ADIOS2 .def( - "__init__", - [](dolfinx::io::ADIOS2Wrapper* v, MPICommWrapper comm, - const std::string filename, std::string tag, - std::string engine_type = "BP5", std::string mode = "write") - { - new (v) dolfinx::io::ADIOS2Wrapper(comm.get(), filename, tag, - engine_type, mode); - }, - nb::arg("comm"), nb::arg("filename"), nb::arg("tag"), - nb::arg("engine_type"), nb::arg("mode")) + "__init__", [](dolfinx::io::ADIOS2Wrapper* v, MPICommWrapper comm) + { new (v) dolfinx::io::ADIOS2Wrapper(comm.get()); }, nb::arg("comm")) .def( "__init__", [](dolfinx::io::ADIOS2Wrapper* v, std::string config, - MPICommWrapper comm, const std::string filename, std::string tag, - std::string mode = "write") - { - new (v) dolfinx::io::ADIOS2Wrapper(config, comm.get(), filename, - tag, mode); - }, - nb::arg("config"), nb::arg("comm"), nb::arg("filename"), - nb::arg("tag"), nb::arg("mode")) - .def("close", &dolfinx::io::ADIOS2Wrapper::close); + MPICommWrapper comm) + { new (v) dolfinx::io::ADIOS2Wrapper(config, comm.get()); }, + nb::arg("config"), nb::arg("comm")) + .def( + "add_io", + [](dolfinx::io::ADIOS2Wrapper& self, const std::string filename, + std::string tag, std::string engine_type = "BP5", + std::string mode = "append") + { self.add_io(filename, tag, engine_type, mode); }, + nb::arg("filename"), nb::arg("tag"), nb::arg("engine_type"), + nb::arg("mode"), "Create IO and Engine") + .def( + "close", [](dolfinx::io::ADIOS2Wrapper& self) { self.close(); }, + "Close all engines") + .def( + "close", [](dolfinx::io::ADIOS2Wrapper& self, std::string tag) + { self.close(tag); }, nb::arg("tag"), + "Close engine associated with tag"); // dolfinx::io::impl_native::read_mesh_variant m.def( "read_mesh", - [](dolfinx::io::ADIOS2Wrapper& ADIOS2, MPICommWrapper comm, - dolfinx::mesh::GhostMode ghost_mode) + [](dolfinx::io::ADIOS2Wrapper& ADIOS2, std::string tag, + MPICommWrapper comm, dolfinx::mesh::GhostMode ghost_mode) { - auto io = ADIOS2.io(); - auto engine = ADIOS2.engine(); + auto io = ADIOS2.io(tag); + auto engine = ADIOS2.engine(tag); return dolfinx::io::impl_native::read_mesh_variant( *io, *engine, comm.get(), ghost_mode); }, - nb::arg("adios2"), nb::arg("comm"), nb::arg("ghost_mode"), + nb::arg("adios2"), nb::arg("tag"), nb::arg("comm"), nb::arg("ghost_mode"), "Read mesh from file using ADIOS2"); declare_write_mesh(m, "float32"); diff --git a/python/test/unit/io/test_adios2.py b/python/test/unit/io/test_adios2.py index 1fa29ef2de7..ed047e44724 100644 --- a/python/test/unit/io/test_adios2.py +++ b/python/test/unit/io/test_adios2.py @@ -56,25 +56,19 @@ def test_mesh_read_write(encoder, suffix, ghost_mode, dtype, dim, simplex, tmp_p mesh = generate_mesh(dim, simplex, N, dtype) - adios = ADIOS2( - mesh.comm, - filename=str(file.with_suffix(suffix)), - tag="mesh-write", - engine_type=encoder, - mode="write", - ) + adios = ADIOS2(mesh.comm) + tag = "mesh-write" + adios.add_io(filename=str(file.with_suffix(suffix)), tag=tag, engine_type=encoder, mode="write") - write_mesh(adios, mesh) + write_mesh(adios, tag, mesh) - adios_read = ADIOS2( - MPI.COMM_WORLD, - filename=str(file.with_suffix(suffix)), - tag="mesh-read", - engine_type=encoder, - mode="read", + adios_read = ADIOS2(MPI.COMM_WORLD) + tag = "mesh-read" + adios_read.add_io( + filename=str(file.with_suffix(suffix)), tag=tag, engine_type=encoder, mode="read" ) - mesh_adios = read_mesh(adios_read, MPI.COMM_WORLD, ghost_mode=ghost_mode) + mesh_adios = read_mesh(adios_read, tag, MPI.COMM_WORLD, ghost_mode=ghost_mode) mesh_adios.comm.Barrier() mesh.comm.Barrier() From 8bec1a1ea2b90ffdf9ef895058c4dd7d051d3a9a Mon Sep 17 00:00:00 2001 From: ampdes Date: Mon, 26 Aug 2024 11:51:02 +0200 Subject: [PATCH 57/60] Add time dependent mesh write --- cpp/demo/checkpointing/main.cpp | 1 + cpp/dolfinx/io/checkpointing.cpp | 211 ++++++++++++++++++------------- cpp/dolfinx/io/checkpointing.h | 2 +- 3 files changed, 124 insertions(+), 90 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 06b9ad00d1e..6536aba238e 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -40,6 +40,7 @@ int main(int argc, char* argv[]) adios2::Engine engine = io.Open("mesh.bp", adios2::Mode::Append); io::native::write_mesh(io, engine, *mesh); + io::native::write_mesh(io, engine, *mesh, 0.5); engine.Close(); } diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 8fbed23a51e..72ac485292c 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -51,6 +51,24 @@ std::map string_to_variant{ {"bernstein", basix::element::lagrange_variant::bernstein}, }; +template +adios2::Variable define_var(adios2::IO& io, std::string name, + const adios2::Dims& shape = adios2::Dims(), + const adios2::Dims& start = adios2::Dims(), + const adios2::Dims& count = adios2::Dims()) +{ + if (adios2::Variable var = io.InquireVariable(name); var) + { + if (var.Count() != count) + var.SetSelection({start, count}); + + return var; + } + else + return io.DefineVariable(name, shape, start, count, + adios2::ConstantDims); +} + } // namespace namespace dolfinx::io::native @@ -58,126 +76,141 @@ namespace dolfinx::io::native //----------------------------------------------------------------------------- template void write_mesh(adios2::IO& io, adios2::Engine& engine, - const dolfinx::mesh::Mesh& mesh) + const dolfinx::mesh::Mesh& mesh, double time) { const mesh::Geometry& geometry = mesh.geometry(); std::shared_ptr topology = mesh.topology(); assert(topology); + std::shared_ptr geom_imap = geometry.index_map(); - // Variables/attributes to save - std::int32_t dim = geometry.dim(); - std::int32_t tdim = topology->dim(); - std::uint64_t num_nodes_global, offset, num_cells_global, cell_offset; - std::uint32_t num_nodes_local, num_cells_local, degree; - std::string cell_type, lagrange_variant; - std::vector array_global; - std::vector offsets_global; - - std::shared_ptr geom_imap; + std::size_t currentstep = engine.CurrentStep(); + assert(!engine.BetweenStepPairs()); + engine.BeginStep(); - // Nodes information + if (currentstep == 0) { - geom_imap = geometry.index_map(); - num_nodes_global = geom_imap->size_global(); - num_nodes_local = geom_imap->size_local(); - offset = geom_imap->local_range()[0]; - } + // Variables/attributes to save + std::int32_t dim = geometry.dim(); + std::int32_t tdim = topology->dim(); + std::uint64_t num_cells_global, cell_offset; + std::uint32_t num_cells_local, degree; + std::string cell_type, lagrange_variant; + std::vector array_global; + std::vector offsets_global; + + // Cells information + { + const std::shared_ptr topo_imap + = topology->index_map(tdim); + assert(topo_imap); - // Cells information - { - const std::shared_ptr topo_imap - = topology->index_map(tdim); - assert(topo_imap); + num_cells_global = topo_imap->size_global(); + num_cells_local = topo_imap->size_local(); + cell_offset = topo_imap->local_range()[0]; + } - num_cells_global = topo_imap->size_global(); - num_cells_local = topo_imap->size_local(); - cell_offset = topo_imap->local_range()[0]; - } + // Coordinate element information + { + const fem::CoordinateElement& cmap = geometry.cmap(); + cell_type = mesh::to_string(cmap.cell_shape()); + degree = cmap.degree(); + lagrange_variant = variant_to_string[cmap.variant()]; + } - // Coordinate element information - { - const fem::CoordinateElement& cmap = geometry.cmap(); - cell_type = mesh::to_string(cmap.cell_shape()); - degree = cmap.degree(); - lagrange_variant = variant_to_string[cmap.variant()]; - } + std::uint32_t num_dofs_per_cell; + // Connectivity + { + auto dofmap = geometry.dofmap(); + num_dofs_per_cell = dofmap.extent(1); + std::vector connectivity; + connectivity.reserve(num_cells_local * num_dofs_per_cell); + for (std::size_t i = 0; i < num_cells_local; ++i) + for (std::size_t j = 0; j < num_dofs_per_cell; ++j) + connectivity.push_back(dofmap(i, j)); + + array_global.resize(num_cells_local * num_dofs_per_cell); + std::iota(array_global.begin(), array_global.end(), 0); + + geom_imap->local_to_global(connectivity, array_global); + offsets_global.resize(num_cells_local + 1); + + // FIXME: use std::ranges::transform + for (std::size_t i = 0; i < num_cells_local + 1; ++i) + offsets_global[i] = (i + cell_offset) * num_dofs_per_cell; + } - const std::span mesh_x = geometry.x(); + // ADIOS2 write attributes and variables + { + io.DefineAttribute("version", dolfinx::version()); + io.DefineAttribute("git_hash", dolfinx::git_commit_hash()); + io.DefineAttribute("name", mesh.name); + io.DefineAttribute("dim", dim); + io.DefineAttribute("tdim", tdim); + io.DefineAttribute("cell_type", cell_type); + io.DefineAttribute("degree", degree); + io.DefineAttribute("lagrange_variant", lagrange_variant); + + adios2::Variable var_num_cells + = io.DefineVariable("num_cells"); + + adios2::Variable var_topology_array + = io.DefineVariable( + "topology_array", {num_cells_global * num_dofs_per_cell}, + {cell_offset * num_dofs_per_cell}, + {num_cells_local * num_dofs_per_cell}, adios2::ConstantDims); + + adios2::Variable var_topology_offsets + = io.DefineVariable( + "topology_offsets", {num_cells_global + 1}, {cell_offset}, + {num_cells_local + 1}, adios2::ConstantDims); + + engine.Put(var_num_cells, num_cells_global); + engine.Put(var_topology_array, array_global.data()); + engine.Put(var_topology_offsets, offsets_global.data()); + } + } - std::uint32_t num_dofs_per_cell; - // Connectivity + std::uint64_t num_nodes_global, offset; + std::uint32_t num_nodes_local; + // Nodes information { - auto dofmap = geometry.dofmap(); - num_dofs_per_cell = dofmap.extent(1); - std::vector connectivity; - connectivity.reserve(num_cells_local * num_dofs_per_cell); - for (std::size_t i = 0; i < num_cells_local; ++i) - for (std::size_t j = 0; j < num_dofs_per_cell; ++j) - connectivity.push_back(dofmap(i, j)); - - array_global.resize(num_cells_local * num_dofs_per_cell); - std::iota(array_global.begin(), array_global.end(), 0); - - geom_imap->local_to_global(connectivity, array_global); - offsets_global.resize(num_cells_local + 1); - - // FIXME: use std::ranges::transform - for (std::size_t i = 0; i < num_cells_local + 1; ++i) - offsets_global[i] = (i + cell_offset) * num_dofs_per_cell; + // geom_imap = geometry.index_map(); + num_nodes_global = geom_imap->size_global(); + num_nodes_local = geom_imap->size_local(); + offset = geom_imap->local_range()[0]; } + const std::span mesh_x = geometry.x(); - // ADIOS2 write attributes and variables + if (currentstep == 0) { - io.DefineAttribute("version", dolfinx::version()); - io.DefineAttribute("git_hash", dolfinx::git_commit_hash()); - io.DefineAttribute("name", mesh.name); - io.DefineAttribute("dim", dim); - io.DefineAttribute("tdim", tdim); - io.DefineAttribute("cell_type", cell_type); - io.DefineAttribute("degree", degree); - io.DefineAttribute("lagrange_variant", lagrange_variant); - adios2::Variable var_num_nodes = io.DefineVariable("num_nodes"); - adios2::Variable var_num_cells - = io.DefineVariable("num_cells"); + engine.Put(var_num_nodes, num_nodes_global); + } - adios2::Variable var_x - = io.DefineVariable("x", {num_nodes_global, 3}, {offset, 0}, - {num_nodes_local, 3}, adios2::ConstantDims); + adios2::Variable var_time = define_var(io, "time", {}, {}, {}); - adios2::Variable var_topology_array - = io.DefineVariable( - "topology_array", {num_cells_global * num_dofs_per_cell}, - {cell_offset * num_dofs_per_cell}, - {num_cells_local * num_dofs_per_cell}, adios2::ConstantDims); + adios2::Variable var_x = define_var(io, "x", {num_nodes_global, 3}, + {offset, 0}, {num_nodes_local, 3}); - adios2::Variable var_topology_offsets - = io.DefineVariable( - "topology_offsets", {num_cells_global + 1}, {cell_offset}, - {num_cells_local + 1}, adios2::ConstantDims); + engine.Put(var_time, time); + engine.Put(var_x, mesh_x.subspan(0, num_nodes_local * 3).data()); - assert(!engine.BetweenStepPairs()); - engine.BeginStep(); - engine.Put(var_num_nodes, num_nodes_global); - engine.Put(var_num_cells, num_cells_global); - engine.Put(var_x, mesh_x.subspan(0, num_nodes_local * 3).data()); - engine.Put(var_topology_array, array_global.data()); - engine.Put(var_topology_offsets, offsets_global.data()); - engine.EndStep(); + engine.EndStep(); - spdlog::info("Mesh written"); - } + spdlog::info("Mesh written"); } //----------------------------------------------------------------------------- /// @cond template void write_mesh(adios2::IO& io, adios2::Engine& engine, - const dolfinx::mesh::Mesh& mesh); + const dolfinx::mesh::Mesh& mesh, + double time); template void write_mesh(adios2::IO& io, adios2::Engine& engine, - const dolfinx::mesh::Mesh& mesh); + const dolfinx::mesh::Mesh& mesh, + double time); /// @endcond diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index 4646e226128..e51e5322136 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -81,7 +81,7 @@ namespace dolfinx::io::native /// @param[in] mesh Mesh of type float or double to write to the file template void write_mesh(adios2::IO& io, adios2::Engine& engine, - const dolfinx::mesh::Mesh& mesh); + const dolfinx::mesh::Mesh& mesh, double time = 0); /// @brief Read mesh from a file. /// From 24bab34ac1cfe19c84c3478de640190bf6379ad6 Mon Sep 17 00:00:00 2001 From: ampdes Date: Fri, 30 Aug 2024 01:00:08 +0200 Subject: [PATCH 58/60] Add read_timestamps in readrandomaccess mode Add update_mesh --- cpp/demo/checkpointing/main.cpp | 63 +++++++++++++++++--- cpp/dolfinx/io/checkpointing.cpp | 93 ++++++++++++++++++++++++++++++ cpp/dolfinx/io/checkpointing.h | 20 +++++++ python/demo/demo_checkpointing.py | 35 +++++++---- python/dolfinx/io/__init__.py | 6 +- python/dolfinx/io/utils.py | 30 +++++++++- python/dolfinx/wrappers/io.cpp | 37 +++++++++++- python/test/unit/io/test_adios2.py | 4 +- 8 files changed, 263 insertions(+), 25 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index 6536aba238e..eacec6e1161 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -40,6 +40,12 @@ int main(int argc, char* argv[]) adios2::Engine engine = io.Open("mesh.bp", adios2::Mode::Append); io::native::write_mesh(io, engine, *mesh); + std::span x = mesh->geometry().x(); + for (std::size_t i = 0; i < x.size(); ++i) + { + x[i] *= 4; + } + io::native::write_mesh(io, engine, *mesh, 0.5); engine.Close(); @@ -52,12 +58,40 @@ int main(int argc, char* argv[]) try { - // Set up ADIOS2 IO and Engine + // Read mode : set up ADIOS2 IO and Engine adios2::ADIOS adios_read(MPI_COMM_WORLD); adios2::IO io_read = adios_read.DeclareIO("mesh-read"); io_read.SetEngine("BP5"); adios2::Engine engine_read = io_read.Open("mesh.bp", adios2::Mode::Read); + // ReadRandomAccess mode : set up ADIOS2 IO and Engine + adios2::IO io_rra = adios_read.DeclareIO("mesh-rra"); + io_rra.SetEngine("BP5"); + adios2::Engine engine_rra + = io_rra.Open("mesh.bp", adios2::Mode::ReadRandomAccess); + + // Write mode : set up ADIOS2 IO and Engine + adios2::IO io_write = adios_read.DeclareIO("mesh-write"); + io_write.SetEngine("BP5"); + adios2::Engine engine_write + = io_write.Open("mesh2.bp", adios2::Mode::Write); + + // Find the time stamps array + auto var_time = io_rra.InquireVariable("time"); + const std::vector::Info>> timestepsinfo + = var_time.AllStepsBlocksInfo(); + + std::size_t num_steps = timestepsinfo.size(); + std::vector times(num_steps); + + for (std::size_t step = 0; step < num_steps; ++step) + { + var_time.SetStepSelection({step, 1}); + engine_rra.Get(var_time, times[step]); + } + engine_rra.Close(); + + // Read mesh engine_read.BeginStep(); auto mesh_read = io::native::read_mesh(io_read, engine_read, MPI_COMM_WORLD); @@ -65,15 +99,30 @@ int main(int argc, char* argv[]) { engine_read.EndStep(); } + // Write mesh + io::native::write_mesh(io_write, engine_write, mesh_read); - engine_read.Close(); + // Update mesh + double time = 0.5; + std::size_t querystep; + auto pos = std::ranges::find(times, time); + if (pos != times.end()) + { + querystep = std::ranges::distance(times.begin(), pos); + std::cout << "Query step is : " << querystep << "\n"; + } + else + { + throw std::runtime_error("Step corresponding to time : " + + std::to_string(time) + " not found"); + } - adios2::IO io_write = adios_read.DeclareIO("mesh-write"); - io_write.SetEngine("BP5"); - adios2::Engine engine_write - = io_write.Open("mesh2.bp", adios2::Mode::Write); + io::native::update_mesh(io_read, engine_read, mesh_read, querystep); - io::native::write_mesh(io_write, engine_write, mesh_read); + // Write updated mesh + io::native::write_mesh(io_write, engine_write, mesh_read, time); + + engine_read.Close(); engine_write.Close(); } catch (std::exception& e) diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 72ac485292c..42dfab74f82 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -335,6 +335,99 @@ read_mesh(adios2::IO& io, adios2::Engine& engine, MPI_Comm comm, /// @endcond +//----------------------------------------------------------------------------- +template +void update_mesh(adios2::IO& io, adios2::Engine& engine, + dolfinx::mesh::Mesh& mesh, std::size_t step) +{ + if (!engine.BetweenStepPairs()) + { + engine.BeginStep(); + } + + std::uint32_t num_nodes_local = mesh.geometry().index_map()->size_local(); + std::vector x_raw(num_nodes_local * 3); + + // Read variables + { + double time; + adios2::Variable var_time = io.InquireVariable("time"); + var_time.SetStepSelection({step, 1}); + if (var_time) + { + engine.Get(var_time, time); + spdlog::info("Updating geometry at time : {}", time); + } + else + { + throw std::runtime_error("Step : " + std::to_string(step) + " not found"); + } + + adios2::Variable var_x = io.InquireVariable("x"); + var_x.SetStepSelection({step, 1}); + if (var_x) + { + std::uint64_t nodes_offset + = mesh.geometry().index_map()->local_range()[0]; + var_x.SetSelection({{nodes_offset, 0}, {num_nodes_local, 3}}); + engine.Get(var_x, x_raw.data(), adios2::Mode::Sync); + } + else + { + throw std::runtime_error("Coordinates data not found at step : " + step); + } + } + + engine.EndStep(); + + // Redistribute adios2 input coordinate data and find updated coordinates of + // the mesh + std::vector x_new = dolfinx::MPI::distribute_data( + mesh.comm(), mesh.geometry().input_global_indices(), mesh.comm(), x_raw, + 3); + + std::span x = mesh.geometry().x(); + for (std::size_t i = 0; i < num_nodes_local * 3; ++i) + { + x[i] = x_new[i]; + } +} + +//----------------------------------------------------------------------------- +/// @cond +template void update_mesh(adios2::IO& io, adios2::Engine& engine, + dolfinx::mesh::Mesh& mesh, + std::size_t step); + +template void update_mesh(adios2::IO& io, adios2::Engine& engine, + dolfinx::mesh::Mesh& mesh, + std::size_t step); + +/// @endcond + +//----------------------------------------------------------------------------- +std::vector read_timestamps(adios2::IO& io, adios2::Engine& engine) +{ + if (engine.OpenMode() != adios2::Mode::ReadRandomAccess) + { + throw std::runtime_error( + "Time stamps can only be read in ReadRandomAccess mode"); + } + adios2::Variable var_time = io.InquireVariable("time"); + const std::vector::Info>> timestepsinfo + = var_time.AllStepsBlocksInfo(); + + std::size_t num_steps = timestepsinfo.size(); + std::vector times(num_steps); + + for (std::size_t step = 0; step < num_steps; ++step) + { + var_time.SetStepSelection({step, 1}); + engine.Get(var_time, times[step]); + } + return times; +} + } // namespace dolfinx::io::native namespace dolfinx::io::impl_native diff --git a/cpp/dolfinx/io/checkpointing.h b/cpp/dolfinx/io/checkpointing.h index e51e5322136..8fe1579efcf 100644 --- a/cpp/dolfinx/io/checkpointing.h +++ b/cpp/dolfinx/io/checkpointing.h @@ -79,6 +79,7 @@ namespace dolfinx::io::native /// @param[in] io ADIOS2 IO /// @param[in] engine ADIOS2 Engine /// @param[in] mesh Mesh of type float or double to write to the file +/// @param[in] time Time associated with the mesh with moving geometry template void write_mesh(adios2::IO& io, adios2::Engine& engine, const dolfinx::mesh::Mesh& mesh, double time = 0); @@ -96,6 +97,25 @@ dolfinx::mesh::Mesh read_mesh(adios2::IO& io, adios2::Engine& engine, dolfinx::mesh::GhostMode ghost_mode = dolfinx::mesh::GhostMode::shared_facet); +/// @brief Update geometry of mesh by reading from a file. +/// +/// @param[in] io ADIOS2 IO +/// @param[in] engine ADIOS2 Engine +/// @param[in] mesh Mesh of type float or double +/// @param[in] step ADIOS2 Engine Step to read geometry from +/// @note This method expects that ReadRandomAccess mode is used to determine +/// the step in which a given time stamp exists +template +void update_mesh(adios2::IO& io, adios2::Engine& engine, + dolfinx::mesh::Mesh& mesh, std::size_t step); + +/// @brief Read time stamps. +/// +/// @param[in] io ADIOS2 IO +/// @param[in] engine ADIOS2 Engine +/// @note The Engine should be opened in ReadRandomAccess mode +std::vector read_timestamps(adios2::IO& io, adios2::Engine& engine); + } // namespace dolfinx::io::native #endif diff --git a/python/demo/demo_checkpointing.py b/python/demo/demo_checkpointing.py index 1d8af76622c..5de8075db38 100644 --- a/python/demo/demo_checkpointing.py +++ b/python/demo/demo_checkpointing.py @@ -17,6 +17,8 @@ # + import os +import numpy as np + import dolfinx if not dolfinx.common.has_adios2: @@ -54,27 +56,40 @@ # + io.write_mesh(adios2, tag, msh) + +msh.geometry.x[:] += 4 +io.write_mesh(adios2, tag, msh, 0.5) + +msh.geometry.x[:] += 4 +io.write_mesh(adios2, tag, msh, 1.0) + adios2.close(tag) # - -# # + -# adios2_read = io.ADIOS2(msh.comm) -# tag = "mesh-read" -# adios2_read.add_io(filename="mesh.bp", tag=tag, engine_type="BP5", mode="read") -# # - +# + +tag = "mesh-readrandomaccess" +adios2.add_io(filename="mesh.bp", tag=tag, engine_type="BP5", mode="readrandomaccess") +# - -# # + -# msh_read = io.read_mesh(adios2_read, tag, msh.comm) -# adios2_read.close(tag) -# # - +# + +times = io.read_timestamps(adios2, tag) +print(f"Time stamps : {times}") +# - # + -# adios2_read = io.ADIOS2(msh.comm) tag = "mesh-read" adios2.add_io(filename="mesh.bp", tag=tag, engine_type="BP5", mode="read") # - # + msh_read = io.read_mesh(adios2, tag, msh.comm) +print(np.max(msh_read.geometry.x)) + +io.update_mesh(adios2, tag, msh_read, 1) +print(np.max(msh_read.geometry.x)) + +io.update_mesh(adios2, tag, msh_read, 2) +print(np.max(msh_read.geometry.x)) + adios2.close(tag) # - diff --git a/python/dolfinx/io/__init__.py b/python/dolfinx/io/__init__.py index 4cd6420b68e..cd6eff9e1ea 100644 --- a/python/dolfinx/io/__init__.py +++ b/python/dolfinx/io/__init__.py @@ -20,6 +20,8 @@ VTXMeshPolicy, VTXWriter, read_mesh, + read_timestamps, + update_mesh, write_mesh, ) @@ -30,6 +32,8 @@ "FidesMeshPolicy", "VTXMeshPolicy", "ADIOS2", - "write_mesh", "read_mesh", + "read_timestamps", + "update_mesh", + "write_mesh", ] diff --git a/python/dolfinx/io/utils.py b/python/dolfinx/io/utils.py index e63c167bdd5..445690dea73 100644 --- a/python/dolfinx/io/utils.py +++ b/python/dolfinx/io/utils.py @@ -52,6 +52,8 @@ def _extract_cpp_objects(functions: typing.Union[Mesh, Function, tuple[Function] "VTXMeshPolicy", "ADIOS2", "read_mesh", + "read_timestamps", + "update_mesh", "write_mesh", ] @@ -227,13 +229,14 @@ def close(self, tag: str): """Close IO and Engine associated with the given tag""" super().close(tag) - def write_mesh(ADIOS2: ADIOS2, tag: str, mesh: Mesh) -> None: + def write_mesh(ADIOS2: ADIOS2, tag: str, mesh: Mesh, time: np.floating = 0) -> None: """Write mesh to a file using ADIOS2 Args: ADIOS2: Wrapper around ADIOS2. tag: Name of the IO to use. mesh: Mesh to write to the file. + time: Associated time stamp """ dtype = mesh.geometry.x.dtype # type: ignore @@ -242,7 +245,7 @@ def write_mesh(ADIOS2: ADIOS2, tag: str, mesh: Mesh) -> None: elif np.issubdtype(dtype, np.float64): _writer = _cpp.io.write_mesh_float64 - return _writer(ADIOS2, tag, mesh._cpp_object) + return _writer(ADIOS2, tag, mesh._cpp_object, time) def read_mesh( ADIOS2: ADIOS2, tag: str, comm: _MPI.Comm, ghost_mode: GhostMode = GhostMode.shared_facet @@ -271,6 +274,29 @@ def read_mesh( return Mesh(msh, domain) + def read_timestamps(ADIOS2: ADIOS2, tag: str) -> np.ndarray: + """Read timestamps from a file using ADIOS2 + + Args: + ADIOS2: Wrapper around ADIOS2 + tag: Name of the IO to use + Returns: + vector of timestamps + """ + return _cpp.io.read_timestamps(ADIOS2, tag) + + def update_mesh(ADIOS2: ADIOS2, tag: str, mesh: Mesh, step: np.int64) -> None: + """Update mesh geometry with geometry stored with the given time stamp in the file + + Args: + ADIOS2: Wrapper around ADIOS2 + tag: Name of the IO to use + mesh: Mesh to update the geometry + step: ADIOS2 Step at which the corresponding geometry is to be read from the file + """ + + return _cpp.io.update_mesh(ADIOS2, tag, mesh._cpp_object, step) + class VTKFile(_cpp.io.VTKFile): """Interface to VTK files. diff --git a/python/dolfinx/wrappers/io.cpp b/python/dolfinx/wrappers/io.cpp index c668ac839bb..c10deb05506 100644 --- a/python/dolfinx/wrappers/io.cpp +++ b/python/dolfinx/wrappers/io.cpp @@ -243,16 +243,33 @@ void declare_write_mesh(nb::module_& m, std::string type) m.def( pyfunction_write_mesh_name.c_str(), [](dolfinx::io::ADIOS2Wrapper& ADIOS2, std::string tag, - dolfinx::mesh::Mesh& mesh) + dolfinx::mesh::Mesh& mesh, double time) { auto io = ADIOS2.io(tag); auto engine = ADIOS2.engine(tag); - return dolfinx::io::native::write_mesh(*io, *engine, mesh); + return dolfinx::io::native::write_mesh(*io, *engine, mesh, time); }, - nb::arg("adios2"), nb::arg("tag"), nb::arg("mesh"), + nb::arg("adios2"), nb::arg("tag"), nb::arg("mesh"), nb::arg("time") = 0.0, "Write mesh to file using ADIOS2"); } +template +void declare_update_mesh(nb::module_& m) +{ + // dolfinx::io::native::update_mesh + m.def( + "update_mesh", + [](dolfinx::io::ADIOS2Wrapper& ADIOS2, std::string tag, + dolfinx::mesh::Mesh& mesh, std::size_t step) + { + auto io = ADIOS2.io(tag); + auto engine = ADIOS2.engine(tag); + return dolfinx::io::native::update_mesh(*io, *engine, mesh, step); + }, + nb::arg("adios2"), nb::arg("tag"), nb::arg("mesh"), nb::arg("step"), + "Update mesh with geometry associated with a given ADIOS2 step"); +} + #endif } // namespace @@ -303,8 +320,22 @@ void io(nb::module_& m) nb::arg("adios2"), nb::arg("tag"), nb::arg("comm"), nb::arg("ghost_mode"), "Read mesh from file using ADIOS2"); + // dolfinx::io::native::read_timestamps + m.def( + "read_timestamps", + [](dolfinx::io::ADIOS2Wrapper& ADIOS2, std::string tag) + { + auto io = ADIOS2.io(tag); + auto engine = ADIOS2.engine(tag); + return dolfinx::io::native::read_timestamps(*io, *engine); + }, + nb::arg("adios2"), nb::arg("tag"), + "Update mesh with geometry associated with a given ADIOS2 step"); + declare_write_mesh(m, "float32"); declare_write_mesh(m, "float64"); + declare_update_mesh(m); + declare_update_mesh(m); #endif diff --git a/python/test/unit/io/test_adios2.py b/python/test/unit/io/test_adios2.py index ed047e44724..cca7bb9aaca 100644 --- a/python/test/unit/io/test_adios2.py +++ b/python/test/unit/io/test_adios2.py @@ -38,9 +38,9 @@ def generate_mesh(dim: int, simplex: bool, N: int = 5, dtype=None): raise RuntimeError("Unsupported dimension") -# TODO: Fix problems with ("HDF5", ".h5") +# TODO: Fix problems with ("HDF5", ".h5"), ("BP4", ".bp"), @pytest.mark.adios2 -@pytest.mark.parametrize("encoder, suffix", [("BP4", ".bp"), ("BP5", ".bp")]) +@pytest.mark.parametrize("encoder, suffix", [("BP5", ".bp")]) @pytest.mark.parametrize("ghost_mode", [GhostMode.shared_facet, GhostMode.none]) @pytest.mark.parametrize("dtype", [np.float32, np.float64]) @pytest.mark.parametrize("dim", [2, 3]) From a0543eb9d93eb7804e1ed3921b1d10e788f68285 Mon Sep 17 00:00:00 2001 From: ampdes Date: Fri, 30 Aug 2024 10:35:06 +0200 Subject: [PATCH 59/60] Replace basic loops with std::ranges::transform MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jørgen Schartum Dokken --- cpp/demo/checkpointing/main.cpp | 5 +---- cpp/dolfinx/io/checkpointing.cpp | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/cpp/demo/checkpointing/main.cpp b/cpp/demo/checkpointing/main.cpp index eacec6e1161..dd04dafe688 100644 --- a/cpp/demo/checkpointing/main.cpp +++ b/cpp/demo/checkpointing/main.cpp @@ -41,10 +41,7 @@ int main(int argc, char* argv[]) io::native::write_mesh(io, engine, *mesh); std::span x = mesh->geometry().x(); - for (std::size_t i = 0; i < x.size(); ++i) - { - x[i] *= 4; - } + std::ranges::transform(x, x.begin(), [](auto xi) { return xi *= 4; }); io::native::write_mesh(io, engine, *mesh, 0.5); diff --git a/cpp/dolfinx/io/checkpointing.cpp b/cpp/dolfinx/io/checkpointing.cpp index 42dfab74f82..326191608ea 100644 --- a/cpp/dolfinx/io/checkpointing.cpp +++ b/cpp/dolfinx/io/checkpointing.cpp @@ -387,10 +387,7 @@ void update_mesh(adios2::IO& io, adios2::Engine& engine, 3); std::span x = mesh.geometry().x(); - for (std::size_t i = 0; i < num_nodes_local * 3; ++i) - { - x[i] = x_new[i]; - } + std::ranges::transform(x_new, x.begin(), [](auto it) { return it; }); } //----------------------------------------------------------------------------- From 9583a3b8322297249c5822f83021772f50965ae2 Mon Sep 17 00:00:00 2001 From: ampdes Date: Sat, 31 Aug 2024 00:25:08 +0200 Subject: [PATCH 60/60] Add tests for time dependent mesh --- python/test/unit/io/test_adios2.py | 95 ++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/python/test/unit/io/test_adios2.py b/python/test/unit/io/test_adios2.py index cca7bb9aaca..e21b9741d80 100644 --- a/python/test/unit/io/test_adios2.py +++ b/python/test/unit/io/test_adios2.py @@ -91,6 +91,101 @@ def test_mesh_read_write(encoder, suffix, ghost_mode, dtype, dim, simplex, tmp_p ) +# TODO: Fix problems with ("HDF5", ".h5"), ("BP4", ".bp"), +@pytest.mark.adios2 +@pytest.mark.parametrize("encoder, suffix", [("BP5", ".bp")]) +@pytest.mark.parametrize("ghost_mode", [GhostMode.shared_facet, GhostMode.none]) +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +@pytest.mark.parametrize("dim", [2, 3]) +@pytest.mark.parametrize("simplex", [True, False]) +def test_timedep_mesh_read_write(encoder, suffix, ghost_mode, dtype, dim, simplex, tmp_path): + "Test writing of a time dependent mesh" + from dolfinx.io import ADIOS2, read_mesh, read_timestamps, update_mesh, write_mesh + + N = 5 + # Consistent tmp dir across processes + fname = MPI.COMM_WORLD.bcast(tmp_path, root=0) + file = fname / f"adios_timedep_mesh_{encoder}" + + mesh = generate_mesh(dim, simplex, N, dtype) + + def displace(x): + return np.asarray( + [ + x[0] + 0.1 * np.sin(x[0]) * np.cos(x[1]), + x[1] + 0.6 * np.cos(x[0]) * np.sin(x[1]), + x[2], + ] + ) + + adios = ADIOS2(mesh.comm) + tag_write = "mesh-write" + adios.add_io( + filename=str(file.with_suffix(suffix)), tag=tag_write, engine_type=encoder, mode="write" + ) + + # Write mesh + write_mesh(adios, tag_write, mesh, time=0.0) + + delta_x1 = displace(mesh.geometry.x.T).T + mesh.geometry.x[:] += delta_x1 + + write_mesh(adios, tag_write, mesh, time=1.0) + + delta_x2 = displace(mesh.geometry.x.T).T + mesh.geometry.x[:] += delta_x2 + + write_mesh(adios, tag_write, mesh, time=2.0) + + adios.close(tag_write) + + # reset mesh geometry to the one at time=0.0 + mesh.geometry.x[:] -= delta_x1 + delta_x2 + + tag_rra = "mesh-readrandomaccess" + adios.add_io( + filename=str(file.with_suffix(suffix)), + tag=tag_rra, + engine_type=encoder, + mode="readrandomaccess", + ) + times = read_timestamps(adios, tag_rra) + adios.close(tag_rra) + assert np.all(np.isclose(times, [0.0, 1.0, 2.0])) + + tag_read = "mesh-read" + adios.add_io( + filename=str(file.with_suffix(suffix)), tag=tag_read, engine_type=encoder, mode="read" + ) + mesh_adios = read_mesh(adios, tag_read, MPI.COMM_WORLD, ghost_mode=ghost_mode) + + mesh_adios.comm.Barrier() + mesh.comm.Barrier() + + # Check that integration over different entities are consistent + measures = [ufl.ds, ufl.dx] if ghost_mode is GhostMode.none else [ufl.ds, ufl.dS, ufl.dx] + for step, time in enumerate(times): + if step == 1: + mesh.geometry.x[:] += delta_x1 + if step == 2: + mesh.geometry.x[:] += delta_x2 + + # FIXME: update_mesh at time time=0.0 should work!? + if step > 0: + update_mesh(adios, tag_read, mesh_adios, step) + + mesh_adios.comm.Barrier() + mesh.comm.Barrier() + + for measure in measures: + c_adios = assemble_scalar(form(1 * measure(domain=mesh_adios), dtype=dtype)) + c_ref = assemble_scalar(form(1 * measure(domain=mesh), dtype=dtype)) + assert np.isclose( + mesh_adios.comm.allreduce(c_adios, MPI.SUM), + mesh.comm.allreduce(c_ref, MPI.SUM), + ) + + @pytest.mark.adios2 class TestFides: @pytest.mark.parametrize("dim", [2, 3])