From 3adf0d137d0d97f8153feafc242defda01bb5b76 Mon Sep 17 00:00:00 2001 From: tommelt Date: Wed, 27 Nov 2024 17:22:07 +0000 Subject: [PATCH] wip --- core/test/CMakeLists.txt | 22 ++++++ core/test/HaloExchange_test.cpp | 131 ++++++++++++++++++++++---------- 2 files changed, 113 insertions(+), 40 deletions(-) diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt index 65b72c514..b1eb1fb40 100644 --- a/core/test/CMakeLists.txt +++ b/core/test/CMakeLists.txt @@ -77,6 +77,28 @@ if(ENABLE_MPI) ) target_link_libraries(testModelMetadata_MPI3 PRIVATE nextsimlib doctest::doctest) + # set(ModelArrayStructure "discontinuousgalerkin") + set(CoreDir "../../core/src/") + + # set(MODEL_INCLUDE_DIR "./testmodelarraydetails") + add_executable(testHaloExchange_MPI3 + "HaloExchange_test.cpp" + "MainMPI.cpp" + "../src/ModelArray.cpp" + "../src/ModelArraySlice.cpp" + "${MODEL_INCLUDE_DIR}/ModelArrayDetails.cpp" + ) + target_compile_definitions(testHaloExchange_MPI3 PRIVATE USE_MPI) + target_include_directories( + testHaloExchange_MPI3 PRIVATE + "../src" + "../../dynamics/src" + ${MODEL_INCLUDE_DIR} + ) + target_link_libraries(testHaloExchange_MPI3 PRIVATE doctest::doctest Eigen3::Eigen) + + set(MODEL_INCLUDE_DIR "../../core/src/discontinuousgalerkin") + add_executable(testParaGrid_MPI2 "ParaGrid_test.cpp" "MainMPI.cpp") target_compile_definitions( testParaGrid_MPI2 diff --git a/core/test/HaloExchange_test.cpp b/core/test/HaloExchange_test.cpp index 933f52734..3d2aa915d 100644 --- a/core/test/HaloExchange_test.cpp +++ b/core/test/HaloExchange_test.cpp @@ -5,62 +5,113 @@ * @author Tom Meltzer */ +#include #include -#include +#include "include/DGModelArray.hpp" #include "include/ModelArraySlice.hpp" +#include "include/ParametricMesh.hpp" +#include "mpi.h" #include -const auto nx = 23; -const auto ny = 17; -const auto nz = 5; +using Slice = ArraySlicer::Slice; +using SliceIter = ArraySlicer::SliceIter; + +const auto nx = 9; +const auto ny = 5; +const auto nz = 3; namespace Nextsim { -TEST_SUITE_BEGIN("ModelMetadata"); -MPI_TEST_CASE("Test getPartitionMetadata closed boundary", 3) +TEST_SUITE_BEGIN("Halo exchange"); +MPI_TEST_CASE("test halo exchange", 3) { - ModelArray::setDimension(ModelArray::Dimension::X, nx, nx / 4, 0); - ModelArray::setDimension(ModelArray::Dimension::Y, ny, ny / 4, 0); - - // Test 1 dimensional slices into and out of common buffer types - // std::vector - OneDField oned(ModelArray::Type::ONED); - oned.resize(); - const auto x0 = 3; - const auto x1 = 11; - auto oneSlice = oned[{ { { x0, x1 } } }]; - - // Check the functions throw when given too small a buffer - std::vector shortBuffer; - shortBuffer.resize(x1 - x0 - 1); - REQUIRE_THROWS(oneSlice = shortBuffer); - REQUIRE_THROWS(oneSlice.copyToBuffer(shortBuffer)); - // Fill the array with index values - for (auto i = 0; i < oned.size(); ++i) { - oned[i] = i; - } + size_t localNx = nx / test_nb_procs; + size_t offsetX = test_rank * localNx; + ModelArray::setDimension(ModelArray::Dimension::X, nx, localNx, offsetX); + ModelArray::setDimension(ModelArray::Dimension::Y, ny, ny, 0); + + // create example 2D field of size 3x5 (x,y) on each process + auto testfield = ModelArray::HField(); + testfield.resize(); - std::vector vectorBuffer; - vectorBuffer.resize(x1 - x0); - // Fill with index values - for (auto i = 0; i < vectorBuffer.size(); ++i) { - vectorBuffer[i] = i; + // initialize with mock data + for (size_t j = 0; j < ny; ++j) { + for (size_t i = 0; i < localNx; ++i) { + testfield(i, j) = test_rank * 100 + (i + offsetX) * 10 + j; + } } - // assign from the buffer - oneSlice = vectorBuffer; - REQUIRE(oned[x0 - 1] == x0 - 1); - REQUIRE(oned[x1] == x1); - REQUIRE(oned[x0 + 0] == 0); - REQUIRE(oned[x1 - 1] == x1 - x0 - 1); - // Refill with index values - for (auto i = 0; i < oned.size(); ++i) { - oned[i] = i; + + // create example 2D field for dynamics code which needs +1 halo on all edges, leading to +2 in + // x and y i.e., 5x7 on each process + offsetX = test_rank * (localNx + 2); + + // auto dynamicsfield = ModelArray::TwoDFieldHALO(); + // dynamicsfield.resize(); + static const int DG = 3; + + // Create the ParametricMesh object + auto CoordinateSystem = Nextsim::CARTESIAN; + ParametricMesh smesh(CoordinateSystem); + smesh.nx = nx + 2; + smesh.ny = ny + 2; + smesh.nnodes = smesh.nx * smesh.ny; + smesh.nelements = smesh.nnodes; + smesh.vertices.resize(smesh.nelements, Eigen::NoChange); + for (size_t i = 0; i < nx; ++i) { + for (size_t j = 0; j < ny; ++j) { + smesh.vertices(i * ny + j, 0) = i; + smesh.vertices(i * ny + j, 1) = j; + } } + // DGModelArray::ma2dg(testfield, dgvec); + + DGVector dgvec(smesh); + + ModelArraySlice source(testfield, { { {}, {} } }); + + source.copyToDataSlice(dgvec, { { { 1, localNx + 1 }, { 1, ny + 1 } } }); + + int ierr; + enum Edge { BOTTOM, RIGHT, TOP, LEFT, N_EDGE }; + constexpr std::array edges = { BOTTOM, RIGHT, TOP, LEFT }; + std::map> neighbours, haloStarts; + + neighbours.try_emplace(BOTTOM, std::vector({ 0, 1, 2 })); + neighbours.try_emplace(TOP, std::vector({ 0, 1, 2 })); + neighbours.try_emplace(RIGHT, std::vector({ 1, 2, 0 })); + neighbours.try_emplace(LEFT, std::vector({ 2, 0, 1 })); + + haloStarts.try_emplace(BOTTOM, std::vector(3, 12)); + haloStarts.try_emplace(TOP, std::vector(3, 0)); + haloStarts.try_emplace(LEFT, std::vector(3, 2)); + haloStarts.try_emplace(RIGHT, std::vector(3, 0)); + + const long int x0 = offsetX; + const long int x1 = offsetX + localNx; + auto slice = testfield[{ { { 0, localNx }, { { 0 } } } }]; + + std::vector vectorBuffer = std::vector(localNx, -1); + + slice.copyToBuffer(vectorBuffer); + + // ModelArray source(ModelArray::Type::H); + // source.resize(); + // // Fill with data + // for (size_t i = 0; i < nx; ++i) { + // for (size_t j = 0; j < ny; j++) { + // source(i, j) = j + mx * (i); + // } + // } + + // CHECK(source(14, 12) == 12 + mx * (14)); + + // ierr = MPI_Scatterv() if (test_rank == 0) { + std::vector neighbour_R = { 1, 2, 0 }; } else if (test_rank == 1) { } else if (test_rank == 2) { } else {