Merge branch 'master' into a1_lt_lam_turb

.github/workflows/ci.yml

@@ -11,20 +11,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Clone
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Check formatting
-      uses: DoozyX/clang-format-lint-action@v0.13
+      uses: DoozyX/clang-format-lint-action@v0.17
       with:
         source: 'nalu.C unit_tests.C ./include ./src ./unit_tests'
         extensions: 'H,h,cpp,C'
-        clangFormatVersion: 13
+        clangFormatVersion: 17
   CPU-Trilinos:
     needs: Formatting
     runs-on: ubuntu-latest
     container:
       image: ecpe4s/exawind-snapshot
       env:
-          SPACK_MANAGER: /spack-manager
+          EXAWIND_MANAGER: /exawind-manager
           E4S_MACHINE: true
     steps:
       - name: Cancel previous runs
@@ -38,13 +38,16 @@ jobs:
       - name: Tests
         run: |
           /bin/bash -c " \
-            source ${SPACK_MANAGER}/start.sh && \
-            mkdir -p ${SPACK_MANAGER}/environments/exawind && \
-            cd ${SPACK_MANAGER}/environments/exawind && \
+            source ${EXAWIND_MANAGER}/start.sh && \
+            mkdir -p ${EXAWIND_MANAGER}/environments/exawind && \
+            cd ${EXAWIND_MANAGER}/environments/exawind && \
             ln -s ${GITHUB_WORKSPACE} nalu-wind && \
-            source ${SPACK_MANAGER}/start.sh && \
+            spack-start && \
+            spack -e e4s-build config add view:true && \
+            spack -e e4s-build add cmake && \
+            spack -e e4s-build install && \
             quick-create-dev -s nalu-wind@master+tioga+hypre && \
-            spack manager external --latest --blacklist nalu-wind gmake && \
+            spack manager external ${EXAWIND_MANAGER}/environments/e4s-build --exclude nalu-wind gmake && \
             spack install && \
             spack cd -b nalu-wind && \
             spack build-env nalu-wind ctest -j $(nproc) -L unit --output-on-failure \
@@ -55,7 +58,7 @@ jobs:
     container:
       image: ecpe4s/exawind-snapshot
       env:
-          SPACK_MANAGER: /spack-manager
+          EXAWIND_MANAGER: /exawind-manager
           E4S_MACHINE: true
     steps:
       - name: Cancel previous runs
@@ -69,13 +72,16 @@ jobs:
       - name: Tests
         run: |
           /bin/bash -c " \
-            source ${SPACK_MANAGER}/start.sh && \
-            mkdir -p ${SPACK_MANAGER}/environments/exawind && \
-            cd ${SPACK_MANAGER}/environments/exawind && \
+            source ${EXAWIND_MANAGER}/start.sh && \
+            mkdir -p ${EXAWIND_MANAGER}/environments/exawind && \
+            cd ${EXAWIND_MANAGER}/environments/exawind && \
             ln -s ${GITHUB_WORKSPACE} nalu-wind && \
-            source ${SPACK_MANAGER}/start.sh && \
+            spack-start && \
+            spack -e e4s-build config add view:true && \
+            spack -e e4s-build add cmake && \
+            spack -e e4s-build install && \
             quick-create-dev -s nalu-wind@master+tioga+hypre~trilinos-solvers && \
-            spack manager external --latest --blacklist nalu-wind gmake && \
+            spack manager external ${EXAWIND_MANAGER}/environments/e4s-build --exclude nalu-wind gmake && \
             spack install && \
             spack cd -b nalu-wind && \
             spack build-env nalu-wind ctest -j $(nproc) -L unit --output-on-failure \

CMakeLists.txt

@@ -56,10 +56,6 @@ endif()
 
 add_library(nalu "")
 
-########################## MPI ####################################
-find_package(MPI REQUIRED)
-target_link_libraries(nalu PUBLIC $<$<BOOL:${MPI_CXX_FOUND}>:MPI::MPI_CXX>)
-
 if(ENABLE_CUDA)
   enable_language(CUDA)
   find_package(CUDAToolkit REQUIRED)
@@ -185,6 +181,7 @@ if(ENABLE_OPENFAST_FSI)
 endif()
 if(ENABLE_OPENFAST)
   set(CMAKE_PREFIX_PATH ${OpenFAST_DIR} ${CMAKE_PREFIX_PATH})
+  enable_language(Fortran)
   find_package(OpenFAST QUIET REQUIRED)
   message(STATUS "Found OpenFAST = ${OpenFAST_LIBRARY_DIRS}")
   target_link_libraries(nalu PUBLIC ${OpenFAST_LIBRARIES} ${OpenFAST_CPP_LIBRARIES})
@@ -205,6 +202,10 @@ if(ENABLE_TIOGA)
   endif()
 endif()
 
+########################## MPI ####################################
+find_package(MPI REQUIRED)
+target_link_libraries(nalu PUBLIC $<$<BOOL:${MPI_CXX_FOUND}>:MPI::MPI_CXX>)
+target_link_libraries(nalu PUBLIC $<$<BOOL:${MPI_Fortran_FOUND}>:MPI::MPI_Fortran>)
 
 ############################ MATRIXREE #####################################
 if(ENABLE_MATRIXFREE)

docs/source/theory/codeAbstractions.rst

@@ -259,12 +259,11 @@ elemental, nodal, face and edge of desired size.
       const int numStates = realm_.number_of_states();
 
       // declare it
-      density_ 
-        =  &(metaData_.declare_field<ScalarFieldType>(stk::topology::NODE_RANK, 
-                                                     "density", numStates));
+      density_  = &(metaData_.declare_field<double>(stk::topology::NODE_RANK,
+                                                    "density", numStates));
 
       // put it on this part
-      stk::mesh::put_field(*density_, *part);
+      stk::mesh::put_field_on_mesh(*density_, *part, nullptr);
     }
 
 :math:`\bullet` edge field registration:
@@ -277,9 +276,11 @@ elemental, nodal, face and edge of desired size.
     {
       const int nDim = metaData_.spatial_dimension();
       edgeAreaVec_ 
-        = &(metaData_.declare_field<VectorFieldType>(stk::topology::EDGE_RANK, 
-                                                    "edge_area_vector"));
-      stk::mesh::put_field(*edgeAreaVec_, *part, nDim);
+        = &(metaData_.declare_field<double>(stk::topology::EDGE_RANK,
+                                            "edge_area_vector"));
+      stk::mesh::put_field_on_mesh(*edgeAreaVec_, *part, nDim, nullptr);
+      stk::io::set_field_output_type(*edgeAreaVec_,
+                                     stk::io::FieldOutputType::VECTOR_3D);
     }
 
 :math:`\bullet` side/face field registration:
@@ -297,9 +298,10 @@ elemental, nodal, face and edge of desired size.
         stk::topology::rank_t sideRank 
           = static_cast<stk::topology::rank_t>(metaData_.side_rank());
         GenericFieldType *wallFrictionVelocityBip 
-          =  &(metaData_.declare_field<GenericFieldType>
+          =  &(metaData_.declare_field<double>
               (sideRank, "wall_friction_velocity_bip"));
-        stk::mesh::put_field(*wallFrictionVelocityBip, *part, numIp);
+        stk::mesh::put_field_on_mesh(*wallFrictionVelocityBip, *part, numIp,
+                                     nullptr);
       }
     }
 
@@ -317,23 +319,21 @@ addition to the field template type,
 .. code-block:: c++
 
     VectorFieldType *velocityRTM 
-      = metaData_.get_field<VectorFieldType>(stk::topology::NODE_RANK, 
-                                            "velocity");
+      = metaData_.get_field<double>(stk::topology::NODE_RANK, "velocity");
     ScalarFieldType *density 
-      = metaData_.get_field<ScalarFieldType>(stk::topology::NODE_RANK, 
-                                            "density");}
+      = metaData_.get_field<double>(stk::topology::NODE_RANK, "density");}
 
     VectorFieldType *edgeAreaVec 
-      = metaData_.get_field<VectorFieldType>(stk::topology::EDGE_RANK, 
-                                            "edge_area_vector");
+      = metaData_.get_field<double>(stk::topology::EDGE_RANK,
+                                    "edge_area_vector");
 
-    GenericFieldType  *massFlowRate
-      = metaData_.get_field<GenericFieldType>(stk::topology::ELEMENT_RANK, 
-                                             "mass_flow_rate_scs");
+    GenericFieldType *massFlowRate
+      = metaData_.get_field<double>(stk::topology::ELEMENT_RANK,
+                                    "mass_flow_rate_scs");
 
     GenericFieldType *wallFrictionVelocityBip_ 
-      = metaData_.get_field<GenericFieldType>(metaData_.side_rank(), 
-                                             "wall_friction_velocity_bip");
+      = metaData_.get_field<double>(metaData_.side_rank(),
+                                    "wall_friction_velocity_bip");
 
 State
 ~~~~~
@@ -346,8 +346,7 @@ extracted,
 .. code-block:: c++
 
     ScalarFieldType *density 
-      = metaData_.get_field<ScalarFieldType>(stk::topology::NODE_RANK, 
-                                            "density");
+      = metaData_.get_field<double>(stk::topology::NODE_RANK, "density");
     densityNm1_ = &(density->field_of_state(stk::mesh::StateNM1));
     densityN_ = &(density->field_of_state(stk::mesh::StateN));
     densityNp1_ = &(density->field_of_state(stk::mesh::StateNP1));

include/Algorithm.h

@@ -25,6 +25,7 @@ class Realm;
 class MasterElement;
 class SupplementalAlgorithm;
 class Kernel;
+class FieldManager;
 
 class Algorithm
 {
@@ -43,6 +44,7 @@ class Algorithm
 
   Realm& realm_;
   stk::mesh::PartVector partVec_;
+  const FieldManager& fieldManager_;
   std::vector<SupplementalAlgorithm*> supplementalAlg_;
 
   std::vector<Kernel*> activeKernels_;

include/AssembleElemSolverAlgorithm.h

@@ -87,7 +87,7 @@ class AssembleElemSolverAlgorithm : public SolverAlgorithm
         auto& b = ngpMesh.get_bucket(entityRank, bktId);
 
 #if !defined(KOKKOS_ENABLE_GPU)
-        ThrowAssertMsg(
+        STK_ThrowAssertMsg(
           b.topology().num_nodes() == (unsigned)nodesPerEntity_,
           "AssembleElemSolverAlgorithm expected nodesPerEntity_ = "
             << nodesPerEntity_

include/AssembleFaceElemSolverAlgorithm.h

@@ -83,7 +83,7 @@ class AssembleFaceElemSolverAlgorithm : public SolverAlgorithm
         auto& b = ngpMesh.get_bucket(sideRank, bktId);
 
 #if !defined(KOKKOS_ENABLE_GPU)
-        ThrowAssertMsg(
+        STK_ThrowAssertMsg(
           b.topology().num_nodes() == (unsigned)nodesPerFace_,
           "AssembleFaceElemSolverAlgorithm expected nodesPerEntity_ = "
             << nodesPerFace_
@@ -111,7 +111,7 @@ class AssembleFaceElemSolverAlgorithm : public SolverAlgorithm
                 stk::mesh::Entity face =
                   b[bktIndex * simdLen + numFacesProcessed + simdFaceIndex];
                 const auto ngpFaceIndex = ngpMesh.fast_mesh_index(face);
-                // ThrowAssertMsg(
+                // STK_ThrowAssertMsg(
                 //   bulk.num_elements(face) == 1,
                 //   "Expecting just 1 element attached to face!");
                 int thisElemFaceOrdinal =

include/CopyAndInterleave.h

@@ -59,11 +59,11 @@ copy_and_interleave(
   for (unsigned viewIndex = 0; viewIndex < numViews; ++viewIndex) {
     for (int simdIndex = 0; simdIndex < simdElems; ++simdIndex) {
       src[simdIndex] = data[simdIndex]->get_1D_view_by_index(viewIndex).data();
-      NGP_ThrowAssert(
+      STK_NGP_ThrowAssert(
         data[simdIndex]->get_1D_view_by_index(viewIndex).size() ==
         simdData.get_1D_view_by_index(viewIndex).size());
-      NGP_ThrowAssert(src[simdIndex] != nullptr);
-      NGP_ThrowAssert(
+      STK_NGP_ThrowAssert(src[simdIndex] != nullptr);
+      STK_NGP_ThrowAssert(
         src[simdIndex][0] ==
         data[simdIndex]->get_1D_view_by_index(viewIndex).data()[0]);
     }
@@ -74,11 +74,11 @@ copy_and_interleave(
   for (unsigned viewIndex = 0; viewIndex < numViews; ++viewIndex) {
     for (int simdIndex = 0; simdIndex < simdElems; ++simdIndex) {
       src[simdIndex] = data[simdIndex]->get_2D_view_by_index(viewIndex).data();
-      NGP_ThrowAssert(
+      STK_NGP_ThrowAssert(
         data[simdIndex]->get_2D_view_by_index(viewIndex).size() ==
         simdData.get_2D_view_by_index(viewIndex).size());
-      NGP_ThrowAssert(src[simdIndex] != nullptr);
-      NGP_ThrowAssert(
+      STK_NGP_ThrowAssert(src[simdIndex] != nullptr);
+      STK_NGP_ThrowAssert(
         src[simdIndex][0] ==
         data[simdIndex]->get_2D_view_by_index(viewIndex).data()[0]);
     }
@@ -89,11 +89,11 @@ copy_and_interleave(
   for (unsigned viewIndex = 0; viewIndex < numViews; ++viewIndex) {
     for (int simdIndex = 0; simdIndex < simdElems; ++simdIndex) {
       src[simdIndex] = data[simdIndex]->get_3D_view_by_index(viewIndex).data();
-      NGP_ThrowAssert(
+      STK_NGP_ThrowAssert(
         data[simdIndex]->get_3D_view_by_index(viewIndex).size() ==
         simdData.get_3D_view_by_index(viewIndex).size());
-      NGP_ThrowAssert(src[simdIndex] != nullptr);
-      NGP_ThrowAssert(
+      STK_NGP_ThrowAssert(src[simdIndex] != nullptr);
+      STK_NGP_ThrowAssert(
         src[simdIndex][0] ==
         data[simdIndex]->get_3D_view_by_index(viewIndex).data()[0]);
     }

include/ElemDataRequests.h

@@ -217,7 +217,7 @@ class ElemDataRequests
   get_coordinates_field(const COORDS_TYPES cType) const
   {
     auto it = coordsFields_.find(cType);
-    NGP_ThrowRequireMsg(
+    STK_NGP_ThrowRequireMsg(
       it != coordsFields_.end(),
       "ElemDataRequests:get_coordinates_field: Coordinates field "
       "must be registered to ElemDataRequests before access");

include/Enums.h

@@ -105,6 +105,7 @@ static const std::string EquationTypeMap[] = {
   "PNG_U",
   "PNG_TKE",
   "Wall_Distance",
+  "Gamma_Transition",
   "Volume_of_Fluid"};
 
 enum UserDataType {

include/FieldDefinitions.h

@@ -15,30 +15,33 @@
 namespace sierra {
 namespace nalu {
 
-template <typename T>
+enum class FieldLayout { SCALAR, VECTOR, TENSOR, ARRAY };
+
+template <typename T, FieldLayout Layout = FieldLayout::SCALAR>
 struct FieldDefinition
 {
-  using FieldType = T;
+  using DataType = T;
   const stk::topology::rank_t rank;
   const int num_states{1};
   const int num_components{1};
+  const FieldLayout layout{Layout};
 };
 
-using FieldDefScalar = FieldDefinition<ScalarFieldType>;
-using FieldDefVector = FieldDefinition<VectorFieldType>;
-using FieldDefTensor = FieldDefinition<TensorFieldType>;
-using FieldDefGeneric = FieldDefinition<GenericFieldType>;
-using FieldDefGenericInt = FieldDefinition<GenericIntFieldType>;
-using FieldDefTpetraId = FieldDefinition<TpetIDFieldType>;
-using FieldDefLocalId = FieldDefinition<LocalIdFieldType>;
-using FieldDefGlobalId = FieldDefinition<GlobalIdFieldType>;
-using FieldDefHypreId = FieldDefinition<HypreIDFieldType>;
-using FieldDefScalarInt = FieldDefinition<ScalarIntFieldType>;
+using FieldDefScalar = FieldDefinition<double>;
+using FieldDefVector = FieldDefinition<double, FieldLayout::VECTOR>;
+using FieldDefTensor = FieldDefinition<double, FieldLayout::TENSOR>;
+using FieldDefGeneric = FieldDefinition<double, FieldLayout::ARRAY>;
+using FieldDefGenericInt = FieldDefinition<int, FieldLayout::ARRAY>;
+using FieldDefTpetraId = FieldDefinition<TpetIdType>;
+using FieldDefLocalId = FieldDefinition<LocalId>;
+using FieldDefGlobalId = FieldDefinition<stk::mesh::EntityId>;
+using FieldDefHypreId = FieldDefinition<HypreIntType>;
+using FieldDefScalarInt = FieldDefinition<int>;
 
 // Type redundancy can occur between HypreId and ScalarInt
 // which will break std::variant
 using FieldDefTypes = std::conditional<
-  std::is_same_v<ScalarIntFieldType, HypreIDFieldType>,
+  std::is_same_v<int, HypreIntType>,
   std::variant<
     FieldDefScalar,
     FieldDefVector,
@@ -61,29 +64,22 @@ using FieldDefTypes = std::conditional<
     FieldDefScalarInt,
     FieldDefHypreId>>::type;
 
+// Trouble!
 using FieldPointerTypes = std::conditional<
-  std::is_same_v<ScalarIntFieldType, HypreIDFieldType>,
+  std::is_same_v<int, HypreIntType>,
   std::variant<
-    ScalarFieldType*,
-    VectorFieldType*,
-    TensorFieldType*,
-    GenericFieldType*,
-    GenericIntFieldType*,
-    TpetIDFieldType*,
-    LocalIdFieldType*,
-    GlobalIdFieldType*,
-    ScalarIntFieldType*>,
+    stk::mesh::Field<double>*,
+    stk::mesh::Field<int>*,
+    stk::mesh::Field<LocalId>*,
+    stk::mesh::Field<stk::mesh::EntityId>*,
+    stk::mesh::Field<TpetIdType>*>,
   std::variant<
-    ScalarFieldType*,
-    VectorFieldType*,
-    TensorFieldType*,
-    GenericFieldType*,
-    GenericIntFieldType*,
-    TpetIDFieldType*,
-    LocalIdFieldType*,
-    GlobalIdFieldType*,
-    ScalarIntFieldType*,
-    HypreIDFieldType*>>::type;
+    stk::mesh::Field<double>*,
+    stk::mesh::Field<int>*,
+    stk::mesh::Field<LocalId>*,
+    stk::mesh::Field<stk::mesh::EntityId>*,
+    stk::mesh::Field<TpetIdType>*,
+    stk::mesh::Field<HypreIntType>*>>::type;
 
 } // namespace nalu
 } // namespace sierra