From b310e7e09416b029cf51c4bc9831592a620afe02 Mon Sep 17 00:00:00 2001 From: psakiev Date: Wed, 3 May 2023 16:55:41 -0600 Subject: [PATCH] Example usage of FieldManager This PR demonstrates how a Kernel can be converted to use a FieldManager. The main goal is to move the field registration away from higher level abstractions like realm and equationsystem, to lower ones (algs and kernels). This will essentially eliminate the need for `RegisterNodalFields` type calls, and will eliminate a range of bugs that can come in from not registering fields in the right place. Having kernels and algs register the fields they need will also reduce the amount of fixturing needed for unit tests since constructing the object will register the fields for you. You can then finalize the mesh and populate the fields with initial values. (Not demonstrated in this PR). --- include/FieldManager.h | 25 ++++++++++---- include/node_kernels/TKESSTNodeKernel.h | 14 ++++---- src/FieldManager.C | 4 +-- src/TurbKineticEnergyEquationSystem.C | 3 +- src/node_kernels/TKESSTNodeKernel.C | 34 ++++++++++--------- .../node_kernels/UnitTestSSTNodeKernel.C | 8 +++-- 6 files changed, 52 insertions(+), 36 deletions(-) diff --git a/include/FieldManager.h b/include/FieldManager.h index d84572185c..8777d21cfa 100644 --- a/include/FieldManager.h +++ b/include/FieldManager.h @@ -14,7 +14,6 @@ #include #include "stk_mesh/base/GetNgpField.hpp" #include -#include namespace stk { namespace mesh { @@ -62,12 +61,12 @@ class FieldManager T* register_field( const std::string& name, const stk::mesh::PartVector& parts, - const void* init_val = nullptr, + const void* initVal = nullptr, stk::mesh::FieldState state = stk::mesh::FieldState::StateNone) { const int numStates = 0; const int numComponents = 0; - register_field(name, parts, numStates, numComponents, init_val); + register_field(name, parts, numStates, numComponents, initVal); return get_field_ptr(name, state); } @@ -84,10 +83,10 @@ class FieldManager const stk::mesh::PartVector& parts, const int numStates, const int numComponents, - const void* init_val = nullptr, + const void* initVal = nullptr, stk::mesh::FieldState state = stk::mesh::FieldState::StateNone) { - register_field(name, parts, numStates, numComponents, init_val); + register_field(name, parts, numStates, numComponents, initVal); return get_field_ptr(name, state); } @@ -116,7 +115,7 @@ class FieldManager /// would otherwise be defined in the field Registry class. /// /// If numStates = 0 then the number of states comes from the - /// field Registry. Same for numComponents = 0 and init_val = nullptr. + /// field Registry. Same for numComponents = 0 and initVal = nullptr. /// /// This is useful for dynamic fields that depend on the input /// options to define the number of states or number of components since the @@ -128,7 +127,19 @@ class FieldManager const stk::mesh::PartVector& parts, const int numStates = 0, const int numComponents = 0, - const void* init_val = nullptr); + const void* initVal = nullptr) const; + + FieldPointerTypes register_field( + const std::string& name, + stk::mesh::Part* part, + const int numStates = 0, + const int numComponents = 0, + const void* initVal = nullptr) const + { + stk::mesh::PartVector parts; + parts.push_back(part); + register_field(name, parts, numStates, numComponents, initVal); + } /// Given the named field that has already been registered on the CPU /// return the GPU version of the same field. diff --git a/include/node_kernels/TKESSTNodeKernel.h b/include/node_kernels/TKESSTNodeKernel.h index c85f20e04f..766bb52d8a 100644 --- a/include/node_kernels/TKESSTNodeKernel.h +++ b/include/node_kernels/TKESSTNodeKernel.h @@ -10,12 +10,14 @@ #ifndef TKESSTNODEKERNEL_H #define TKESSTNODEKERNEL_H +#include "LinearSystem.h" #include "node_kernels/NodeKernel.h" #include "FieldTypeDef.h" #include "stk_mesh/base/BulkData.hpp" #include "stk_mesh/base/Ngp.hpp" #include "stk_mesh/base/NgpField.hpp" #include "stk_mesh/base/Types.hpp" +#include namespace sierra { namespace nalu { @@ -25,7 +27,10 @@ class Realm; class TKESSTNodeKernel : public NGPNodeKernel { public: - TKESSTNodeKernel(const stk::mesh::MetaData&); + TKESSTNodeKernel( + const stk::mesh::MetaData&, + const FieldManager&, + stk::mesh::Part* part); TKESSTNodeKernel() = delete; @@ -48,13 +53,6 @@ class TKESSTNodeKernel : public NGPNodeKernel stk::mesh::NgpField dudx_; stk::mesh::NgpField dualNodalVolume_; - unsigned tkeID_{stk::mesh::InvalidOrdinal}; - unsigned sdrID_{stk::mesh::InvalidOrdinal}; - unsigned densityID_{stk::mesh::InvalidOrdinal}; - unsigned tviscID_{stk::mesh::InvalidOrdinal}; - unsigned dudxID_{stk::mesh::InvalidOrdinal}; - unsigned dualNodalVolumeID_{stk::mesh::InvalidOrdinal}; - NodeKernelTraits::DblType betaStar_; NodeKernelTraits::DblType tkeProdLimitRatio_; NodeKernelTraits::DblType tkeAmb_; diff --git a/src/FieldManager.C b/src/FieldManager.C index 1e30e72631..b778a25540 100644 --- a/src/FieldManager.C +++ b/src/FieldManager.C @@ -36,7 +36,7 @@ FieldManager::register_field( const stk::mesh::PartVector& parts, const int numStates, const int numComponents, - const void* init_val) + const void* initVal) const { auto definition = FieldRegistry::query(numDimensions_, numStates_, name); @@ -48,7 +48,7 @@ FieldManager::register_field( const int num_components = numComponents ? numComponents : def.num_components; - const val_type* init = static_cast(init_val); + const val_type* init = static_cast(initVal); auto* id = &(meta_.declare_field(def.rank, name, num_states)); for (auto&& part : parts) { stk::mesh::put_field_on_mesh(*id, *part, num_components, init); diff --git a/src/TurbKineticEnergyEquationSystem.C b/src/TurbKineticEnergyEquationSystem.C index 0f7d988af9..54cde32397 100644 --- a/src/TurbKineticEnergyEquationSystem.C +++ b/src/TurbKineticEnergyEquationSystem.C @@ -303,7 +303,8 @@ TurbKineticEnergyEquationSystem::register_interior_algorithm( nodeAlg.add_kernel(realm_.meta_data()); break; case TurbulenceModel::SST: - nodeAlg.add_kernel(realm_.meta_data()); + nodeAlg.add_kernel( + realm_.meta_data(), *(realm_.fieldManager_.get()), part); break; case TurbulenceModel::SSTLR: nodeAlg.add_kernel(realm_.meta_data()); diff --git a/src/node_kernels/TKESSTNodeKernel.C b/src/node_kernels/TKESSTNodeKernel.C index 6fe7202f1c..f4486d305c 100644 --- a/src/node_kernels/TKESSTNodeKernel.C +++ b/src/node_kernels/TKESSTNodeKernel.C @@ -19,29 +19,31 @@ namespace sierra { namespace nalu { -TKESSTNodeKernel::TKESSTNodeKernel(const stk::mesh::MetaData& meta) - : NGPNodeKernel(), - tkeID_(get_field_ordinal(meta, "turbulent_ke")), - sdrID_(get_field_ordinal(meta, "specific_dissipation_rate")), - densityID_(get_field_ordinal(meta, "density")), - tviscID_(get_field_ordinal(meta, "turbulent_viscosity")), - dudxID_(get_field_ordinal(meta, "dudx")), - dualNodalVolumeID_(get_field_ordinal(meta, "dual_nodal_volume")), - nDim_(meta.spatial_dimension()) +TKESSTNodeKernel::TKESSTNodeKernel( + const stk::mesh::MetaData& meta, + const FieldManager& manager, + stk::mesh::Part* part) + : NGPNodeKernel(), nDim_(meta.spatial_dimension()) { + manager.register_field("turbulent_ke", part); + manager.register_field("specific_dissipation_rate", part); + manager.register_field("density", part); + manager.register_field("turbulent_viscosity", part); + manager.register_field("dudx", part); + manager.register_field("dual_nodal_volume", part); } void TKESSTNodeKernel::setup(Realm& realm) { - const auto& fieldMgr = realm.ngp_field_manager(); + const auto& fieldMgr = *(realm.fieldManager_.get()); - tke_ = fieldMgr.get_field(tkeID_); - sdr_ = fieldMgr.get_field(sdrID_); - density_ = fieldMgr.get_field(densityID_); - tvisc_ = fieldMgr.get_field(tviscID_); - dudx_ = fieldMgr.get_field(dudxID_); - dualNodalVolume_ = fieldMgr.get_field(dualNodalVolumeID_); + tke_ = fieldMgr.get_ngp_field_ptr("turbulent_ke"); + sdr_ = fieldMgr.get_ngp_field_ptr("specific_dissipation_rate"); + density_ = fieldMgr.get_ngp_field_ptr("density"); + tvisc_ = fieldMgr.get_ngp_field_ptr("turbulent_viscosity"); + dudx_ = fieldMgr.get_ngp_field_ptr("dudx"); + dualNodalVolume_ = fieldMgr.get_ngp_field_ptr("dual_nodal_volume"); // Update turbulence model constants betaStar_ = realm.get_turb_model_constant(TM_betaStar); diff --git a/unit_tests/node_kernels/UnitTestSSTNodeKernel.C b/unit_tests/node_kernels/UnitTestSSTNodeKernel.C index d41791f038..acf24511ce 100644 --- a/unit_tests/node_kernels/UnitTestSSTNodeKernel.C +++ b/unit_tests/node_kernels/UnitTestSSTNodeKernel.C @@ -636,6 +636,8 @@ TEST_F(SSTKernelHex8Mesh, NGP_tke_sst_node) if (bulk_->parallel_size() > 1) return; + // TDODO we can eliminate all the excess fields if we decide to do our field + // init after we add the kernels fill_mesh_and_init_fields(); // Setup solution options @@ -646,7 +648,8 @@ TEST_F(SSTKernelHex8Mesh, NGP_tke_sst_node) unit_test_utils::NodeHelperObjects helperObjs( bulk_, stk::topology::HEX_8, 1, partVec_[0]); - helperObjs.nodeAlg->add_kernel(*meta_); + helperObjs.nodeAlg->add_kernel( + *meta_, *(helperObjs.realm.fieldManager_.get()), partVec_[0]); helperObjs.execute(); @@ -685,7 +688,8 @@ TEST_F(SSTKernelHex8Mesh, NGP_tke_sst_sust_node) realm.solutionOptions_->turbModelConstantMap_[sierra::nalu::TM_tkeAmb] = 5.0; realm.solutionOptions_->turbModelConstantMap_[sierra::nalu::TM_sdrAmb] = 50.0; - helperObjs.nodeAlg->add_kernel(*meta_); + helperObjs.nodeAlg->add_kernel( + *meta_, *(realm.fieldManager_.get()), partVec_[0]); helperObjs.execute();