Skip to content

Commit

Permalink
Add more use of the new FieldManager in the unit tests. (#1178)
Browse files Browse the repository at this point in the history
  • Loading branch information
@overfelt
overfelt authored May 3, 2023
1 parent 58ca0ce commit 472317f
Show file tree
Hide file tree
Showing 17 changed files with 122 additions and 836 deletions.
3 changes: 1 addition & 2 deletions include/AssembleElemSolverAlgorithm.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,7 @@ class AssembleElemSolverAlgorithm : public SolverAlgorithm

const stk::mesh::NgpMesh& ngpMesh = realm_.ngp_mesh();
const nalu_ngp::FieldManager& fieldMgr = realm_.ngp_field_manager();
ElemDataRequestsGPU dataNeededNGP(
fieldMgr, dataNeededByKernels_, meta_data.get_fields().size());
ElemDataRequestsGPU dataNeededNGP(fieldMgr, dataNeededByKernels_);

const auto reqType = (entityRank_ == stk::topology::ELEM_RANK)
? ElemReqType::ELEM
Expand Down
5 changes: 2 additions & 3 deletions include/AssembleFaceElemSolverAlgorithm.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ class AssembleFaceElemSolverAlgorithm : public SolverAlgorithm
run_face_elem_algorithm(stk::mesh::BulkData& bulk, LambdaFunction lamdbaFunc)
{
int nDim = bulk.mesh_meta_data().spatial_dimension();
int totalNumFields = bulk.mesh_meta_data().get_fields().size();

// Register face ME instance in elemdata also to obtain face integration
// points
Expand All @@ -60,8 +59,8 @@ class AssembleFaceElemSolverAlgorithm : public SolverAlgorithm

const stk::mesh::NgpMesh& ngpMesh = realm_.ngp_mesh();
const nalu_ngp::FieldManager& fieldMgr = realm_.ngp_field_manager();
ElemDataRequestsGPU faceDataNGP(fieldMgr, faceDataNeeded_, totalNumFields);
ElemDataRequestsGPU elemDataNGP(fieldMgr, elemDataNeeded_, totalNumFields);
ElemDataRequestsGPU faceDataNGP(fieldMgr, faceDataNeeded_);
ElemDataRequestsGPU elemDataNGP(fieldMgr, elemDataNeeded_);

const int bytes_per_team = 0;
const int bytes_per_thread = calculate_shared_mem_bytes_per_thread(
Expand Down
7 changes: 3 additions & 4 deletions include/ElemDataRequestsGPU.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,7 @@ class ElemDataRequestsGPU
FieldInfoView;

template <typename T>
ElemDataRequestsGPU(
const T& fieldMgr, const ElemDataRequests& dataReq, unsigned totalFields);
ElemDataRequestsGPU(const T& fieldMgr, const ElemDataRequests& dataReq);

KOKKOS_FUNCTION ~ElemDataRequestsGPU() {}

Expand Down Expand Up @@ -226,14 +225,14 @@ class ElemDataRequestsGPU

template <typename T>
inline ElemDataRequestsGPU::ElemDataRequestsGPU(
const T& fieldMgr, const ElemDataRequests& dataReq, unsigned totalFields)
const T& fieldMgr, const ElemDataRequests& dataReq)
: dataEnums(),
hostDataEnums(),
coordsFields_(),
hostCoordsFields_(),
coordsFieldsTypes_(),
hostCoordsFieldsTypes_(),
totalNumFields(totalFields),
totalNumFields(fieldMgr.size()),
fields(),
hostFields(),
meFC_(MasterElementRepo::get_surface_dev_ptr_from_host_ptr(
Expand Down
2 changes: 2 additions & 0 deletions include/FieldManager.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include <stk_mesh/base/FieldState.hpp>
#include "stk_mesh/base/GetNgpField.hpp"
#include <string>
#include <type_traits>

namespace stk {
namespace mesh {
Expand Down Expand Up @@ -73,6 +74,7 @@ class FieldManager
/// Check to see if the field has been registered.
bool field_exists(const std::string& name);

unsigned size() const { return meta_.get_fields().size(); }
/// Register a Generic field.
/// A Generic field is of type: SingleStateElemGeneric,
/// SingleStateEdgeGeneric, SingleStateNodeGeneric,... For a generic field the
Expand Down
7 changes: 4 additions & 3 deletions include/FieldRegistry.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,10 @@ class FieldRegistry
auto fieldDefIter = db->find(name);

if (fieldDefIter == db->end()) {
std::string message = "Attempting to access an undefined field: " + name +
" with spatial dimension " + to_string(numDim) +
" and number of states " + to_string(numStates);
std::string message = "Attempting to access an undefined field: '" +
name + "' with spatial dimension " +
to_string(numDim) + " and number of states " +
to_string(numStates);
throw std::runtime_error(message);
}
return fieldDefIter->second;
Expand Down
2 changes: 2 additions & 0 deletions include/ngp_utils/NgpFieldManager.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ class FieldManager
FieldManager(const FieldManager& rhs) = delete;
FieldManager(FieldManager&& rhs) = delete;

unsigned size() const { return m_meta.get_fields().size(); }

template <typename T>
stk::mesh::NgpField<T>& get_field(unsigned fieldOrdinal) const
{
Expand Down
19 changes: 8 additions & 11 deletions include/ngp_utils/NgpLoopUtils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,7 @@ run_elem_algorithm(
const auto& ngpMesh = meshInfo.ngp_mesh();
const auto& fieldMgr = meshInfo.ngp_field_manager();

ElemDataRequestsGPU dataReqNGP(fieldMgr, dataReqs, meshInfo.num_fields());
ElemDataRequestsGPU dataReqNGP(fieldMgr, dataReqs);

const int nodesPerElement = nodes_per_entity(dataReqNGP);
NGP_ThrowRequire(nodesPerElement != 0);
Expand Down Expand Up @@ -433,7 +433,7 @@ run_elem_par_reduce(
const auto& ngpMesh = meshInfo.ngp_mesh();
const auto& fieldMgr = meshInfo.ngp_field_manager();

ElemDataRequestsGPU dataReqNGP(fieldMgr, dataReqs, meshInfo.num_fields());
ElemDataRequestsGPU dataReqNGP(fieldMgr, dataReqs);

const int nodesPerElement = nodes_per_entity(dataReqNGP);
NGP_ThrowRequire(nodesPerElement != 0);
Expand Down Expand Up @@ -518,10 +518,9 @@ run_face_elem_algorithm(
const auto& ndim = meshInfo.ndim();
const auto& ngpMesh = meshInfo.ngp_mesh();
const auto& fieldMgr = meshInfo.ngp_field_manager();
const auto& numFields = meshInfo.num_fields();

ElemDataRequestsGPU faceDataNGP(fieldMgr, faceDataReqs, numFields);
ElemDataRequestsGPU elemDataNGP(fieldMgr, elemDataReqs, numFields);
ElemDataRequestsGPU faceDataNGP(fieldMgr, faceDataReqs);
ElemDataRequestsGPU elemDataNGP(fieldMgr, elemDataReqs);

const int nodesPerElement = nodes_per_entity(elemDataNGP);
const int nodesPerFace = nodes_per_entity(faceDataNGP, METype::FACE);
Expand Down Expand Up @@ -643,10 +642,9 @@ run_face_elem_par_reduce(
const auto& ndim = meshInfo.ndim();
const auto& ngpMesh = meshInfo.ngp_mesh();
const auto& fieldMgr = meshInfo.ngp_field_manager();
const auto& numFields = meshInfo.num_fields();

ElemDataRequestsGPU faceDataNGP(fieldMgr, faceDataReqs, numFields);
ElemDataRequestsGPU elemDataNGP(fieldMgr, elemDataReqs, numFields);
ElemDataRequestsGPU faceDataNGP(fieldMgr, faceDataReqs);
ElemDataRequestsGPU elemDataNGP(fieldMgr, elemDataReqs);

const int nodesPerElement = nodes_per_entity(elemDataNGP);
const int nodesPerFace = nodes_per_entity(faceDataNGP, METype::FACE);
Expand Down Expand Up @@ -772,10 +770,9 @@ run_face_elem_algorithm_nosimd(
const auto& ndim = meshInfo.ndim();
const auto& ngpMesh = meshInfo.ngp_mesh();
const auto& fieldMgr = meshInfo.ngp_field_manager();
const auto& numFields = meshInfo.num_fields();

ElemDataRequestsGPU faceDataNGP(fieldMgr, faceDataReqs, numFields);
ElemDataRequestsGPU elemDataNGP(fieldMgr, elemDataReqs, numFields);
ElemDataRequestsGPU faceDataNGP(fieldMgr, faceDataReqs);
ElemDataRequestsGPU elemDataNGP(fieldMgr, elemDataReqs);

const int nodesPerElement = nodes_per_entity(elemDataNGP);
const int nodesPerFace = nodes_per_entity(faceDataNGP, METype::FACE);
Expand Down
10 changes: 9 additions & 1 deletion src/FieldManager.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,15 @@ FieldManager::register_field(
for (auto&& part : parts) {
stk::mesh::put_field_on_mesh(*id, *part, num_components, init);
}

#if 0
std::cout << "Registring field '" << name << "' on parts:";
for (const auto& part : parts)
std::cout << " '" << part->name() << "'";
std::cout << " with number of states " << num_states;
std::cout << " and spatial dimension " << numDimensions_;
std::cout << " with number of components " << num_components;
std::cout << std::endl;
#endif
return id;
},
definition);
Expand Down
13 changes: 13 additions & 0 deletions src/FieldRegistry.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ Registry()

FieldDefScalar SingleStateNodalScalar = {stk::topology::NODE_RANK};
FieldDefScalar SingleStateElemScalar = {stk::topology::ELEM_RANK};
FieldDefVector SingleStateElemVector = {stk::topology::ELEM_RANK, 1, NUM_DIM};
FieldDefScalar MultiStateNodalScalar = {stk::topology::NODE_RANK, NUM_STATES};

FieldDefVector SingleStateNodalVector = {stk::topology::NODE_RANK, 1, NUM_DIM};
Expand All @@ -46,6 +47,8 @@ Registry()
{"current_coordinates", SingleStateNodalVector},
{"density", SingleStateNodalScalar},
{"dhdx", SingleStateNodalVector},
{"diffFluxCoeff", SingleStateNodalScalar},
{"discreteLaplacian", SingleStateNodalScalar},
{"div_mesh_velocity", SingleStateNodalScalar},
{"dkdx", SingleStateNodalVector},
{"dpdx", SingleStateNodalVector},
Expand All @@ -54,21 +57,31 @@ Registry()
{"dwdx", SingleStateNodalVector},
{"edge_area_vector", SingleStateEdgeVector},
{"effective_viscosity" , SingleStateNodalScalar},
{"elemCentroid", SingleStateElemVector},
{"element_volume", SingleStateElemScalar},
{"elemScalarField", SingleStateElemScalar}, // Used in testing
{"elemTensorField", SingleStateElemGeneric}, // Used in testing
{"elemVectorField", SingleStateElemGeneric}, // Used in testing
{"exposed_area_vector", SingleStateEdgeGeneric},
{"Gju", SingleStateNodeGeneric},
{"hypre_global_id", HypreId},
{"iblank", NodalScalarInt},
{"idField", SingleStateNodalScalar},
{"mass_flow_rate_scs", SingleStateElemGeneric},
{"mesh_displacement", MultiStateNodalVector},
{"mesh_velocity", SingleStateNodalVector},
{"minimum_distance_to_wall", SingleStateNodalScalar},
{"momentum_diag", SingleStateNodalScalar},
{"nalu_global_id", GlobalId},
{"nodalScalarField", SingleStateNodalScalar}, // Used in testing
{"nodalGenericField", SingleStateNodeGeneric}, // Used in testing
{"nodalPressure", SingleStateNodalScalar},
{"nodalTensorField", SingleStateNodalTensor}, // Used in testing
{"open_mass_flow_rate" , SingleStateEdgeGeneric},
{"open_tke_bc", SingleStateNodalScalar},
{"pressure", SingleStateNodalScalar},
{"rans_time_scale" , SingleStateNodalScalar},
{"scalarQ", SingleStateNodalScalar},
{"specific_dissipation_rate", SingleStateNodalScalar},
{"specific_heat" , SingleStateNodalScalar},
{"sst_f_one_blending" , SingleStateNodalScalar},
Expand Down
4 changes: 1 addition & 3 deletions unit_tests/UnitTestElemDataRequests.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,7 @@ do_the_test_gpu(
stk::mesh::BulkData& bulk,
std::shared_ptr<sierra::nalu::FieldManager> fieldMgr)
{
unsigned totalNumFields_guess = 10;
sierra::nalu::ElemDataRequestsGPU ngpDataReq(
*fieldMgr, dataReq, totalNumFields_guess);
sierra::nalu::ElemDataRequestsGPU ngpDataReq(*fieldMgr, dataReq);

unsigned numCorrectTests = 0;
int threadsPerTeam = 1;
Expand Down
18 changes: 12 additions & 6 deletions unit_tests/UnitTestElemSuppAlg.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,23 +129,28 @@ private:
class TestElemAlgorithmWithSuppAlgViews
{
public:
TestElemAlgorithmWithSuppAlgViews(stk::mesh::BulkData& bulk)
: suppAlgs_(), dataNeededByKernels_(bulk.mesh_meta_data()), bulkData_(bulk)
TestElemAlgorithmWithSuppAlgViews(
stk::mesh::BulkData& bulk,
std::shared_ptr<sierra::nalu::FieldManager> fieldMgr)
: suppAlgs_(),
dataNeededByKernels_(bulk.mesh_meta_data()),
fieldManager(fieldMgr),
bulkData_(bulk)
{
}

void execute()
{
const stk::mesh::MetaData& meta = bulkData_.mesh_meta_data();
stk::mesh::MetaData& meta = bulkData_.mesh_meta_data();

const stk::mesh::BucketVector& elemBuckets = bulkData_.get_buckets(
stk::topology::ELEM_RANK, meta.locally_owned_part());

stk::mesh::NgpMesh ngpMesh(bulkData_);
sierra::nalu::nalu_ngp::FieldManager fieldMgr(bulkData_);
const int num_states = 2;

sierra::nalu::ElemDataRequestsGPU dataNeededNGP(
fieldMgr, dataNeededByKernels_, meta.get_fields().size());
*fieldManager, dataNeededByKernels_);
const int bytes_per_team = 0;
const int bytes_per_thread =
sierra::nalu::get_num_bytes_pre_req_data<DoubleType>(
Expand Down Expand Up @@ -179,6 +184,7 @@ public:

std::vector<SuppAlg*> suppAlgs_;
sierra::nalu::ElemDataRequests dataNeededByKernels_;
std::shared_ptr<sierra::nalu::FieldManager> fieldManager;

private:
stk::mesh::BulkData& bulkData_;
Expand All @@ -188,7 +194,7 @@ TEST_F(Hex8Mesh, elem_supp_alg_views)
{
fill_mesh_and_initialize_test_fields("generated:20x20x20");

TestElemAlgorithmWithSuppAlgViews testAlgorithm(*bulk);
TestElemAlgorithmWithSuppAlgViews testAlgorithm(*bulk, fieldManager);

// DiscreteLapacianSuppAlg constructor says which data it needs, by inserting
// things into the 'dataNeededByKernels_' container.
Expand Down
20 changes: 10 additions & 10 deletions unit_tests/UnitTestScratchViews.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ typedef Kokkos::
void
do_the_test(
stk::mesh::BulkData& bulk,
const sierra::nalu::FieldManager& fieldManager,
sierra::nalu::ScalarFieldType* pressure,
sierra::nalu::VectorFieldType* velocity)
{
Expand All @@ -87,18 +88,15 @@ do_the_test(
EXPECT_EQ(3u, dataReq.get_fields().size());

const stk::mesh::MetaData& meta = bulk.mesh_meta_data();
sierra::nalu::nalu_ngp::FieldManager fieldMgr(bulk);

sierra::nalu::ElemDataRequestsGPU dataNGP(
fieldMgr, dataReq, meta.get_fields().size());
sierra::nalu::ElemDataRequestsGPU dataNGP(fieldManager, dataReq);

const int numNodes = elemTopo.num_nodes();
const int rhsSize = numNodes;
const int lhsSize = rhsSize * rhsSize;

const unsigned velocityOrdinal = velocity->mesh_meta_data_ordinal();
const unsigned pressureOrdinal = pressure->mesh_meta_data_ordinal();

const int bytes_per_team = 0;
const int bytes_per_thread =
(sierra::nalu::calculate_shared_mem_bytes_per_thread(
Expand Down Expand Up @@ -146,7 +144,6 @@ do_the_test(
dataNGP, ngpMesh, stk::topology::ELEM_RANK, element, scrviews);
auto& velocityView = scrviews.get_scratch_view_2D(velocityOrdinal);
auto& pressureView = scrviews.get_scratch_view_1D(pressureOrdinal);

result.d_view(0) =
stk::simd::get_data((pressureView(0) - 1.0), 0) < 1.e-9 ? 1 : 0;
result.d_view(1) =
Expand Down Expand Up @@ -177,6 +174,10 @@ TEST_F(Hex8MeshWithNSOFields, NGPScratchViews)
const int nDim = 3;
const double velVec[nDim] = {1.0, 0.0, 0.0};

const int numStates = 2;
stk::mesh::MetaData& meta = bulk->mesh_meta_data();
const sierra::nalu::FieldManager fieldManager(meta, numStates);

stk::mesh::EntityVector nodes;
stk::mesh::get_entities(*bulk, stk::topology::NODE_RANK, nodes);

Expand All @@ -188,7 +189,7 @@ TEST_F(Hex8MeshWithNSOFields, NGPScratchViews)
}
}

do_the_test(*bulk, pressure, velocity);
do_the_test(*bulk, fieldManager, pressure, velocity);
}

void
Expand Down Expand Up @@ -246,6 +247,7 @@ TEST_F(Hex8MeshWithNSOFields, NGPAssembleElemSolver)
void
do_the_smdata_test(
stk::mesh::BulkData& bulk,
const std::shared_ptr<sierra::nalu::FieldManager> fieldManager,
sierra::nalu::ScalarFieldType* pressure,
sierra::nalu::VectorFieldType* velocity)
{
Expand All @@ -268,10 +270,8 @@ do_the_smdata_test(
EXPECT_EQ(3u, dataReq.get_fields().size());

const stk::mesh::MetaData& meta = bulk.mesh_meta_data();
sierra::nalu::nalu_ngp::FieldManager fieldMgr(bulk);

sierra::nalu::ElemDataRequestsGPU dataNGP(
fieldMgr, dataReq, meta.get_fields().size());
sierra::nalu::ElemDataRequestsGPU dataNGP(*fieldManager, dataReq);

const int numNodes = elemTopo.num_nodes();
const int rhsSize = numNodes;
Expand Down Expand Up @@ -354,7 +354,7 @@ TEST_F(Hex8MeshWithNSOFields, NGPSharedMemData)
if (stk::parallel_machine_size(comm) == 1) {
fill_mesh_and_initialize_test_fields("generated:2x2x2");

do_the_smdata_test(*bulk, pressure, velocity);
do_the_smdata_test(*bulk, fieldManager, pressure, velocity);
}
}

Expand Down
Loading

0 comments on commit 472317f

Please sign in to comment.