From c7ea3fd4a0bbf9a8fd4569178a7b23169d81c790 Mon Sep 17 00:00:00 2001 From: "Stefan P. Domino" Date: Fri, 21 Sep 2018 14:08:27 -0600 Subject: [PATCH] Deprecate internal usage of NaluComm in favor of STK (#405) as per: https://github.com/Exawind/nalu-wind/pull/40/commits/77787807d043521176d273b3b306e4b24fc4c8d1 Notes: Ready for tag and release. --- include/NaluCommBufferV.hpp | 123 ------------------ include/NaluCommNeighbors.hpp | 140 --------------------- include/TpetraLinearSystem.h | 6 +- src/NaluCommNeighbors.C | 226 ---------------------------------- src/TpetraLinearSystem.C | 22 ++-- 5 files changed, 14 insertions(+), 503 deletions(-) delete mode 100644 include/NaluCommBufferV.hpp delete mode 100644 include/NaluCommNeighbors.hpp delete mode 100644 src/NaluCommNeighbors.C diff --git a/include/NaluCommBufferV.hpp b/include/NaluCommBufferV.hpp deleted file mode 100644 index a6b20479..00000000 --- a/include/NaluCommBufferV.hpp +++ /dev/null @@ -1,123 +0,0 @@ -// ***** TEMPORARY COPY OF STK FILE ******** -// Will be delete after stk is updated with performance improvements in -// stk_util/parallel/CommBufferV.hpp and stk/parallel/CommNeighbors.[hpp,cpp] -// ***** TEMPORARY COPY OF STK FILE ******** -// -// Copyright (c) 2013, Sandia Corporation. -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following -// disclaimer in the documentation and/or other materials provided -// with the distribution. -// -// * Neither the name of Sandia Corporation nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// - -#ifndef nalu_CommBufferV_hpp -#define nalu_CommBufferV_hpp - -#include -#include -#include - -//------------------------------------------------------------------------ - -namespace nalu_stk { - -class CommBufferV { -public: - CommBufferV() - : data_buffer(), unpack_iterator(data_buffer.begin()) {} - ~CommBufferV(){} - - size_t size_in_bytes() const { return data_buffer.end() - unpack_iterator; } - size_t capacity_in_bytes() const { return data_buffer.capacity(); } - - void reserve(size_t num_bytes) { - data_buffer.reserve(num_bytes); - unpack_iterator = data_buffer.begin(); - } - - void resize(size_t num_bytes) { - data_buffer.resize(num_bytes); - unpack_iterator = data_buffer.begin(); - } - - unsigned char* raw_buffer() { return data_buffer.data(); } - const unsigned char* raw_buffer() const { return data_buffer.data(); } - - template - void pack(const T& item) { - pack_internal(&item, 1); - } - - template - void pack(const T* items, size_t num_items) { - pack_internal(items, num_items); - } - - template - void unpack(T& item) { - enum { item_size_in_bytes = sizeof(T) }; - unsigned char* char_ptr = &(*unpack_iterator); - T* item_to_unpack = reinterpret_cast(char_ptr); - item = *item_to_unpack; - unpack_iterator += item_size_in_bytes; - } - - template - void unpack(T* items, size_t num_items) { - enum { item_size_in_bytes = sizeof(T) }; - unsigned char* char_ptr = &(*unpack_iterator); - T* items_to_unpack = reinterpret_cast(char_ptr); - size_t num_bytes = item_size_in_bytes * num_items; - std::memcpy(items, items_to_unpack, num_bytes); - unpack_iterator += item_size_in_bytes * num_items; - } - -private: - template - void pack_internal(const T* items, size_t num_items) { - enum { item_size_in_bytes = sizeof(T) }; - const size_t num_bytes = item_size_in_bytes*num_items; - if (num_bytes > (data_buffer.capacity() - data_buffer.size())) { - data_buffer.reserve(std::max(num_bytes, data_buffer.capacity()*2)); - } - const unsigned char* item_chars = reinterpret_cast(items); - data_buffer.insert(data_buffer.end(), item_chars, item_chars+num_bytes); - unpack_iterator = data_buffer.begin(); - } - - std::vector data_buffer; - std::vector::iterator unpack_iterator; -}; - -} - -//---------------------------------------------------------------------- - -#endif - diff --git a/include/NaluCommNeighbors.hpp b/include/NaluCommNeighbors.hpp deleted file mode 100644 index 1f44398f..00000000 --- a/include/NaluCommNeighbors.hpp +++ /dev/null @@ -1,140 +0,0 @@ -// ***** TEMPORARY COPY OF STK FILE ******** -// Will be delete after stk is updated with performance improvements in -// stk_util/parallel/CommBufferV.hpp and stk/parallel/CommNeighbors.[hpp,cpp] -// ***** TEMPORARY COPY OF STK FILE ******** -// -// Copyright (c) 2013, Sandia Corporation. -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following -// disclaimer in the documentation and/or other materials provided -// with the distribution. -// -// * Neither the name of Sandia Corporation nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// - -#ifndef nalu_CommNeighbors_hpp -#define nalu_CommNeighbors_hpp - -#include // for size_t, ptrdiff_t -#include -#include // for ParallelMachine -#include - -//------------------------------------------------------------------------ - -namespace nalu_stk { - -class CommNeighbors { -public: - - stk::ParallelMachine parallel() const { return m_comm ; } - int parallel_size() const { return m_size ; } - int parallel_rank() const { return m_rank ; } - - /** Obtain the message buffer for a given processor */ - CommBufferV & send_buffer( int p ) - { -#ifndef NDEBUG - if ( m_size <= p ) { rank_error("send_buffer",p); } -#endif - return m_send[p] ; - } - - /** Obtain the message buffer for a given processor */ - CommBufferV & recv_buffer( int p ) - { -#ifndef NDEBUG - if ( m_size <= p ) { rank_error("recv_buffer",p); } -#endif - return m_recv[p] ; - } - - //---------------------------------------- - /** Construct for a to-be-sized communication. - * Comm scenario: - * 1) constructor argument 'neighbor_procs' specifies the processors - * which may be communicated with (neighbors==send-procs==recv-procs). - * 2) send-buffers are packed with data to be sent - * All processors sent to, must be members of neighbor_procs. - * 3) communicate() performs the communication and stores recvd data - * in recv_buffers. - * All processors recvd from, are members of neighbor_procs. - */ - CommNeighbors( stk::ParallelMachine comm, const std::vector& neighbor_procs ); - CommNeighbors( stk::ParallelMachine comm, const std::vector& send_procs, const std::vector& recv_procs ); - - //---------------------------------------- - /** Communicate send buffers to receive buffers. */ - void communicate(); - - /** Reset, but do not reallocate, message buffers for reprocessing. - * Sets 'size() == 0' and 'remaining() == capacity()'. - */ - void reset_buffers(); - - ~CommNeighbors(); - -protected: - - virtual stk::ParallelMachine setup_neighbor_comm(stk::ParallelMachine fullComm, - const std::vector& sendProcs, - const std::vector& recvProcs); - - virtual void perform_neighbor_communication(MPI_Comm neighborComm, - const std::vector& sendBuf, - const std::vector& sendCounts, - const std::vector& sendDispls, - std::vector& recvBuf, - std::vector& recvCounts, - std::vector& recvDispls); - //---------------------------------------- - /** default Constructor not allowed - */ - CommNeighbors() = delete; - - CommNeighbors( const CommNeighbors & ); - CommNeighbors & operator = ( const CommNeighbors & ); - - void rank_error( const char * , int ) const ; - - stk::ParallelMachine m_comm ; - int m_size ; - int m_rank ; - std::vector m_send; - std::vector m_recv; - std::vector m_send_data; - std::vector m_recv_data; - std::vector m_send_procs; - std::vector m_recv_procs; -}; - -} - -//---------------------------------------------------------------------- - -#endif - diff --git a/include/TpetraLinearSystem.h b/include/TpetraLinearSystem.h index 14a98736..5a888de2 100644 --- a/include/TpetraLinearSystem.h +++ b/include/TpetraLinearSystem.h @@ -25,7 +25,7 @@ #include #include -namespace nalu_stk { +namespace stk { class CommNeighbors; } @@ -148,13 +148,13 @@ class TpetraLinearSystem : public LinearSystem void compute_send_lengths(const std::vector& rowEntities, const std::vector >& connections, const std::vector& neighborProcs, - nalu_stk::CommNeighbors& commNeighbors); + stk::CommNeighbors& commNeighbors); void compute_graph_row_lengths(const std::vector& rowEntities, const std::vector >& connections, LinSys::RowLengths& sharedNotOwnedRowLengths, LinSys::RowLengths& locallyOwnedRowLengths, - nalu_stk::CommNeighbors& commNeighbors); + stk::CommNeighbors& commNeighbors); void insert_graph_connections(const std::vector& rowEntities, const std::vector >& connections, diff --git a/src/NaluCommNeighbors.C b/src/NaluCommNeighbors.C deleted file mode 100644 index 19bd09dc..00000000 --- a/src/NaluCommNeighbors.C +++ /dev/null @@ -1,226 +0,0 @@ -// ***** TEMPORARY COPY OF STK FILE ******** -// Will be delete after stk is updated with performance improvements in -// stk_util/parallel/CommBufferV.hpp and stk/parallel/CommNeighbors.[hpp,cpp] -// ***** TEMPORARY COPY OF STK FILE ******** -// -// Copyright (c) 2013, Sandia Corporation. -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following -// disclaimer in the documentation and/or other materials provided -// with the distribution. -// -// * Neither the name of Sandia Corporation nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace nalu_stk { - -//----------------------------------------------------------------------- - -#if defined( STK_HAS_MPI ) - -void CommNeighbors::rank_error( const char * method , int p ) const -{ - std::ostringstream os ; - os << "stk::CommNeighbors::" << method - << "(" << p << ") ERROR: Not in [0:" << m_size << ")" ; - throw std::range_error( os.str() ); -} - -//---------------------------------------------------------------------- - -CommNeighbors::~CommNeighbors() -{ - m_comm = stk::parallel_machine_null(); - m_size = 0 ; - m_rank = 0 ; -} - -stk::ParallelMachine CommNeighbors::setup_neighbor_comm(stk::ParallelMachine fullComm, - const std::vector& sendProcs, - const std::vector& recvProcs) -{ - MPI_Info info; - MPI_Info_create(&info); - int reorder = 0; - const int* weights = (int*)MPI_UNWEIGHTED; - stk::ParallelMachine neighborComm; - MPI_Dist_graph_create_adjacent(fullComm, - recvProcs.size(), recvProcs.data(), weights, - sendProcs.size(), sendProcs.data(), weights, - info, reorder, &neighborComm); - MPI_Info_free(&info); - return neighborComm; -} - -CommNeighbors::CommNeighbors( stk::ParallelMachine comm, const std::vector& neighbor_procs) - : m_comm( comm ), - m_size( stk::parallel_machine_size( comm ) ), - m_rank( stk::parallel_machine_rank( comm ) ), - m_send(), - m_recv(), - m_send_data(), - m_recv_data(), - m_send_procs(neighbor_procs), - m_recv_procs(neighbor_procs) -{ - m_send.resize(m_size); - m_recv.resize(m_size); - stk::util::sort_and_unique(m_send_procs); - stk::util::sort_and_unique(m_recv_procs); - - m_comm = setup_neighbor_comm(comm, m_send_procs, m_recv_procs); -} - -CommNeighbors::CommNeighbors( stk::ParallelMachine comm, const std::vector& send_procs, const std::vector& recv_procs) - : m_comm( comm ), - m_size( stk::parallel_machine_size( comm ) ), - m_rank( stk::parallel_machine_rank( comm ) ), - m_send(), - m_recv(), - m_send_data(), - m_recv_data(), - m_send_procs(send_procs), - m_recv_procs(recv_procs) -{ - m_send.resize(m_size); - m_recv.resize(m_size); - stk::util::sort_and_unique(m_send_procs); - stk::util::sort_and_unique(m_recv_procs); - - m_comm = setup_neighbor_comm(comm, m_send_procs, m_recv_procs); -} - -void setup_buffers(const std::vector& procs, - const std::vector& data, - std::vector& counts, - std::vector& displs, - std::vector& buf) -{ - counts.resize(procs.size()); - displs.resize(procs.size()); - - int totalBytes = 0; - for(size_t i=0; i& recvBuf, - const std::vector& recvCounts, - const std::vector& recvDispls, - const std::vector& recvProcs, - std::vector& m_recv) -{ - for(size_t i=0; i& sendBuf, - const std::vector& sendCounts, - const std::vector& sendDispls, - std::vector& recvBuf, - std::vector& recvCounts, - std::vector& recvDispls) -{ - ThrowAssertMsg(sendCounts.size()==m_send_procs.size(), "Error, sendCounts should be same size as m_send_procs."); - ThrowAssertMsg(sendDispls.size()==m_send_procs.size(), "Error, sendDispls should be same size as m_send_procs."); - ThrowAssertMsg(recvCounts.size()==m_recv_procs.size(), "Error, recvCounts should be same size as m_recv_procs."); - ThrowAssertMsg(recvDispls.size()==m_recv_procs.size(), "Error, recvDispls should be same size as m_recv_procs."); - - MPI_Neighbor_alltoall(sendCounts.data(), 1, MPI_INT, - recvCounts.data(), 1, MPI_INT, neighborComm); - - int totalRecv = 0; - for(size_t i=0; i sendCounts, recvCounts(m_recv_procs.size(), 0); - std::vector sendDispls, recvDispls(m_recv_procs.size(), 0); - std::vector sendBuf, recvBuf; - - setup_buffers(m_send_procs, m_send, sendCounts, sendDispls, sendBuf); - - perform_neighbor_communication(m_comm, sendBuf, sendCounts, sendDispls, - recvBuf, recvCounts, recvDispls); - - store_recvd_data(recvBuf, recvCounts, recvDispls, m_recv_procs, m_recv); -} - -#endif - -} - diff --git a/src/TpetraLinearSystem.C b/src/TpetraLinearSystem.C index ddc07697..6c81f5a3 100644 --- a/src/TpetraLinearSystem.C +++ b/src/TpetraLinearSystem.C @@ -26,7 +26,7 @@ #include #include -#include +#include #include #include #include @@ -674,7 +674,7 @@ void add_to_length(ViewType& v_owned, ViewType& v_shared, unsigned numDof, } void add_lengths_to_comm(const stk::mesh::BulkData& bulk, - nalu_stk::CommNeighbors& commNeighbors, + stk::CommNeighbors& commNeighbors, int entity_a_owner, stk::mesh::EntityId entityId_a, unsigned numDof, @@ -683,7 +683,7 @@ void add_lengths_to_comm(const stk::mesh::BulkData& bulk, const int* colOwners) { int owner = entity_a_owner; - nalu_stk::CommBufferV& sbuf = commNeighbors.send_buffer(owner); + stk::CommBufferV& sbuf = commNeighbors.send_buffer(owner); GlobalOrdinal rowGid = GID_(entityId_a, numDof , 0); sbuf.pack(rowGid); @@ -697,7 +697,7 @@ void add_lengths_to_comm(const stk::mesh::BulkData& bulk, void communicate_remote_columns(const stk::mesh::BulkData& bulk, const std::vector& neighborProcs, - nalu_stk::CommNeighbors& commNeighbors, + stk::CommNeighbors& commNeighbors, unsigned numDof, const Teuchos::RCP& ownedRowsMap, Kokkos::View& deviceLocallyOwnedRowLengths, @@ -706,7 +706,7 @@ void communicate_remote_columns(const stk::mesh::BulkData& bulk, commNeighbors.communicate(); for(int p : neighborProcs) { - nalu_stk::CommBufferV& rbuf = commNeighbors.recv_buffer(p); + stk::CommBufferV& rbuf = commNeighbors.recv_buffer(p); size_t bufSize = rbuf.size_in_bytes(); while(rbuf.size_in_bytes() > 0) { GlobalOrdinal rowGid = 0; @@ -748,7 +748,7 @@ void TpetraLinearSystem::compute_send_lengths(const std::vector& rowEntities, const std::vector >& connections, const std::vector& neighborProcs, - nalu_stk::CommNeighbors& commNeighbors) + stk::CommNeighbors& commNeighbors) { const stk::mesh::BulkData& bulk = realm_.bulk_data(); std::vector sendLengths(neighborProcs.size(), 0); @@ -792,7 +792,7 @@ TpetraLinearSystem::compute_send_lengths(const std::vector& r } for(size_t i=0; i >& connections, LinSys::RowLengths& sharedNotOwnedRowLengths, LinSys::RowLengths& locallyOwnedRowLengths, - nalu_stk::CommNeighbors& commNeighbors) + stk::CommNeighbors& commNeighbors) { Kokkos::View deviceSharedNotOwnedRowLengths = sharedNotOwnedRowLengths.view(); Kokkos::View deviceLocallyOwnedRowLengths = locallyOwnedRowLengths.view(); @@ -915,7 +915,7 @@ TpetraLinearSystem::insert_graph_connections(const std::vector& neighborProcs, - nalu_stk::CommNeighbors& commNeighbors, + stk::CommNeighbors& commNeighbors, unsigned numDof, LocalGraphArrays& ownedGraph, const LinSys::Map& rowMap, @@ -923,7 +923,7 @@ void insert_communicated_col_indices(const std::vector& neighborProcs, { std::vector colLids; for(int p : neighborProcs) { - nalu_stk::CommBufferV& rbuf = commNeighbors.recv_buffer(p); + stk::CommBufferV& rbuf = commNeighbors.recv_buffer(p); while(rbuf.size_in_bytes() > 0) { stk::mesh::EntityId rowGid = 0; rbuf.unpack(rowGid); @@ -1263,7 +1263,7 @@ TpetraLinearSystem::finalizeLinearSystem() std::vector neighborProcs; fill_neighbor_procs(neighborProcs, bulkData, realm_); - nalu_stk::CommNeighbors commNeighbors(bulkData.parallel(), neighborProcs); + stk::CommNeighbors commNeighbors(bulkData.parallel(), neighborProcs); compute_send_lengths(ownedAndSharedNodes_, connections_, neighborProcs, commNeighbors); compute_graph_row_lengths(ownedAndSharedNodes_, connections_, sharedNotOwnedRowLengths, locallyOwnedRowLengths, commNeighbors);