Add more synchronization for HG race take II.

brobey · Jun 27, 2018 · 20e6a69 · 20e6a69
1 parent 407e50f
commit 20e6a69
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 7 deletions.
diff --git a/src/sdsdkv-client.h b/src/sdsdkv-client.h
@@ -47,9 +47,6 @@ struct sdsdkv_client : public personality {
         //
         // TODO(skg) For both the client and the server, figure out if we want
         // progress threads and what the rpc thread count sould be for both.
-
-        // Wait for server-side setup completion.
-        m_mpi->barrier(m_mpi->get_world_comm());
         //
         m_mid = margo_init(
                     m_config->comm_protocol.c_str(),
@@ -166,12 +163,20 @@ struct sdsdkv_client : public personality {
     int
     open(void)
     {
+        // NOTE(skg): I know all this barrier stuff is ugly and complicated to
+        // get right, but this helps fix an HG race condition...
+        m_mpi->barrier(m_mpi->get_world_comm());
+        //
         int rc = m_margo_init();
         if (rc != SDSDKV_SUCCESS) return rc;
         //
         rc = xchange_gid();
         if (rc != SDSDKV_SUCCESS) return rc;
         //
+        m_mpi->barrier(m_mpi->get_world_comm());
+        //
+        m_mpi->barrier(m_mpi->get_world_comm());
+        //
         rc = m_ssg_init();
         if (rc != SDSDKV_SUCCESS) return rc;
         //

diff --git a/src/sdsdkv-server.h b/src/sdsdkv-server.h
@@ -74,9 +74,6 @@ struct sdsdkv_server : public personality {
         }
         //
         margo_enable_remote_shutdown(m_mid);
-        // Make sure that server setup is done before moving on. Note that the
-        // clients will wait for all servers to init before starting their init.
-        m_mpi->barrier(m_mpi->get_world_comm());
         //
         return SDSDKV_SUCCESS;
     }
@@ -152,17 +149,24 @@ struct sdsdkv_server : public personality {
     int
     open(void)
     {
-        //
+        // NOTE(skg): I know all this barrier stuff is ugly and complicated to
+        // get right, but this helps fix an HG race condition...
         int rc = m_margo_init();
         if (rc != SDSDKV_SUCCESS) return rc;
         //
+        m_mpi->barrier(m_mpi->get_world_comm());
+        //
         rc = m_ssg_init();
         if (rc != SDSDKV_SUCCESS) return rc;
         //
         rc = xchange_gid();
         if (rc != SDSDKV_SUCCESS) return rc;
         //
+        m_mpi->barrier(m_mpi->get_world_comm());
+        //
         rc = m_keyval_register_provider();
+        //
+        m_mpi->barrier(m_mpi->get_world_comm());
         if (rc != SDSDKV_SUCCESS) return rc;
         //
         rc = m_keyval_add_db();