Updating distributing key-value for newer mochi components

README.md

@@ -25,25 +25,54 @@ spack bootstrap
 
 ```
 # Checkout sds-repo
-git clone https://xgitlab.cels.anl.gov/sds/sds-repo.git
+git clone https://github.com/mochi-hpc/mochi-spack-packages.git
 # Add to spack.
-spack repo add sds-repo
+spack repo add mochi-spack-packages
 # Install via spack.
-spack install --dirty sdskeyval+leveldb
-spack install --dirty ssg+mpi
-spack install --dirty ch-placement
+#spack install --dirty mochi-sdskv+leveldb
+#spack install --dirty mochi-ssg+mpi
+#spack install --dirty mochi-ch-placement
+
+
+# Be sure and have libfabric build the proper network layers. For the trivial test,
+# the tcp and rxm network layers are required. To build these, add the variants to
+# the spack packages in ~/.spack/packages.yaml or ~/.spack/linux/packages.yaml
+#   libfabric:
+#     variants: fabrics=tcp,rxm
+
+spack install mochi-sdskv^[email protected] mochi-ssg+mpi mochi-ch-placement pkg-config
 # Refresh spack environment.
 source spack/share/spack/setup-env.sh
 # Load required modules.
 #spack load -r autoconf
 #spack load -r automake
 #spack load -r libtool
-source <(spack module tcl loads --dependencies sdskeyval ssg ch-placement autoconf automake libtool)
+source <(spack module tcl loads mochi-sdskv mochi-ssg mochi-ch-placement autoconf automake libtool pkg-config)
 ```
 
 ### Get and build sdsdkv.
 
 ```
-git clone https://xgitlab.cels.anl.gov/sds/sdsdkv.git
-cd sdsdkv && autoreconf --force --install && ./configure CXX=mpic++ && make
+git clone https://github.com/mochi-hpc/mochi-sdsdkv.git
+cd mochi-sdsdkv && autoreconf --force --install && ./configure CXX=mpic++ && make
 ```
+
+Some of the possible errors and their solutions:
+
+configure.ac:53: error: possibly undefined macro: AC\_MSG\_ERROR
+   This is caused by not having pkg-config
+
+sdsdkv-config.h:17:10: fatal error: sdskv-common.h: No such file or directory
+   17 | #include "sdskv-common.h"
+
+   This is from a change in behavior of the PKG\_CHECK\_MODULES macro in configure.ac. A fix has been added
+   to append the the paths to the CXX\_FLAGS and LIBS variables
+
+sdsdkv-misci.h:29:15: error: ‘SDSKV\_ERR\_MERCURY’ was not declared in this scope; did you mean ‘SDSKV\_ERR\_ERASE’?
+   29 |         case (SDSKV\_ERR\_MERCURY):     return SDSDKV\_ERR\_SERVICE;
+
+   The error codes have changed in SDSKV. Check the versions of SDSKV and SDSDKV.
+
+[error] Could not initialize hg\_class while running the trivial test
+
+    Check that you are building libfabric with tcp and rxm support

cleanup.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+#spack repo rm sds
+#spack mirror rm local_filesystem
+spack module tcl rm -y
+rm -f ${SPACK_ROOT}/share/spack/dotkit/linux-centos7-x86_64/*.dk
+spack uninstall --all -y
+spack clean --all
+#rm -f modules
+#rm -f ${HOME}/.spack/linux/packages.yaml
+#rm -f ${HOME}/.spack/linux/compilers.yaml
+
+#cd sds-repo
+#git checkout packages/sdskeyval/package.py
+#cd ..

configure.ac

@@ -111,6 +111,9 @@ PKG_CHECK_MODULES(
 [AC_MSG_ERROR([ch-placement pkg-config not found. ***])]
 )
 
+CXXFLAGS="${CXXFLAGS} ${SDSKV_SERVER_CFLAGS} ${SSG_CFLAGS} ${CH_PLACEMENT_CFLAGS}"
+LIBS="${LIBS} ${SDSKV_SERVER_LIBS} ${SSG_LIBS} ${CH_PLACEMENT_LIBS}"
+
 AC_CONFIG_FILES([
     Makefile
     src/Makefile

setup.sh

@@ -0,0 +1,10 @@
+#!/bin/bash 
+
+set -v
+
+source ~/spack/share/spack/setup-env.sh
+spack install mochi-sdskv^[email protected] mochi-ssg+mpi mochi-ch-placement pkg-config
+source <(spack module tcl loads mochi-sdskv mochi-ssg mochi-ch-placement autoconf automake libtool pkg-config)
+autoreconf --force --install && ./configure CXX=mpic++ && make
+
+mpirun -n 2 tests/trivial

src/sdsdkv-client.h

@@ -67,9 +67,9 @@ struct sdsdkv_client : public personality {
     {
         int rc = ssg_init();
         if (rc != SSG_SUCCESS) return SDSDKV_ERR_SERVICE;
-        //
-        rc = ssg_group_observe(m_mid, m_gid);
-        if (rc != SSG_SUCCESS) return SDSDKV_ERR_SERVICE;
+        // These are only for better diagnostics
+//      rc = ssg_group_observe(m_mid, m_gid);
+//      if (rc != SSG_SUCCESS) return SDSDKV_ERR_SERVICE;
         //
         return SDSDKV_SUCCESS;
     }
@@ -80,7 +80,8 @@ struct sdsdkv_client : public personality {
         hg_size_t gsize = ssg_get_group_size(m_gid);
         //
         for (decltype(gsize) i = 0; i < gsize; ++i) {
-            hg_addr_t server_addr = ssg_get_group_member_addr(m_gid, i);
+            ssg_member_id_t ssg_id = ssg_get_group_member_id_from_rank(m_gid, i);
+            hg_addr_t server_addr = ssg_get_group_member_addr(m_gid, ssg_id);
             if (server_addr == HG_ADDR_NULL) return SDSDKV_ERR_SERVICE;
             m_server_addrs.push_back(server_addr);
             //
@@ -165,20 +166,20 @@ struct sdsdkv_client : public personality {
     {
         // NOTE(skg): I know all this barrier stuff is ugly and complicated to
         // get right, but this helps fix an HG race condition...
-        m_mpi->barrier(m_mpi->get_world_comm());
+        //m_mpi->barrier(m_mpi->get_world_comm());
         //
         int rc = m_margo_init();
         if (rc != SDSDKV_SUCCESS) return rc;
         //
+        rc = m_ssg_init();
+        if (rc != SSG_SUCCESS) return SDSDKV_ERR_SERVICE;
+        //
+        if (rc != SDSDKV_SUCCESS) return rc;
         rc = xchange_gid();
         if (rc != SDSDKV_SUCCESS) return rc;
         //
-        m_mpi->barrier(m_mpi->get_world_comm());
-        //
-        m_mpi->barrier(m_mpi->get_world_comm());
-        //
-        rc = m_ssg_init();
-        if (rc != SDSDKV_SUCCESS) return rc;
+        rc = ssg_group_observe(m_mid, m_gid);
+        if (rc != SSG_SUCCESS) return SDSDKV_ERR_SERVICE;
         //
         rc = m_placement_init();
         if (rc != SDSDKV_SUCCESS) return rc;
@@ -200,8 +201,10 @@ struct sdsdkv_client : public personality {
                 if (rc != SDSKV_SUCCESS) return sdskv2irc(rc);
             }
         }
-        int rc = m_mpi->barrier(m_mpi->get_peronality_comm());
-        if (rc != SDSDKV_ERR_MPI) return rc;
+        //int rc = m_mpi->barrier(m_mpi->get_peronality_comm());
+        //if (rc != SDSDKV_ERR_MPI) return rc;
+        int rc = ssg_group_unobserve(m_gid);
+        if (rc != SSG_SUCCESS) return SDSDKV_ERR_SERVICE;
         //
         for (auto &t : m_ph_dbs) {
             sdskv_provider_handle_release(t.first);
@@ -210,10 +213,7 @@ struct sdsdkv_client : public personality {
             margo_addr_free(m_mid, sa);
         }
         sdskv_client_finalize(m_kvcl);
-        //
-        rc = ssg_group_unobserve(m_gid);
-        if (rc != SSG_SUCCESS) return SDSDKV_ERR_SERVICE;
-        rc = ssg_group_destroy(m_gid);
+
         if (rc != SSG_SUCCESS) return SDSDKV_ERR_SERVICE;
         rc = ssg_finalize();
         if (rc != SSG_SUCCESS) return SDSDKV_ERR_SERVICE;
@@ -261,7 +261,7 @@ struct sdsdkv_client : public personality {
             return(rc);
         }
         //
-        int num_addrs = 0;
+        int num_addrs = 1;
         ssg_group_id_deserialize(gid_bits, gid_size_ul, &num_addrs, &m_gid);
         //
         if (gid_bits) free(gid_bits);

src/sdsdkv-misci.h

@@ -26,7 +26,7 @@ sdskv2irc(int rc)
         case (SDSKV_SUCCESS):         return SDSDKV_SUCCESS;
         case (SDSKV_ERR_ALLOCATION):  return SDSDKV_ERR_OOR;
         case (SDSKV_ERR_INVALID_ARG): return SDSDKV_ERR_INVLD_ARG;
-        case (SDSKV_ERR_MERCURY):     return SDSDKV_ERR_SERVICE;
+//      case (SDSKV_ERR_MERCURY):     return SDSDKV_ERR_SERVICE;
         case (SDSKV_ERR_DB_CREATE):   return SDSDKV_ERR_DB_CREATE;
         case (SDSKV_ERR_DB_NAME):     return SDSDKV_ERR_DB_NAME;
         case (SDSKV_ERR_UNKNOWN_DB):  return SDSDKV_ERR_DB_UNKNOWN;
@@ -35,6 +35,13 @@ sdskv2irc(int rc)
         case (SDSKV_ERR_UNKNOWN_KEY): return SDSDKV_ERR_UNKNOWN_KEY;
         case (SDSKV_ERR_SIZE):        return SDSDKV_ERR_SIZE;
         case (SDSKV_ERR_ERASE):       return SDSDKV_ERR_ERASE;
+
+        case (SDSKV_ERR_PR_EXISTS):   return SDSDKV_ERR;
+        case (SDSKV_ERR_MIGRATION):   return SDSDKV_ERR;
+        case (SDSKV_OP_NOT_IMPL):     return SDSDKV_ERR;
+        case (SDSKV_ERR_COMP_FUNC):   return SDSDKV_ERR;
+        case (SDSKV_ERR_REMI):        return SDSDKV_ERR;
+        case (SDSKV_ERR_MAX):         return SDSDKV_ERR;
     }
     return SDSDKV_ERR_INVLD_ARG;
 }

src/sdsdkv-server.h

@@ -34,17 +34,18 @@ finalize_cb(void *cba)
 
 static void
 group_update_cb(
-    ssg_membership_update_t update,
-    void *cb_dat
+    void *cb_dat,
+    ssg_member_id_t member_id,
+    ssg_member_update_type_t update
 ) {
     int world_id = *(int *)cb_dat;
 
-    switch (update.type) {
+    switch (update) {
         case SSG_MEMBER_JOINED:
-            printf("%d SSG update: JOINED member %lu\n", world_id, update.member);
+            printf("%d SSG update: JOINED member %lu\n", world_id, member_id);
             break;
         case SSG_MEMBER_LEFT:
-            printf("%d SSG update: RM member %lu\n", world_id, update.member);
+            printf("%d SSG update: RM member %lu\n", world_id, member_id);
             break;
     }
 }
@@ -85,11 +86,14 @@ struct sdsdkv_server : public personality {
         if (rc != SSG_SUCCESS) return SDSDKV_ERR_SERVICE;
         //
         int world_id = m_mpi->get_world_id();
-        m_gid = ssg_group_create_mpi(
+        m_gid = ssg_group_create_mpi(m_mid,
                     m_config->group_name.c_str(),
                     m_mpi->get_peronality_comm(),
-                    &group_update_cb,
-                    &world_id
+                    //&group_update_cb,
+                    //&world_id
+                    NULL,
+                    NULL,
+                    NULL
                 );
         if (m_gid == SSG_GROUP_ID_INVALID) return SDSDKV_ERR_SERVICE;
         //
@@ -162,20 +166,20 @@ struct sdsdkv_server : public personality {
         int rc = m_margo_init();
         if (rc != SDSDKV_SUCCESS) return rc;
         //
-        m_mpi->barrier(m_mpi->get_world_comm());
+        //m_mpi->barrier(m_mpi->get_world_comm());
         //
         rc = m_ssg_init();
         if (rc != SDSDKV_SUCCESS) return rc;
         //
         rc = xchange_gid();
         if (rc != SDSDKV_SUCCESS) return rc;
         //
-        m_mpi->barrier(m_mpi->get_world_comm());
+        //m_mpi->barrier(m_mpi->get_world_comm());
         //
         rc = m_keyval_register_provider();
         if (rc != SDSDKV_SUCCESS) return rc;
         //
-        m_mpi->barrier(m_mpi->get_world_comm());
+        //m_mpi->barrier(m_mpi->get_world_comm());
         //
         rc = m_keyval_add_db();
         if (rc != SDSDKV_SUCCESS) return rc;
@@ -214,7 +218,7 @@ struct sdsdkv_server : public personality {
         char *gid_bits = nullptr;
         size_t gid_size = 0;
         //
-        ssg_group_id_serialize(m_gid, SSG_ALL_MEMBERS, &gid_bits, &gid_size);
+        ssg_group_id_serialize(m_gid, 1, &gid_bits, &gid_size);
         if (!gid_bits || gid_size == 0) {
             return SDSDKV_ERR_SERVICE;
         }

src/sdsdkv.h

@@ -55,6 +55,8 @@ enum {
     SDSDKV_ERR_PUT,
     /** Unknown key error. */
     SDSDKV_ERR_UNKNOWN_KEY,
+    /** Key exists error. */
+    SDSDKV_ERR_KEYEXISTS,
     /** Size error. */
     SDSDKV_ERR_SIZE,
     /** Erase error. */