Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DAOS-17207 daos: upgrade to SPDK 24.09 #16013

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
+109 −30
Open

DAOS-17207 daos: upgrade to SPDK 24.09 #16013

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
08c9a40
DAOS-17207 daos: upgrade to SPDK 24.09
wangdi1 Mar 4, 2025
6cda205
DAOS-17207 build: fix style
wangdi1 Mar 4, 2025
d5bf1b4
DAOS-17027 bio: fix style
wangdi1 Mar 4, 2025
5a8186a
DAOS-17207 build: fix style issue
wangdi1 Mar 4, 2025
651a8cd
DAOS-17207 bio: update the patch
wangdi1 Mar 6, 2025
d878b6e
DAOS-17207 bio: update patch
wangdi1 Mar 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions site_scons/components/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,15 +347,13 @@ def define_components(reqs):
'--prefix=$SPDK_PREFIX',
'--disable-tests',
'--disable-unit-tests',
'--disable-apps',
'--without-vhost',
'--without-crypto',
'--without-pmdk',
'--without-rbd',
'--without-iscsi-initiator',
'--without-isal',
'--without-vtune',
'--with-shared',
'--without-nvme-cuse',
f'--target-arch={spdk_arch}'],
['make', f'CONFIG_ARCH={spdk_arch}'],
['make', 'install'],
Expand All @@ -364,9 +362,8 @@ def define_components(reqs):
['mkdir', '-p', '$SPDK_PREFIX/share/spdk'],
['cp', '-r', 'include', 'scripts', '$SPDK_PREFIX/share/spdk'],
['cp', 'build/examples/lsvmd', '$SPDK_PREFIX/bin/spdk_nvme_lsvmd'],
['cp', 'build/examples/nvme_manage', '$SPDK_PREFIX/bin/spdk_nvme_manage'],
['cp', 'build/examples/identify', '$SPDK_PREFIX/bin/spdk_nvme_identify'],
['cp', 'build/examples/perf', '$SPDK_PREFIX/bin/spdk_nvme_perf']],
['cp', 'build/examples/nvme_manage', '$SPDK_PREFIX/bin/spdk_nvme_manage']
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why remove perf and identify apps?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

perf and identify has been moved under app, since I remove --disable-apps, they should be included.

],
headers=['spdk/nvme.h'],
patch_rpath=['lib', 'bin'])

Expand Down
4 changes: 3 additions & 1 deletion site_scons/prereq_tools/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1457,11 +1457,13 @@ def _patch_rpaths(self):
if folder != 'bin' and not lib.endswith(".so"):
# Assume every file in bin can be patched
continue
if lib.endswith(".py"):
continue
full_lib = os.path.join(path, lib)
cmd = ['patchelf', '--set-rpath', ':'.join(rpath), full_lib]
res = RUNNER.run_commands([cmd])
if not res:
if lib == 'libspdk.so' and res.rc == 1:
if lib in ('libspdk.so', 'spdk_cli', 'spdk_rpc') and res.rc == 1:
print(f'Skipped patching {full_lib}')
else:
raise BuildFailure(f'Error running patchelf on {full_lib} ({res.rc})')
Expand Down
86 changes: 80 additions & 6 deletions src/bio/bio_xstream.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/**
* (C) Copyright 2018-2024 Intel Corporation.
* (C) Copyright 2025 Google LLC
* (C) Copyright 2025 Hewlett Packard Enterprise Development LP
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
Expand All @@ -21,6 +22,7 @@
#include <spdk/blob_bdev.h>
#include <spdk/blob.h>
#include <spdk/rpc.h>
#include <spdk/file.h>
#include <spdk/env_dpdk.h>
#include "bio_internal.h"
#include <daos_srv/smd.h>
Expand Down Expand Up @@ -154,6 +156,7 @@ bio_spdk_env_init(void)
/* Only print error and more severe to stderr. */
spdk_log_set_print_level(SPDK_LOG_ERROR);

opts.opts_size = sizeof(opts);
spdk_env_opts_init(&opts);
opts.name = "daos_engine";
opts.env_context = (char *)dpdk_cli_override_opts;
Expand Down Expand Up @@ -484,10 +487,29 @@ common_init_cb(void *arg, int rc)
cp_arg->cca_rc = daos_errno2der(-rc);
}

struct subsystem_init_arg {
struct common_cp_arg *cp_arg;
void *json_data;
size_t json_data_size;
};

static void
subsys_init_cb(int rc, void *arg)
{
struct subsystem_init_arg *init_arg = arg;

if (init_arg->json_data != NULL) {
free(init_arg->json_data);
init_arg->json_data = NULL;
}

if (rc)
D_ERROR("subsystem init failed: %d\n", rc);

init_arg->cp_arg->cca_rc = rc;
common_init_cb(arg, rc);

return;
}

static void
Expand Down Expand Up @@ -1554,6 +1576,57 @@ bio_xsctxt_free(struct bio_xs_context *ctxt)
D_FREE(ctxt);
}

static void
subsystem_init_cb(int rc, void *arg)
{
struct subsystem_init_arg *init_arg;

if (rc) {
subsys_init_cb(rc, arg);
return;
}

init_arg = arg;

/* Set RUNTIME state and load config again for RUNTIME methods */
spdk_rpc_set_state(SPDK_RPC_RUNTIME);
spdk_subsystem_load_config(init_arg->json_data, (ssize_t)init_arg->json_data_size,
subsys_init_cb, init_arg, true);
}

static void
load_config_cb(int rc, void *arg)
{
if (rc) {
subsys_init_cb(rc, arg);
return;
}

/* init subsystem */
spdk_subsystem_init(subsystem_init_cb, arg);
}

static int
bio_xsctxt_init_by_config(struct common_cp_arg *cp_arg)
{
struct subsystem_init_arg init_arg;
void *json_data;
size_t json_data_size;

json_data = spdk_posix_file_load_from_name(nvme_glb.bd_nvme_conf, &json_data_size);
if (json_data == NULL) {
D_ERROR("failed to load nvme conf %s\n", nvme_glb.bd_nvme_conf);
return -DER_NOMEM;
}

init_arg.cp_arg = cp_arg;
init_arg.json_data = json_data;
init_arg.json_data_size = json_data_size;
spdk_subsystem_load_config(json_data, (ssize_t)json_data_size, load_config_cb, &init_arg,
true);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure if I follow this, can't we call spdk_subsystem_init() directly?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, the RPC state has to be set to RUN_TIME, then you can run those aio_create cmd in the config jason file. So the first load_config() is to running those methods bdev_aio_create(), then set the RPC state as RUN_TIME explicitly, then do second load_config(). In the previous version(22.01), you can do that in a single API, but they remove that in the current version, which make the process a bit strange indeed.

"subsystems": [
{
"subsystem": "bdev",
"config": [
{
"params": {
"bdev_io_pool_size": 65536,
"bdev_io_cache_size": 256
},
"method": "bdev_set_options"
},
{
"params": {
"retry_count": 4,
"timeout_us": 0,
"nvme_adminq_poll_period_us": 100000,
"action_on_timeout": "none",
"nvme_ioq_poll_period_us": 0
},
"method": "bdev_nvme_set_options"
},
{
"params": {
"enable": false,
"period_us": 0
},
"method": "bdev_nvme_set_hotplug"
},
{
"params": {
"block_size": 4096,
"name": "AIO_1",
"filename": "/tmp/aio_file"
},
"method": "bdev_aio_create"
}
]
}
]
}

return 0;
}

int
bio_xsctxt_alloc(struct bio_xs_context **pctxt, int tgt_id, bool self_polling)
{
Expand Down Expand Up @@ -1617,13 +1690,14 @@ bio_xsctxt_alloc(struct bio_xs_context **pctxt, int tgt_id, bool self_polling)

/* Initialize all registered subsystems: bdev, vmd, copy. */
common_prep_arg(&cp_arg);
spdk_subsystem_init_from_json_config(nvme_glb.bd_nvme_conf,
SPDK_DEFAULT_RPC_ADDR,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will the same default RPC address be used with these changes?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, https://github.com/spdk/spdk/blob/d0bb354293fb373e21001c4fee100545d2d89744/lib/init/json_config.c#L622

subsys_init_cb, &cp_arg,
true);
rc = bio_xsctxt_init_by_config(&cp_arg);
if (rc != 0) {
D_ERROR("failed to load nvme conf %s\n", nvme_glb.bd_nvme_conf);
goto out;
}

rc = xs_poll_completion(ctxt, &cp_arg.cca_inflights, 0);
D_ASSERT(rc == 0);

if (cp_arg.cca_rc != 0) {
rc = cp_arg.cca_rc;
D_ERROR("failed to init bdevs, rc:%d\n", rc);
Expand All @@ -1648,7 +1722,7 @@ bio_xsctxt_alloc(struct bio_xs_context **pctxt, int tgt_id, bool self_polling)
if ((!nvme_glb.bd_rpc_srv_addr) || (strlen(nvme_glb.bd_rpc_srv_addr) == 0))
nvme_glb.bd_rpc_srv_addr = SPDK_DEFAULT_RPC_ADDR;

rc = spdk_rpc_initialize(nvme_glb.bd_rpc_srv_addr);
rc = spdk_rpc_initialize(nvme_glb.bd_rpc_srv_addr, NULL);
if (rc != 0) {
D_ERROR("failed to start SPDK JSON-RPC server at %s, "DF_RC"\n",
nvme_glb.bd_rpc_srv_addr, DP_RC(daos_errno2der(-rc)));
Expand Down
2 changes: 1 addition & 1 deletion src/control/lib/spdk/ctests/SConscript
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def scons():
libs += ['rte_mempool_ring', 'rte_bus_pci', 'nvme_control']

# Other libs
libs += ['numa', 'dl', 'isal', 'cmocka', 'pthread']
libs += ['numa', 'dl', 'isal', 'cmocka', 'pthread', 'ssl']

if GetOption('help'):
return
Expand Down
10 changes: 6 additions & 4 deletions src/control/lib/spdk/src/nvme_control.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
/**
* (C) Copyright 2018-2022 Intel Corporation.
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
*/
* (C) Copyright 2018-2022 Intel Corporation.
* (C) Copyright 2025 Google LLC
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
*/

#include <spdk/stdinc.h>
#include <spdk/nvme.h>
Expand Down Expand Up @@ -488,6 +489,7 @@ daos_spdk_init(int mem_sz, char *env_ctx, size_t nr_pcil, char **pcil)
struct spdk_env_opts opts = {};
int rc, i;

opts.opts_size = sizeof(opts);
spdk_env_opts_init(&opts);

if (mem_sz > 0)
Expand Down
19 changes: 10 additions & 9 deletions src/control/server/storage/bdev/backend_class_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//
// (C) Copyright 2021-2024 Intel Corporation.
// (C) Copyright 2025 Google LLC
//
// SPDX-License-Identifier: BSD-2-Clause-Patent
//
Expand Down Expand Up @@ -145,7 +146,7 @@ func TestBackend_writeJSONFile(t *testing.T) {
},
{
"params": {
"retry_count": 4,
"bdev_retry_count": 4,
"timeout_us": 0,
"nvme_adminq_poll_period_us": 100000,
"action_on_timeout": "none",
Expand Down Expand Up @@ -201,7 +202,7 @@ func TestBackend_writeJSONFile(t *testing.T) {
},
{
"params": {
"retry_count": 4,
"bdev_retry_count": 4,
"timeout_us": 0,
"nvme_adminq_poll_period_us": 100000,
"action_on_timeout": "none",
Expand Down Expand Up @@ -266,7 +267,7 @@ func TestBackend_writeJSONFile(t *testing.T) {
},
{
"params": {
"retry_count": 4,
"bdev_retry_count": 4,
"timeout_us": 0,
"nvme_adminq_poll_period_us": 100000,
"action_on_timeout": "none",
Expand Down Expand Up @@ -347,7 +348,7 @@ func TestBackend_writeJSONFile(t *testing.T) {
},
{
"params": {
"retry_count": 4,
"bdev_retry_count": 4,
"timeout_us": 0,
"nvme_adminq_poll_period_us": 100000,
"action_on_timeout": "none",
Expand Down Expand Up @@ -419,7 +420,7 @@ func TestBackend_writeJSONFile(t *testing.T) {
},
{
"params": {
"retry_count": 4,
"bdev_retry_count": 4,
"timeout_us": 0,
"nvme_adminq_poll_period_us": 100000,
"action_on_timeout": "none",
Expand Down Expand Up @@ -493,7 +494,7 @@ func TestBackend_writeJSONFile(t *testing.T) {
},
{
"params": {
"retry_count": 4,
"bdev_retry_count": 4,
"timeout_us": 0,
"nvme_adminq_poll_period_us": 100000,
"action_on_timeout": "none",
Expand Down Expand Up @@ -549,7 +550,7 @@ func TestBackend_writeJSONFile(t *testing.T) {
},
{
"params": {
"retry_count": 4,
"bdev_retry_count": 4,
"timeout_us": 0,
"nvme_adminq_poll_period_us": 100000,
"action_on_timeout": "none",
Expand Down Expand Up @@ -605,7 +606,7 @@ func TestBackend_writeJSONFile(t *testing.T) {
},
{
"params": {
"retry_count": 4,
"bdev_retry_count": 4,
"timeout_us": 0,
"nvme_adminq_poll_period_us": 100000,
"action_on_timeout": "none",
Expand Down Expand Up @@ -687,7 +688,7 @@ func TestBackend_writeJSONFile(t *testing.T) {
},
{
"params": {
"retry_count": 4,
"bdev_retry_count": 4,
"timeout_us": 0,
"nvme_adminq_poll_period_us": 100000,
"action_on_timeout": "none",
Expand Down
3 changes: 2 additions & 1 deletion src/control/server/storage/bdev/backend_json.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//
// (C) Copyright 2021-2024 Intel Corporation.
// (C) Copyright 2025 Google LLC
//
// SPDX-License-Identifier: BSD-2-Clause-Patent
//
Expand Down Expand Up @@ -45,7 +46,7 @@ func (_ SetOptionsParams) isSpdkSubsystemConfigParams() {}

// NvmeSetOptionsParams specifies details for a storage.ConfBdevNvmeSetOptions method.
type NvmeSetOptionsParams struct {
RetryCount uint32 `json:"retry_count"`
RetryCount uint32 `json:"bdev_retry_count"`
TimeoutUsec uint64 `json:"timeout_us"`
NvmeAdminqPollPeriodUsec uint32 `json:"nvme_adminq_poll_period_us"`
ActionOnTimeout string `json:"action_on_timeout"`
Expand Down
3 changes: 1 addition & 2 deletions utils/build.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ fused=v1.0.0
pmdk=2.1.0
isal=v2.30.0
isal_crypto=v2.23.0
spdk=v22.01.2
spdk=v24.09
ofi=v1.22.0
mercury=v2.4.0
protobufc=v1.3.3
Expand All @@ -26,7 +26,6 @@ protobufc=https://github.com/protobuf-c/protobuf-c.git
ucx=https://github.com/openucx/ucx.git

[patch_versions]
spdk=https://github.com/spdk/spdk/commit/b0aba3fcd5aceceea530a702922153bc75664978.diff,https://github.com/spdk/spdk/commit/445a4c808badbad3942696ecf16fa60e8129a747.diff
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

are we sure both of these commits are in v24.09?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, I checked.

mercury=https://raw.githubusercontent.com/daos-stack/mercury/f3dc286fb40ec1a3a38a2e17c45497bc2aa6290d/na_ucx.patch,https://raw.githubusercontent.com/daos-stack/mercury/48df263212604336b2dbe0430dcab4482eb43437/na_ucx_ep_flush.patch
pmdk=https://github.com/pmem/pmdk/commit/2abe15ac0b4eed894b6768cd82a3b0a7c4336284.diff
argobots=https://github.com/pmodels/argobots/pull/397/commits/411e5b344642ebc82190fd8b125db512e5b449d1.diff,https://github.com/pmodels/argobots/commit/bb0c908abfac4bfe37852eee621930634183c6aa.diff
1 change: 1 addition & 0 deletions utils/scripts/install-el8.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ dnf --nodocs install \
systemd \
valgrind-devel \
which \
ncurses-devel \
yasm

# ipmctl is only available on x86_64
Expand Down
1 change: 1 addition & 0 deletions utils/scripts/install-el9.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,4 +64,5 @@ dnf --nodocs install \
sudo \
valgrind-devel \
which \
ncurses-devel \
yasm
1 change: 1 addition & 0 deletions utils/scripts/install-ubuntu.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ apt-get install \
python3-venv \
uuid-dev \
valgrind \
libncurses-dev \
yasm

# ipmctl is only available on x86_64
Expand Down
Loading