From 469af303d6d72c52112e43f83c35dda99b46246c Mon Sep 17 00:00:00 2001 From: Chris Freehill Date: Sun, 25 Aug 2019 19:31:46 -0500 Subject: [PATCH] Make bdfid use 32 bit domain if possible If the 32-bit domain is found in the kfd node properties for a device, then it will be used when constructing the bdfid. If it's not present, it will continue to use the 16 bit version. Also, whether or not 32b or 16b are used for the domain, the domain will now be placed in the upper 32b of the 64b bdfid. * Fixed some unrelated doxygen issues Change-Id: Icb5116daa1ab45ee305bdbe6cd5df5736dd3ffa3 --- include/rocm_smi/rocm_smi.h | 21 +++++- include/rocm_smi/rocm_smi_device.h | 38 +++++++++- include/rocm_smi/rocm_smi_kfd.h | 6 ++ src/rocm_smi.cc | 30 +++++++- src/rocm_smi_device.cc | 114 ++++++++++++++++++++++++++++- src/rocm_smi_kfd.cc | 46 +++++++++++- src/rocm_smi_main.cc | 15 ++-- 7 files changed, 254 insertions(+), 16 deletions(-) diff --git a/include/rocm_smi/rocm_smi.h b/include/rocm_smi/rocm_smi.h index 28a5f52c..a3e9d659 100755 --- a/include/rocm_smi/rocm_smi.h +++ b/include/rocm_smi/rocm_smi.h @@ -328,7 +328,7 @@ typedef enum { /** * @brief Pre-set Profile Selections. These bitmasks can be AND'd with the * ::rsmi_power_profile_status_t.available_profiles returned from - * ::rsmi_dev_power_profile_presets_get() to determine which power profiles + * ::rsmi_dev_power_profile_presets_get to determine which power profiles * are supported by the system. */ typedef enum { @@ -996,6 +996,19 @@ rsmi_dev_pci_bandwidth_get(uint32_t dv_ind, rsmi_pcie_bandwidth_t *bandwidth); * (BDFID) associated with device @p dv_ind to the value pointed to by * @p bdfid. * + * The format of @p bdfid will be as follows: + * + * BDFID = ((DOMAIN & 0xffffffff) << 32) | ((BUS & 0xff) << 8) | + * ((DEVICE & 0x1f) <<3 ) | (FUNCTION & 0x7) + * + * | Name | Field | + * ---------- | ------- | + * | Domain | [64:32] | + * | Reserved | [31:16] | + * | Bus | [15: 8] | + * | Device | [ 7: 3] | + * | Function | [ 2: 0] | + * * @param[in] dv_ind a device index * * @param[inout] bdfid a pointer to uint64_t to which the device bdfid value @@ -1851,9 +1864,9 @@ rsmi_dev_firmware_version_get(uint32_t dv_ind, rsmi_fw_block_t block, * * @retval ::RSMI_STATUS_SUCCESS is returned upon successful call. * - * ::RSMI_NOT_SUPPORTED will be returned if either ECC is not enabled for the - * specified block @p block, or if there is no kernel support for ECC for - * that block. + * ::RSMI_STATUS_NOT_SUPPORTED will be returned if either ECC is not enabled + * for the specified block @p block, or if there is no kernel support for ECC + * for that block. * */ rsmi_status_t rsmi_dev_ecc_count_get(uint32_t dv_ind, diff --git a/include/rocm_smi/rocm_smi_device.h b/include/rocm_smi/rocm_smi_device.h index 73ec27de..6803cfe3 100755 --- a/include/rocm_smi/rocm_smi_device.h +++ b/include/rocm_smi/rocm_smi_device.h @@ -51,6 +51,7 @@ #include #include #include +#include #include "rocm_smi/rocm_smi_monitor.h" #include "rocm_smi/rocm_smi_power_mon.h" @@ -64,6 +65,38 @@ extern "C" { namespace amd { namespace smi { +enum DevKFDNodePropTypes { + kDevKFDNodePropCachesCnt, + kDevKFDNodePropIoLinksCnt, + kDevKFDNodePropCPUCoreIdBase, + kDevKFDNodePropSimdIdBase, + kDevKFDNodePropMaxWavePerSimd, + kDevKFDNodePropLdsSz, + kDevKFDNodePropGdsSz, + kDevKFDNodePropNumGWS, + kDevKFDNodePropWaveFrontSize, + kDevKFDNodePropArrCnt, + kDevKFDNodePropSimdArrPerEng, + kDevKFDNodePropCuPerSimdArr, + kDevKFDNodePropSimdPerCU, + kDevKFDNodePropMaxSlotsScratchCu, + kDevKFDNodePropVendorId, + kDevKFDNodePropDeviceId, + kDevKFDNodePropLocationId, + kDevKFDNodePropDrmRenderMinor, + kDevKFDNodePropHiveId, + kDevKFDNodePropNumSdmaEngines, + kDevKFDNodePropNumSdmaXgmiEngs, + kDevKFDNodePropMaxEngClkFComp, + kDevKFDNodePropLocMemSz, + kDevKFDNodePropFwVer, + kDevKFDNodePropCapability, + kDevKFDNodePropDbgProp, + kDevKFDNodePropSdmaFwVer, + kDevKFDNodePropMaxEngClkCComp, + kDevKFDNodePropDomain, +}; + enum DevInfoTypes { kDevPerfLevel, kDevOverDriveLevel, @@ -139,6 +172,9 @@ class Device { int readDevInfo(DevInfoTypes type, std::vector *retVec); int writeDevInfo(DevInfoTypes type, uint64_t val); int writeDevInfo(DevInfoTypes type, std::string val); + int populateKFDNodeProperties(bool force_update = false); + int getKFDNodeProperty(DevKFDNodePropTypes prop, uint64_t *val); + uint32_t index(void) const {return index_;} void set_index(uint32_t index) {index_ = index;} uint32_t drm_render_minor(void) const {return drm_render_minor_;} @@ -146,7 +182,6 @@ class Device { static rsmi_dev_perf_level perfLvlStrToEnum(std::string s); uint64_t bdfid(void) const {return bdfid_;} void set_bdfid(uint64_t val) {bdfid_ = val;} - uint64_t get_bdfid(void) const {return bdfid_;} pthread_mutex_t *mutex(void) {return mutex_.ptr;} evt::dev_evt_grp_set_t* supported_event_groups(void) { return &supported_event_groups_;} @@ -169,6 +204,7 @@ class Device { uint64_t bdfid_; std::unordered_set supported_event_groups_; + std::map kfdNodePropMap_; }; } // namespace smi diff --git a/include/rocm_smi/rocm_smi_kfd.h b/include/rocm_smi/rocm_smi_kfd.h index f9b6440c..0ae429d5 100755 --- a/include/rocm_smi/rocm_smi_kfd.h +++ b/include/rocm_smi/rocm_smi_kfd.h @@ -43,6 +43,9 @@ #ifndef INCLUDE_ROCM_SMI_ROCM_SMI_KFD_H_ #define INCLUDE_ROCM_SMI_ROCM_SMI_KFD_H_ +#include +#include + #include "rocm_smi/rocm_smi.h" namespace amd { @@ -54,6 +57,9 @@ GetProcessInfo(rsmi_process_info_t *procs, uint32_t num_allocated, int GetProcessInfoForPID(uint32_t pid, rsmi_process_info_t *proc); +int +ReadKFDDeviceProperties(uint32_t dev_id, std::vector *retVec); + } // namespace smi } // namespace amd diff --git a/src/rocm_smi.cc b/src/rocm_smi.cc index 742379d2..d2ac0f9a 100755 --- a/src/rocm_smi.cc +++ b/src/rocm_smi.cc @@ -621,7 +621,35 @@ rsmi_dev_pci_id_get(uint32_t dv_ind, uint64_t *bdfid) { DEVICE_MUTEX - *bdfid = dev->get_bdfid(); + *bdfid = dev->bdfid(); + + int32_t ret = dev->populateKFDNodeProperties(); + + if (ret) { + return errno_to_rsmi_status(errno); + } + + uint64_t domain = 0; + + ret = dev->getKFDNodeProperty(amd::smi::kDevKFDNodePropDomain, &domain); + + if (ret == EINVAL) { + // "domain" is not found in properties file; just go with the 16 bit + // domain already found + return RSMI_STATUS_SUCCESS; + } + + // Replace the 16 bit domain originally set like this: + // BDFID = (( & 0xffff) << 32) | (( & 0xff) << 8) | + // ((device& 0x1f) <<3 ) | (function & 0x7) + // with this: + // BDFID = (( & 0xffffffff) << 32) | (( & 0xff) << 8) | + // ((device& 0x1f) <<3 ) | (function & 0x7) + + assert((domain & 0xFFFFFFFF00000000) == 0); + (*bdfid) &= 0xFFFF; // Clear out the old 16 bit domain + *bdfid |= (domain & 0xFFFFFFFF) << 32; + return RSMI_STATUS_SUCCESS; CATCH } diff --git a/src/rocm_smi_device.cc b/src/rocm_smi_device.cc index dc6c0a5b..1caae505 100755 --- a/src/rocm_smi_device.cc +++ b/src/rocm_smi_device.cc @@ -61,6 +61,7 @@ #include "rocm_smi/rocm_smi.h" #include "rocm_smi/rocm_smi_exception.h" #include "rocm_smi/rocm_smi_utils.h" +#include "rocm_smi/rocm_smi_kfd.h" extern "C" { #include "shared_mutex.h" // NOLINT @@ -139,7 +140,71 @@ static const char *kDevFwVersionUvdFName = "fw_version/uvd_fw_version"; static const char *kDevFwVersionVceFName = "fw_version/vce_fw_version"; static const char *kDevFwVersionVcnFName = "fw_version/vcn_fw_version"; - +static const char *kDevKFDNodePropCachesCntSName = "caches_count"; +static const char *kDevKFDNodePropIoLinksCntSName = "io_links_count"; +static const char *kDevKFDNodePropCPUCoreIdBaseSName = "cpu_core_id_base"; +static const char *kDevKFDNodePropSimdIdBaseSName = "simd_id_base"; +static const char *kDevKFDNodePropMaxWavePerSimdSName = "max_waves_per_simd"; +static const char *kDevKFDNodePropLdsSzSName = "lds_size_in_kb"; +static const char *kDevKFDNodePropGdsSzSName = "gds_size_in_kb"; +static const char *kDevKFDNodePropNumGWSSName = "num_gws"; +static const char *kDevKFDNodePropWaveFrontSizeSName = "wave_front_size"; +static const char *kDevKFDNodePropArrCntSName = "array_count"; +static const char *kDevKFDNodePropSimdArrPerEngSName = "simd_arrays_per_engine"; +static const char *kDevKFDNodePropCuPerSimdArrSName = "cu_per_simd_array"; +static const char *kDevKFDNodePropSimdPerCUSName = "simd_per_cu"; +static const char *kDevKFDNodePropMaxSlotsScratchCuSName = + "max_slots_scratch_cu"; +static const char *kDevKFDNodePropVendorIdSName = "vendor_id"; +static const char *kDevKFDNodePropDeviceIdSName = "device_id"; +static const char *kDevKFDNodePropLocationIdSName = "location_id"; +static const char *kDevKFDNodePropDrmRenderMinorSName = "drm_render_minor"; +static const char *kDevKFDNodePropHiveIdSName = "hive_id"; +static const char *kDevKFDNodePropNumSdmaEnginesSName = "num_sdma_engines"; +static const char *kDevKFDNodePropNumSdmaXgmiEngsSName = + "num_sdma_xgmi_engines"; +static const char *kDevKFDNodePropMaxEngClkFCompSName = + "max_engine_clk_fcompute"; +static const char *kDevKFDNodePropLocMemSzSName = "local_mem_size"; +static const char *kDevKFDNodePropFwVerSName = "fw_version"; +static const char *kDevKFDNodePropCapabilitySName = "capability"; +static const char *kDevKFDNodePropDbgPropSName = "debug_prop"; +static const char *kDevKFDNodePropSdmaFwVerSName = "sdma_fw_version"; +static const char *kDevKFDNodePropMaxEngClkCCompSName = + "max_engine_clk_ccompute"; +static const char *kDevKFDNodePropDomainSName = "domain"; + +static const std::map kDevKFDPropNameMap = { + {kDevKFDNodePropCachesCnt, kDevKFDNodePropCachesCntSName}, + {kDevKFDNodePropIoLinksCnt, kDevKFDNodePropIoLinksCntSName}, + {kDevKFDNodePropCPUCoreIdBase, kDevKFDNodePropCPUCoreIdBaseSName}, + {kDevKFDNodePropSimdIdBase, kDevKFDNodePropSimdIdBaseSName}, + {kDevKFDNodePropMaxWavePerSimd, kDevKFDNodePropMaxWavePerSimdSName}, + {kDevKFDNodePropLdsSz, kDevKFDNodePropLdsSzSName}, + {kDevKFDNodePropGdsSz, kDevKFDNodePropGdsSzSName}, + {kDevKFDNodePropNumGWS, kDevKFDNodePropNumGWSSName}, + {kDevKFDNodePropWaveFrontSize, kDevKFDNodePropWaveFrontSizeSName}, + {kDevKFDNodePropArrCnt, kDevKFDNodePropArrCntSName}, + {kDevKFDNodePropSimdArrPerEng, kDevKFDNodePropSimdArrPerEngSName}, + {kDevKFDNodePropCuPerSimdArr, kDevKFDNodePropCuPerSimdArrSName}, + {kDevKFDNodePropSimdPerCU, kDevKFDNodePropSimdPerCUSName}, + {kDevKFDNodePropMaxSlotsScratchCu, kDevKFDNodePropMaxSlotsScratchCuSName}, + {kDevKFDNodePropVendorId, kDevKFDNodePropVendorIdSName}, + {kDevKFDNodePropDeviceId, kDevKFDNodePropDeviceIdSName}, + {kDevKFDNodePropLocationId, kDevKFDNodePropLocationIdSName}, + {kDevKFDNodePropDrmRenderMinor, kDevKFDNodePropDrmRenderMinorSName}, + {kDevKFDNodePropHiveId, kDevKFDNodePropHiveIdSName}, + {kDevKFDNodePropNumSdmaEngines, kDevKFDNodePropNumSdmaEnginesSName}, + {kDevKFDNodePropNumSdmaXgmiEngs, kDevKFDNodePropNumSdmaXgmiEngsSName}, + {kDevKFDNodePropMaxEngClkFComp, kDevKFDNodePropMaxEngClkFCompSName}, + {kDevKFDNodePropLocMemSz, kDevKFDNodePropLocMemSzSName}, + {kDevKFDNodePropFwVer, kDevKFDNodePropFwVerSName}, + {kDevKFDNodePropCapability, kDevKFDNodePropCapabilitySName}, + {kDevKFDNodePropDbgProp, kDevKFDNodePropDbgPropSName}, + {kDevKFDNodePropSdmaFwVer, kDevKFDNodePropSdmaFwVerSName}, + {kDevKFDNodePropMaxEngClkCComp, kDevKFDNodePropMaxEngClkCCompSName}, + {kDevKFDNodePropDomain, kDevKFDNodePropDomainSName}, +}; static const std::map kDevAttribNameMap = { {kDevPerfLevel, kDevPerfLevelFName}, {kDevOverDriveLevel, kDevOverDriveLevelFName}, @@ -520,6 +585,53 @@ int Device::readDevInfo(DevInfoTypes type, std::string *val) { return 0; } +int Device::populateKFDNodeProperties(bool force_update) { + int ret; + + std::vector propVec; + + if (kfdNodePropMap_.size() > 0 && !force_update) { + return 0; + } + + ret = ReadKFDDeviceProperties(index_, &propVec); + + if (ret) { + return ret; + } + + std::string key_str; + // std::string val_str; + uint64_t val_int; // Assume all properties are unsigned integers for now + std::istringstream fs; + + for (uint32_t i = 0; i < propVec.size(); ++i) { + fs.str(propVec[i]); + fs >> key_str; + fs >> val_int; + + kfdNodePropMap_[key_str] = val_int; + + fs.str(""); + fs.clear(); + } + + return 0; +} + +int Device::getKFDNodeProperty(DevKFDNodePropTypes prop, uint64_t *val) { + assert(val != nullptr); + assert(kDevKFDPropNameMap.find(prop) != kDevKFDPropNameMap.end()); + + const char *prop_name = kDevKFDPropNameMap.at(prop); + if (kfdNodePropMap_.find(prop_name) == kfdNodePropMap_.end()) { + return EINVAL; + } + + *val = kfdNodePropMap_.at(prop_name); + return 0; +} + #undef RET_IF_NONZERO } // namespace smi } // namespace amd diff --git a/src/rocm_smi_kfd.cc b/src/rocm_smi_kfd.cc index a118b86f..32149afe 100755 --- a/src/rocm_smi_kfd.cc +++ b/src/rocm_smi_kfd.cc @@ -62,7 +62,7 @@ namespace amd { namespace smi { static const char *kKFDProcPathRoot = "/sys/class/kfd/kfd/proc"; - +static const char *kKFDNodesPathRoot = "/sys/class/kfd/kfd/topology/nodes"; // Sysfs file names static const char *kKFDPasidFName = "pasid"; @@ -70,6 +70,50 @@ static bool is_number(const std::string &s) { return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit); } +int ReadKFDDeviceProperties(uint32_t dev_id, + std::vector *retVec) { + std::string line; + int ret; + std::ifstream fs; + std::string properties_path = kKFDNodesPathRoot; + bool reg_file; + + assert(retVec != nullptr); + + properties_path += '/'; + properties_path += std::to_string(dev_id); + properties_path += "/properties"; + + ret = isRegularFile(properties_path, ®_file); + + if (ret != 0) { + return ret; + } + if (!reg_file) { + return ENOENT; + } + + fs.open(properties_path); + + if (!fs.is_open()) { + return errno; + } + + while (std::getline(fs, line)) { + retVec->push_back(line); + } + + if (retVec->size() == 0) { + return 0; + } + // Remove any *trailing* empty (whitespace) lines + while (retVec->back().find_first_not_of(" \t\n\v\f\r") == std::string::npos) { + retVec->pop_back(); + } + return 0; +} + + int GetProcessInfo(rsmi_process_info_t *procs, uint32_t num_allocated, uint32_t *num_procs_found) { assert(num_procs_found != nullptr); diff --git a/src/rocm_smi_main.cc b/src/rocm_smi_main.cc index f5d3e63b..08d35dda 100755 --- a/src/rocm_smi_main.cc +++ b/src/rocm_smi_main.cc @@ -73,7 +73,6 @@ static const char *kAMDMonitorTypes[] = {"radeon", "amdgpu", ""}; namespace amd { namespace smi { - static uint32_t GetDeviceIndex(const std::string s) { std::string t = s; size_t tmp = t.find_last_not_of("0123456789"); @@ -123,12 +122,13 @@ static int SameDevice(const std::string fileA, const std::string fileB) { // XXXX:XX:XX.X, // domain:bus:device.function // -// where X is a hex integer (lower case is expected) -static bool is_bdfid_path_str(const std::string in_name, uint64_t *bdfid) { +// where X is a hex integer (lower case is expected). If so, write the value +// to bdfid +static bool bdfid_from_path(const std::string in_name, uint64_t *bdfid) { char *p = nullptr; char *name_start; char name[13] = {'\0'}; - uint32_t tmp; + uint64_t tmp; assert(bdfid != nullptr); @@ -139,7 +139,7 @@ static bool is_bdfid_path_str(const std::string in_name, uint64_t *bdfid) { tmp = in_name.copy(name, 12); assert(tmp == 12); - // BDFID = (( & 0xffff) << 13) | (( & 0x1f) << 8) | + // BDFID = (( & 0xffff) << 32) | (( & 0xff) << 8) | // ((device& 0x1f) <<3 ) | (function & 0x7) *bdfid = 0; name_start = name; @@ -150,7 +150,7 @@ static bool is_bdfid_path_str(const std::string in_name, uint64_t *bdfid) { if (*p != ':' || p - name_start != 4) { return false; } - *bdfid |= tmp << 13; + *bdfid |= tmp << 32; // Match this: xxxx:XX:xx.x p++; // Skip past ':' @@ -205,7 +205,7 @@ static uint32_t ConstructBDFID(std::string path, uint64_t *bdfid) { slash_i = tpath_str.find_last_of('/', end_i); tmp = tpath_str.substr(slash_i + 1, end_i - slash_i); - if (is_bdfid_path_str(tmp, bdfid)) { + if (bdfid_from_path(tmp, bdfid)) { return 0; } end_i = slash_i - 1; @@ -335,7 +335,6 @@ RocmSMI::AddToDeviceList(std::string dev_name) { uint32_t d_index = GetDeviceIndex(d_name); dev->set_drm_render_minor(GetDrmRenderMinor(dev_path)); dev->set_index(d_index); - GetSupportedEventGroups(d_index, dev->supported_event_groups()); devices_.push_back(dev);