Skip to content

Commit

Permalink
Make bdfid use 32 bit domain if possible
Browse files Browse the repository at this point in the history 
If the 32-bit domain is found in the kfd node properties for
a device, then it will be used when constructing the bdfid.
If it's not present, it will continue to use the 16 bit version.

Also, whether or not 32b or 16b are used for the domain, the
domain will now be placed in the upper 32b of the 64b bdfid.

* Fixed some unrelated doxygen issues

Change-Id: Icb5116daa1ab45ee305bdbe6cd5df5736dd3ffa3
  • Loading branch information
Chris Freehill authored and Chris Freehill committed Aug 27, 2019
1 parent 01e0800 commit 469af30
Show file tree
Hide file tree
Showing 7 changed files with 254 additions and 16 deletions.
21 changes: 17 additions & 4 deletions include/rocm_smi/rocm_smi.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@ typedef enum {
/**
* @brief Pre-set Profile Selections. These bitmasks can be AND'd with the
* ::rsmi_power_profile_status_t.available_profiles returned from
* ::rsmi_dev_power_profile_presets_get() to determine which power profiles
* ::rsmi_dev_power_profile_presets_get to determine which power profiles
* are supported by the system.
*/
typedef enum {
Expand Down Expand Up @@ -996,6 +996,19 @@ rsmi_dev_pci_bandwidth_get(uint32_t dv_ind, rsmi_pcie_bandwidth_t *bandwidth);
* (BDFID) associated with device @p dv_ind to the value pointed to by
* @p bdfid.
*
* The format of @p bdfid will be as follows:
*
* BDFID = ((DOMAIN & 0xffffffff) << 32) | ((BUS & 0xff) << 8) |
* ((DEVICE & 0x1f) <<3 ) | (FUNCTION & 0x7)
*
* | Name | Field |
* ---------- | ------- |
* | Domain | [64:32] |
* | Reserved | [31:16] |
* | Bus | [15: 8] |
* | Device | [ 7: 3] |
* | Function | [ 2: 0] |
*
* @param[in] dv_ind a device index
*
* @param[inout] bdfid a pointer to uint64_t to which the device bdfid value
Expand Down Expand Up @@ -1851,9 +1864,9 @@ rsmi_dev_firmware_version_get(uint32_t dv_ind, rsmi_fw_block_t block,
*
* @retval ::RSMI_STATUS_SUCCESS is returned upon successful call.
*
* ::RSMI_NOT_SUPPORTED will be returned if either ECC is not enabled for the
* specified block @p block, or if there is no kernel support for ECC for
* that block.
* ::RSMI_STATUS_NOT_SUPPORTED will be returned if either ECC is not enabled
* for the specified block @p block, or if there is no kernel support for ECC
* for that block.
*
*/
rsmi_status_t rsmi_dev_ecc_count_get(uint32_t dv_ind,
Expand Down
38 changes: 37 additions & 1 deletion include/rocm_smi/rocm_smi_device.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
#include <cstdint>
#include <vector>
#include <unordered_set>
#include <map>

#include "rocm_smi/rocm_smi_monitor.h"
#include "rocm_smi/rocm_smi_power_mon.h"
Expand All @@ -64,6 +65,38 @@ extern "C" {
namespace amd {
namespace smi {

enum DevKFDNodePropTypes {
kDevKFDNodePropCachesCnt,
kDevKFDNodePropIoLinksCnt,
kDevKFDNodePropCPUCoreIdBase,
kDevKFDNodePropSimdIdBase,
kDevKFDNodePropMaxWavePerSimd,
kDevKFDNodePropLdsSz,
kDevKFDNodePropGdsSz,
kDevKFDNodePropNumGWS,
kDevKFDNodePropWaveFrontSize,
kDevKFDNodePropArrCnt,
kDevKFDNodePropSimdArrPerEng,
kDevKFDNodePropCuPerSimdArr,
kDevKFDNodePropSimdPerCU,
kDevKFDNodePropMaxSlotsScratchCu,
kDevKFDNodePropVendorId,
kDevKFDNodePropDeviceId,
kDevKFDNodePropLocationId,
kDevKFDNodePropDrmRenderMinor,
kDevKFDNodePropHiveId,
kDevKFDNodePropNumSdmaEngines,
kDevKFDNodePropNumSdmaXgmiEngs,
kDevKFDNodePropMaxEngClkFComp,
kDevKFDNodePropLocMemSz,
kDevKFDNodePropFwVer,
kDevKFDNodePropCapability,
kDevKFDNodePropDbgProp,
kDevKFDNodePropSdmaFwVer,
kDevKFDNodePropMaxEngClkCComp,
kDevKFDNodePropDomain,
};

enum DevInfoTypes {
kDevPerfLevel,
kDevOverDriveLevel,
Expand Down Expand Up @@ -139,14 +172,16 @@ class Device {
int readDevInfo(DevInfoTypes type, std::vector<std::string> *retVec);
int writeDevInfo(DevInfoTypes type, uint64_t val);
int writeDevInfo(DevInfoTypes type, std::string val);
int populateKFDNodeProperties(bool force_update = false);
int getKFDNodeProperty(DevKFDNodePropTypes prop, uint64_t *val);

uint32_t index(void) const {return index_;}
void set_index(uint32_t index) {index_ = index;}
uint32_t drm_render_minor(void) const {return drm_render_minor_;}
void set_drm_render_minor(uint32_t minor) {drm_render_minor_ = minor;}
static rsmi_dev_perf_level perfLvlStrToEnum(std::string s);
uint64_t bdfid(void) const {return bdfid_;}
void set_bdfid(uint64_t val) {bdfid_ = val;}
uint64_t get_bdfid(void) const {return bdfid_;}
pthread_mutex_t *mutex(void) {return mutex_.ptr;}
evt::dev_evt_grp_set_t* supported_event_groups(void) {
return &supported_event_groups_;}
Expand All @@ -169,6 +204,7 @@ class Device {
uint64_t bdfid_;
std::unordered_set<rsmi_event_group_t,
evt::RSMIEventGrpHashFunction> supported_event_groups_;
std::map<std::string, uint64_t> kfdNodePropMap_;
};

} // namespace smi
Expand Down
6 changes: 6 additions & 0 deletions include/rocm_smi/rocm_smi_kfd.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@
#ifndef INCLUDE_ROCM_SMI_ROCM_SMI_KFD_H_
#define INCLUDE_ROCM_SMI_ROCM_SMI_KFD_H_

#include <string>
#include <vector>

#include "rocm_smi/rocm_smi.h"

namespace amd {
Expand All @@ -54,6 +57,9 @@ GetProcessInfo(rsmi_process_info_t *procs, uint32_t num_allocated,
int
GetProcessInfoForPID(uint32_t pid, rsmi_process_info_t *proc);

int
ReadKFDDeviceProperties(uint32_t dev_id, std::vector<std::string> *retVec);

} // namespace smi
} // namespace amd

Expand Down
30 changes: 29 additions & 1 deletion src/rocm_smi.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -621,7 +621,35 @@ rsmi_dev_pci_id_get(uint32_t dv_ind, uint64_t *bdfid) {

DEVICE_MUTEX

*bdfid = dev->get_bdfid();
*bdfid = dev->bdfid();

int32_t ret = dev->populateKFDNodeProperties();

if (ret) {
return errno_to_rsmi_status(errno);
}

uint64_t domain = 0;

ret = dev->getKFDNodeProperty(amd::smi::kDevKFDNodePropDomain, &domain);

if (ret == EINVAL) {
// "domain" is not found in properties file; just go with the 16 bit
// domain already found
return RSMI_STATUS_SUCCESS;
}

// Replace the 16 bit domain originally set like this:
// BDFID = ((<DOMAIN> & 0xffff) << 32) | ((<BUS> & 0xff) << 8) |
// ((device& 0x1f) <<3 ) | (function & 0x7)
// with this:
// BDFID = ((<DOMAIN> & 0xffffffff) << 32) | ((<BUS> & 0xff) << 8) |
// ((device& 0x1f) <<3 ) | (function & 0x7)

assert((domain & 0xFFFFFFFF00000000) == 0);
(*bdfid) &= 0xFFFF; // Clear out the old 16 bit domain
*bdfid |= (domain & 0xFFFFFFFF) << 32;

return RSMI_STATUS_SUCCESS;
CATCH
}
Expand Down
114 changes: 113 additions & 1 deletion src/rocm_smi_device.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
#include "rocm_smi/rocm_smi.h"
#include "rocm_smi/rocm_smi_exception.h"
#include "rocm_smi/rocm_smi_utils.h"
#include "rocm_smi/rocm_smi_kfd.h"

extern "C" {
#include "shared_mutex.h" // NOLINT
Expand Down Expand Up @@ -139,7 +140,71 @@ static const char *kDevFwVersionUvdFName = "fw_version/uvd_fw_version";
static const char *kDevFwVersionVceFName = "fw_version/vce_fw_version";
static const char *kDevFwVersionVcnFName = "fw_version/vcn_fw_version";


static const char *kDevKFDNodePropCachesCntSName = "caches_count";
static const char *kDevKFDNodePropIoLinksCntSName = "io_links_count";
static const char *kDevKFDNodePropCPUCoreIdBaseSName = "cpu_core_id_base";
static const char *kDevKFDNodePropSimdIdBaseSName = "simd_id_base";
static const char *kDevKFDNodePropMaxWavePerSimdSName = "max_waves_per_simd";
static const char *kDevKFDNodePropLdsSzSName = "lds_size_in_kb";
static const char *kDevKFDNodePropGdsSzSName = "gds_size_in_kb";
static const char *kDevKFDNodePropNumGWSSName = "num_gws";
static const char *kDevKFDNodePropWaveFrontSizeSName = "wave_front_size";
static const char *kDevKFDNodePropArrCntSName = "array_count";
static const char *kDevKFDNodePropSimdArrPerEngSName = "simd_arrays_per_engine";
static const char *kDevKFDNodePropCuPerSimdArrSName = "cu_per_simd_array";
static const char *kDevKFDNodePropSimdPerCUSName = "simd_per_cu";
static const char *kDevKFDNodePropMaxSlotsScratchCuSName =
"max_slots_scratch_cu";
static const char *kDevKFDNodePropVendorIdSName = "vendor_id";
static const char *kDevKFDNodePropDeviceIdSName = "device_id";
static const char *kDevKFDNodePropLocationIdSName = "location_id";
static const char *kDevKFDNodePropDrmRenderMinorSName = "drm_render_minor";
static const char *kDevKFDNodePropHiveIdSName = "hive_id";
static const char *kDevKFDNodePropNumSdmaEnginesSName = "num_sdma_engines";
static const char *kDevKFDNodePropNumSdmaXgmiEngsSName =
"num_sdma_xgmi_engines";
static const char *kDevKFDNodePropMaxEngClkFCompSName =
"max_engine_clk_fcompute";
static const char *kDevKFDNodePropLocMemSzSName = "local_mem_size";
static const char *kDevKFDNodePropFwVerSName = "fw_version";
static const char *kDevKFDNodePropCapabilitySName = "capability";
static const char *kDevKFDNodePropDbgPropSName = "debug_prop";
static const char *kDevKFDNodePropSdmaFwVerSName = "sdma_fw_version";
static const char *kDevKFDNodePropMaxEngClkCCompSName =
"max_engine_clk_ccompute";
static const char *kDevKFDNodePropDomainSName = "domain";

static const std::map<DevKFDNodePropTypes, const char *> kDevKFDPropNameMap = {
{kDevKFDNodePropCachesCnt, kDevKFDNodePropCachesCntSName},
{kDevKFDNodePropIoLinksCnt, kDevKFDNodePropIoLinksCntSName},
{kDevKFDNodePropCPUCoreIdBase, kDevKFDNodePropCPUCoreIdBaseSName},
{kDevKFDNodePropSimdIdBase, kDevKFDNodePropSimdIdBaseSName},
{kDevKFDNodePropMaxWavePerSimd, kDevKFDNodePropMaxWavePerSimdSName},
{kDevKFDNodePropLdsSz, kDevKFDNodePropLdsSzSName},
{kDevKFDNodePropGdsSz, kDevKFDNodePropGdsSzSName},
{kDevKFDNodePropNumGWS, kDevKFDNodePropNumGWSSName},
{kDevKFDNodePropWaveFrontSize, kDevKFDNodePropWaveFrontSizeSName},
{kDevKFDNodePropArrCnt, kDevKFDNodePropArrCntSName},
{kDevKFDNodePropSimdArrPerEng, kDevKFDNodePropSimdArrPerEngSName},
{kDevKFDNodePropCuPerSimdArr, kDevKFDNodePropCuPerSimdArrSName},
{kDevKFDNodePropSimdPerCU, kDevKFDNodePropSimdPerCUSName},
{kDevKFDNodePropMaxSlotsScratchCu, kDevKFDNodePropMaxSlotsScratchCuSName},
{kDevKFDNodePropVendorId, kDevKFDNodePropVendorIdSName},
{kDevKFDNodePropDeviceId, kDevKFDNodePropDeviceIdSName},
{kDevKFDNodePropLocationId, kDevKFDNodePropLocationIdSName},
{kDevKFDNodePropDrmRenderMinor, kDevKFDNodePropDrmRenderMinorSName},
{kDevKFDNodePropHiveId, kDevKFDNodePropHiveIdSName},
{kDevKFDNodePropNumSdmaEngines, kDevKFDNodePropNumSdmaEnginesSName},
{kDevKFDNodePropNumSdmaXgmiEngs, kDevKFDNodePropNumSdmaXgmiEngsSName},
{kDevKFDNodePropMaxEngClkFComp, kDevKFDNodePropMaxEngClkFCompSName},
{kDevKFDNodePropLocMemSz, kDevKFDNodePropLocMemSzSName},
{kDevKFDNodePropFwVer, kDevKFDNodePropFwVerSName},
{kDevKFDNodePropCapability, kDevKFDNodePropCapabilitySName},
{kDevKFDNodePropDbgProp, kDevKFDNodePropDbgPropSName},
{kDevKFDNodePropSdmaFwVer, kDevKFDNodePropSdmaFwVerSName},
{kDevKFDNodePropMaxEngClkCComp, kDevKFDNodePropMaxEngClkCCompSName},
{kDevKFDNodePropDomain, kDevKFDNodePropDomainSName},
};
static const std::map<DevInfoTypes, const char *> kDevAttribNameMap = {
{kDevPerfLevel, kDevPerfLevelFName},
{kDevOverDriveLevel, kDevOverDriveLevelFName},
Expand Down Expand Up @@ -520,6 +585,53 @@ int Device::readDevInfo(DevInfoTypes type, std::string *val) {
return 0;
}

int Device::populateKFDNodeProperties(bool force_update) {
int ret;

std::vector<std::string> propVec;

if (kfdNodePropMap_.size() > 0 && !force_update) {
return 0;
}

ret = ReadKFDDeviceProperties(index_, &propVec);

if (ret) {
return ret;
}

std::string key_str;
// std::string val_str;
uint64_t val_int; // Assume all properties are unsigned integers for now
std::istringstream fs;

for (uint32_t i = 0; i < propVec.size(); ++i) {
fs.str(propVec[i]);
fs >> key_str;
fs >> val_int;

kfdNodePropMap_[key_str] = val_int;

fs.str("");
fs.clear();
}

return 0;
}

int Device::getKFDNodeProperty(DevKFDNodePropTypes prop, uint64_t *val) {
assert(val != nullptr);
assert(kDevKFDPropNameMap.find(prop) != kDevKFDPropNameMap.end());

const char *prop_name = kDevKFDPropNameMap.at(prop);
if (kfdNodePropMap_.find(prop_name) == kfdNodePropMap_.end()) {
return EINVAL;
}

*val = kfdNodePropMap_.at(prop_name);
return 0;
}

#undef RET_IF_NONZERO
} // namespace smi
} // namespace amd
46 changes: 45 additions & 1 deletion src/rocm_smi_kfd.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,14 +62,58 @@ namespace amd {
namespace smi {

static const char *kKFDProcPathRoot = "/sys/class/kfd/kfd/proc";

static const char *kKFDNodesPathRoot = "/sys/class/kfd/kfd/topology/nodes";
// Sysfs file names
static const char *kKFDPasidFName = "pasid";

static bool is_number(const std::string &s) {
return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit);
}

int ReadKFDDeviceProperties(uint32_t dev_id,
std::vector<std::string> *retVec) {
std::string line;
int ret;
std::ifstream fs;
std::string properties_path = kKFDNodesPathRoot;
bool reg_file;

assert(retVec != nullptr);

properties_path += '/';
properties_path += std::to_string(dev_id);
properties_path += "/properties";

ret = isRegularFile(properties_path, &reg_file);

if (ret != 0) {
return ret;
}
if (!reg_file) {
return ENOENT;
}

fs.open(properties_path);

if (!fs.is_open()) {
return errno;
}

while (std::getline(fs, line)) {
retVec->push_back(line);
}

if (retVec->size() == 0) {
return 0;
}
// Remove any *trailing* empty (whitespace) lines
while (retVec->back().find_first_not_of(" \t\n\v\f\r") == std::string::npos) {
retVec->pop_back();
}
return 0;
}


int GetProcessInfo(rsmi_process_info_t *procs, uint32_t num_allocated,
uint32_t *num_procs_found) {
assert(num_procs_found != nullptr);
Expand Down
Loading

0 comments on commit 469af30

Please sign in to comment.