Skip to content

Commit

Permalink
Merge pull request #95 from AxisCommunications/raii-inference
Browse files Browse the repository at this point in the history 
Follow RAII in Inference class
  • Loading branch information
@killenheladagen
killenheladagen authored Sep 9, 2024
2 parents 3a061b0 + 9a309de commit 5b8c56a
Show file tree
Hide file tree
Showing 4 changed files with 63 additions and 84 deletions.
11 changes: 2 additions & 9 deletions src/acap_runtime.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,15 +125,8 @@ int RunServer(const string& address,
}
builder.RegisterService(&capture);

// Register inference service
Inference inference;
if (chipId > 0) {
if (!inference.Init(_verbose, chipId, models, &capture)) {
syslog(LOG_ERR, "Could not Init Inference Service");
return EXIT_FAILURE;
}
builder.RegisterService(&inference);
}
Inference inference{_verbose, chipId, models, &capture};
builder.RegisterService(&inference);

// Start server
unique_ptr<Server> server(builder.BuildAndStart());
Expand Down
70 changes: 34 additions & 36 deletions src/inference.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,40 +54,14 @@ const char* const LAYOUTS[] = {"LAROD_TENSOR_LAYOUT_INVALID",
"LAROD_TENSOR_LAYOUT_NCHW",
"LAROD_TENSOR_LAYOUT_420SP"};

Inference::Inference() : _conn(nullptr), _chipId(LAROD_CHIP_INVALID), _verbose(false) {}

Inference::~Inference() {
if (nullptr != _conn) {
// Delete models
TRACELOG << "Deleting loaded models:" << endl;
larodError* error = nullptr;
for (auto& [model_name, model] : _models) {
TRACELOG << "- " << model_name << endl;
if (!larodDeleteModel(_conn, model, &error)) {
PrintError("Failed to delete model", error);
larodClearError(&error);
}
}

// Disconnect from larod service
TRACELOG << "Disconnecting from larod" << endl;
if (!larodDisconnect(&_conn, &error)) {
PrintError("Failed to disconnect", error);
larodClearError(&error);
}
}

for (auto& [model_name, model] : _models) {
larodDestroyModel(&model);
}
}

// Initialize inference
bool Inference::Init(const bool verbose,
Inference::Inference(const bool verbose,
const uint64_t chipId,
const vector<string>& models,
Capture* captureService) {
_verbose = verbose;
Capture* captureService)
: _verbose(verbose) {
if (chipId <= 0)
return;

larodError* error = nullptr;

_captureService = captureService;
Expand All @@ -96,14 +70,14 @@ bool Inference::Init(const bool verbose,

if (pthread_mutex_init(&_mtx, NULL) != 0) {
ERRORLOG << "Init mutex FAILED" << endl;
return false;
throw runtime_error("Could not Init Inference Service");
}

// Connect to larod service
if (!larodConnect(&_conn, &error)) {
PrintError("Connecting to larod FAILED", error);
larodClearError(&error);
return false;
throw runtime_error("Could not Init Inference Service");
}

// List available chip id:s
Expand All @@ -128,11 +102,35 @@ bool Inference::Init(const bool verbose,
_models.clear();
for (auto model : models) {
if (!LoadModel(*_conn, model.c_str(), _chipId, LAROD_ACCESS_PRIVATE)) {
return false;
throw runtime_error("Could not Init Inference Service");
}
}
}

return true;
Inference::~Inference() {
if (nullptr != _conn) {
// Delete models
TRACELOG << "Deleting loaded models:" << endl;
larodError* error = nullptr;
for (auto& [model_name, model] : _models) {
TRACELOG << "- " << model_name << endl;
if (!larodDeleteModel(_conn, model, &error)) {
PrintError("Failed to delete model", error);
larodClearError(&error);
}
}

// Disconnect from larod service
TRACELOG << "Disconnecting from larod" << endl;
if (!larodDisconnect(&_conn, &error)) {
PrintError("Failed to disconnect", error);
larodClearError(&error);
}
}

for (auto& [model_name, model] : _models) {
larodDestroyModel(&model);
}
}

// Run inference on a single image
Expand Down
8 changes: 4 additions & 4 deletions src/inference.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,12 @@ class Inference : public tensorflow::serving::PredictionService::Service {
using Status = grpc::Status;
using TensorProto = tensorflow::TensorProto;

Inference();
~Inference();
bool Init(const bool verbose,
Inference(const bool verbose,
const uint64_t chipId,
const std::vector<std::string>& models,
Capture* captureService);
~Inference();

Status Predict(ServerContext* context,
const PredictRequest* request,
PredictResponse* response) override;
Expand Down Expand Up @@ -72,7 +72,7 @@ class Inference : public tensorflow::serving::PredictionService::Service {
larodError*& error);

bool _verbose;
larodConnection* _conn;
larodConnection* _conn = nullptr;
larodChip _chipId;
std::map<std::string, larodModel*> _models;
larodModel* _ppModel;
Expand Down
58 changes: 23 additions & 35 deletions test/inference_unittest.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -339,25 +339,27 @@ TEST(InferenceUnittest, InitCpu) {
}

const vector<string> models = {cpuModel1};
Inference inference;
ASSERT_TRUE(inference.Init(verbose, cpuChipId, models, &capture));
Inference inference{verbose, cpuChipId, models, &capture};
SUCCEED();
}

TEST(InferenceUnittest, Init_Fail) {
const bool verbose = get_verbose_status();
const vector<string> models = {cpuModel1, "invalid"};

Inference inference;
ASSERT_FALSE(inference.Init(verbose, cpuChipId, models, &capture));
try {
Inference inference{verbose, cpuChipId, models, &capture};
FAIL();
} catch (const runtime_error&) {
SUCCEED();
}
}

TEST(InferenceUnittest, PredictCpuModel1Preload) {
const bool verbose = get_verbose_status();
const vector<string> models = {cpuModel1};
shm_unlink(sharedFile);

Inference inference;
ASSERT_TRUE(inference.Init(verbose, cpuChipId, models, &capture));
Inference inference{verbose, cpuChipId, models, &capture};
PredictModel1(inference, cpuModel1, imageFile1, 0.87890601, 0.58203125, true);
PredictModel1(inference, cpuModel1, imageFile1, 0.87890601, 0.58203125, true);
PredictModel1(inference, cpuModel1, imageFile1, 0.87890601, 0.58203125, true);
Expand All @@ -378,8 +380,7 @@ TEST(InferenceUnittest, PredictCpuModel1) {
const vector<string> models = {};
shm_unlink(sharedFile);

Inference inference;
ASSERT_TRUE(inference.Init(verbose, cpuChipId, models, &capture));
Inference inference{verbose, cpuChipId, models, &capture};
PredictModel1(inference, cpuModel1, imageFile1, 0.87890601, 0.58203125, false);
PredictModel1(inference, cpuModel1, imageFile1, 0.87890601, 0.58203125, false);
PredictModel1(inference, cpuModel1, imageFile1, 0.87890601, 0.58203125, true);
Expand All @@ -400,8 +401,7 @@ TEST(InferenceUnittest, PredictCpuModel2) {
const vector<string> models = {};
shm_unlink(sharedFile);

Inference inference;
ASSERT_TRUE(inference.Init(verbose, cpuChipId, models, &capture));
Inference inference{verbose, cpuChipId, models, &capture};
#ifdef __arm64__
PredictModel2(inference, cpuModel2, imageFile1, 653, 168, false);
PredictModel2(inference, cpuModel2, imageFile1, 653, 168, false);
Expand All @@ -428,8 +428,7 @@ TEST(InferenceUnittest, PredictCpuModel3) {
const vector<string> models = {};
shm_unlink(sharedFile);

Inference inference;
ASSERT_TRUE(inference.Init(verbose, cpuChipId, models, &capture));
Inference inference{verbose, cpuChipId, models, &capture};
#ifdef __arm64__
PredictModel3(inference, cpuModel3, imageFile1, 653, 190, false);
PredictModel3(inference, cpuModel3, imageFile1, 653, 190, false);
Expand All @@ -456,8 +455,7 @@ TEST(InferenceUnittest, PredictCpuModelMix) {
const vector<string> models = {};
shm_unlink(sharedFile);

Inference inference;
ASSERT_TRUE(inference.Init(verbose, cpuChipId, models, &capture));
Inference inference{verbose, cpuChipId, models, &capture};
#ifdef __arm64__
PredictModel1(inference, cpuModel1, imageFile1, 0.87890601, 0.58203125, false);
PredictModel2(inference, cpuModel2, imageFile1, 653, 168, false);
Expand Down Expand Up @@ -485,17 +483,15 @@ TEST(InferenceUnittest, InitDlpu) {
const bool verbose = get_verbose_status();
const vector<string> models = {cpuModel1};

Inference inference;
ASSERT_TRUE(inference.Init(verbose, dlpuChipId, models, &capture));
Inference inference{verbose, dlpuChipId, models, &capture};
}

TEST(InferenceUnittest, PredictDlpuModel1Preload) {
const bool verbose = get_verbose_status();
const vector<string> models = {cpuModel1};
shm_unlink(sharedFile);

Inference inference;
ASSERT_TRUE(inference.Init(verbose, dlpuChipId, models, &capture));
Inference inference{verbose, dlpuChipId, models, &capture};
PredictModel1(inference, cpuModel1, imageFile1, 0.878906, 0.5, true);
PredictModel1(inference, cpuModel1, imageFile1, 0.878906, 0.5, true);
PredictModel1(inference, cpuModel1, imageFile1, 0.878906, 0.5, true);
Expand All @@ -513,8 +509,7 @@ TEST(InferenceUnittest, PredictDlpuModel1) {
const vector<string> models = {};
shm_unlink(sharedFile);

Inference inference;
ASSERT_TRUE(inference.Init(verbose, dlpuChipId, models, &capture));
Inference inference{verbose, dlpuChipId, models, &capture};
PredictModel1(inference, cpuModel1, imageFile1, 0.878906, 0.5, false);
PredictModel1(inference, cpuModel1, imageFile1, 0.878906, 0.5, false);
PredictModel1(inference, cpuModel1, imageFile1, 0.878906, 0.5, true);
Expand All @@ -527,8 +522,7 @@ TEST(InferenceUnittest, PredictDlpuModel2) {
const vector<string> models = {};
shm_unlink(sharedFile);

Inference inference;
ASSERT_TRUE(inference.Init(verbose, dlpuChipId, models, &capture));
Inference inference{verbose, dlpuChipId, models, &capture};
PredictModel2(inference, cpuModel2, imageFile1, 653, 166, false);
PredictModel2(inference, cpuModel2, imageFile1, 653, 166, false);
PredictModel2(inference, cpuModel2, imageFile1, 653, 166, false);
Expand All @@ -544,8 +538,7 @@ TEST(InferenceUnittest, DISABLED_PredictDlpuModel3)
const vector<string> models = {};
shm_unlink(sharedFile);

Inference inference;
ASSERT_TRUE(inference.Init(verbose, dlpuChipId, models, &capture));
Inference inference{verbose, dlpuChipId, models, &capture};
PredictModel3(inference, cpuModel3, imageFile1, 653, 197, false);
PredictModel3(inference, cpuModel3, imageFile1, 653, 197, false);
PredictModel3(inference, cpuModel3, imageFile1, 653, 197, false);
Expand All @@ -557,17 +550,15 @@ TEST(InferenceUnittest, InitTpu) {
const bool verbose = get_verbose_status();
const vector<string> models = {tpuModel1};

Inference inference;
ASSERT_TRUE(inference.Init(verbose, tpuChipId, models, &capture));
Inference inference{verbose, tpuChipId, models, &capture};
}

TEST(InferenceUnittest, PredictTpuModel1Preload) {
const bool verbose = get_verbose_status();
const vector<string> models = {tpuModel1};
shm_unlink(sharedFile);

Inference inference;
ASSERT_TRUE(inference.Init(verbose, tpuChipId, models, &capture));
Inference inference{verbose, tpuChipId, models, &capture};
PredictModel1(inference, tpuModel1, imageFile1, 0.878906, 0.5, true);
PredictModel1(inference, tpuModel1, imageFile1, 0.878906, 0.5, true);
PredictModel1(inference, tpuModel1, imageFile1, 0.878906, 0.5, true);
Expand All @@ -585,8 +576,7 @@ TEST(InferenceUnittest, PredictTpuModel1) {
const vector<string> models = {};
shm_unlink(sharedFile);

Inference inference;
ASSERT_TRUE(inference.Init(verbose, tpuChipId, models, &capture));
Inference inference{verbose, tpuChipId, models, &capture};
PredictModel1(inference, tpuModel1, imageFile1, 0.878906, 0.5, false);
PredictModel1(inference, tpuModel1, imageFile1, 0.878906, 0.5, false);
PredictModel1(inference, tpuModel1, imageFile1, 0.878906, 0.5, true);
Expand All @@ -599,8 +589,7 @@ TEST(InferenceUnittest, PredictTpuModel2) {
const vector<string> models = {};
shm_unlink(sharedFile);

Inference inference;
ASSERT_TRUE(inference.Init(verbose, tpuChipId, models, &capture));
Inference inference{verbose, tpuChipId, models, &capture};
PredictModel2(inference, tpuModel2, imageFile1, 653, 118, false);
PredictModel2(inference, tpuModel2, imageFile1, 653, 118, false);
PredictModel2(inference, tpuModel2, imageFile1, 653, 118, false);
Expand All @@ -613,8 +602,7 @@ TEST(InferenceUnittest, PredictTpuModel3) {
const vector<string> models = {};
shm_unlink(sharedFile);

Inference inference;
ASSERT_TRUE(inference.Init(verbose, tpuChipId, models, &capture));
Inference inference{verbose, tpuChipId, models, &capture};
PredictModel3(inference, tpuModel3, imageFile1, 653, 197, false);
PredictModel3(inference, tpuModel3, imageFile1, 653, 197, false);
PredictModel3(inference, tpuModel3, imageFile1, 653, 197, false);
Expand Down

0 comments on commit 5b8c56a

Please sign in to comment.