Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reader merge and overlay support #124

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Reader merge and overlay support
kyle-figure committed Oct 7, 2023
commit 9daf4ad40b5229a675f8e21d3845987347f290d3
1 change: 0 additions & 1 deletion .github/workflows/cmake.yml
Original file line number Diff line number Diff line change
@@ -131,7 +131,6 @@ jobs:
# Execute tests defined by the CMake configuration.
# See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
run: |
export TEST_DIR=env
ctest -C ${{env.BUILD_TYPE}}
${{github.workspace}}/build/src/include_file_test
for i in `seq 1 13`; do
8 changes: 8 additions & 0 deletions include/flexi_cfg/config/classes.h
Original file line number Diff line number Diff line change
@@ -102,6 +102,14 @@ inline auto operator<<(std::ostream& os, const std::shared_ptr<T>& cfg) -> std::
return (cfg ? (os << *cfg) : (os << "NULL"));
}

inline auto operator==(const ConfigBase& lhs, const ConfigBase& rhs) -> bool {
std::stringstream ss_left;
ss_left << lhs;
std::stringstream ss_rhs;
ss_rhs << rhs;
return ss_left.str() == ss_rhs.str();
}

class ConfigStructLike;
template <typename Key, typename Value>
inline auto operator<<(std::ostream& os, const std::map<Key, Value>& data) -> std::ostream& {
12 changes: 12 additions & 0 deletions include/flexi_cfg/config/helpers.h
Original file line number Diff line number Diff line change
@@ -39,11 +39,23 @@ auto isStructLike(const types::BasePtr& el) -> bool;
auto checkForErrors(const types::CfgMap& cfg1, const types::CfgMap& cfg2, const std::string& key)
-> bool;

/// @brief Merge two dictionaries recursively, prioritizing rhs over lhs in the event of a conflict.
/// @param lhs Base dictionary to merge into
/// @param rhs Overriding dictionary
/// @param is_overlay If true, then the merge will be done in an overlay fashion, meaning that all keys in rhs must exist in lhs with the same value type.
void mergeLeft(types::CfgMap& lhs, const types::CfgMap& rhs, const bool is_overlay = false);

/* Merge dictionaries recursively and keep all nested keys combined between the two dictionaries.
* Any key/value pairs that already exist in the leaves of cfg1 will be overwritten by the same
* key/value pairs from cfg2. */
auto mergeNestedMaps(const types::CfgMap& cfg1, const types::CfgMap& cfg2) -> types::CfgMap;

/// @brief Compare two maps recursively
/// @param lhs First map
/// @param rhs Second map
/// @return true if the two maps are identical
auto compareNestedMaps(const types::CfgMap& lhs, const types::CfgMap& rhs) -> bool;

auto structFromReference(std::shared_ptr<types::ConfigReference>& ref,
const std::shared_ptr<types::ConfigProto>& proto)
-> std::shared_ptr<types::ConfigStruct>;
24 changes: 20 additions & 4 deletions include/flexi_cfg/reader.h
Original file line number Diff line number Diff line change
@@ -70,14 +70,30 @@ class Reader {
/// \return A vector of keys for all structs containing 'key'
[[nodiscard]] auto findStructsWithKey(const std::string& key) const -> std::vector<std::string>;

protected:
[[nodiscard]] auto getNestedConfig(const std::string& key) const
-> std::pair<std::string, const config::types::CfgMap&>;

/// \note: This method is here in order to enable the python bindings to more easily parse list
/// types
[[nodiscard]] auto getCfgMap() const -> const config::types::CfgMap& { return cfg_data_; }

/// \brief Merge another config into this one
/// \param[in] other The other config to merge into this one
/// \note If there are any conflicts, the other config will take precedence
void merge(const Reader& other);

/// @brief Merge another config into this one, enforcing that all keys to be merged exist.
/// @param overlay Another config to merge into this one
void applyOverlay(const Reader& overlay);

[[nodiscard]] auto friend operator==(const Reader& lhs, const Reader& rhs) -> bool {
return config::helpers::compareNestedMaps(lhs.cfg_data_, rhs.cfg_data_);
}
[[nodiscard]] auto friend operator!=(const Reader& lhs, const Reader& rhs) -> bool {
return !config::helpers::compareNestedMaps(lhs.cfg_data_, rhs.cfg_data_);
}

protected:
[[nodiscard]] auto getNestedConfig(const std::string& key) const
-> std::pair<std::string, const config::types::CfgMap&>;

static void convert(const config::types::ValuePtr& value_ptr, float& value);
static void convert(const config::types::ValuePtr& value_ptr, double& value);
static void convert(const config::types::ValuePtr& value_ptr, int& value);
8 changes: 7 additions & 1 deletion python/flexi_cfg_py.cpp
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@
#include <flexi_cfg/parser.h>
#include <flexi_cfg/reader.h>
#include <flexi_cfg/utils.h>
#include <pybind11/operators.h>
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <pybind11/stl/filesystem.h>
@@ -184,6 +185,8 @@ PYBIND11_MODULE(flexi_cfg, m) {
.def("keys", &flexi_cfg::Reader::keys)
.def("getType", &flexi_cfg::Reader::getType)
.def("findStructWithKey", &flexi_cfg::Reader::findStructsWithKey)
.def("merge", &flexi_cfg::Reader::merge)
.def("applyOverlay", &flexi_cfg::Reader::applyOverlay)
// Accessors for single values
.def("getInt", &getValueHelper<int64_t>)
.def("getUint64", &getValueHelper<uint64_t>)
@@ -199,7 +202,10 @@ PYBIND11_MODULE(flexi_cfg, m) {
// Accessor for a sub-reader object
.def("getReader", &getValueHelper<flexi_cfg::Reader>)
// Generic accessor
.def("getValue", &getValueGeneric);
.def("getValue", &getValueGeneric)
// Equality operator
.def(py::self == py::self)
.def(py::self != py::self);

py::class_<flexi_cfg::Parser>(m, "Parser")
.def_static("parse",
70 changes: 69 additions & 1 deletion src/config_helpers.cpp
Original file line number Diff line number Diff line change
@@ -35,7 +35,7 @@ auto isStructLike(const types::BasePtr& el) -> bool {
// Three cases to check for:
// - Both are dictionaries - This is okay
// - Neither are dictionaries - This is bad - even if the same value, we don't allow duplicates
// - Only ones is a dictionary - Also bad. We can't handle this one
// - Only one is a dictionary - Also bad. We can't handle this one
auto checkForErrors(const types::CfgMap& cfg1, const types::CfgMap& cfg2, const std::string& key)
-> bool {
const auto dict_count =
@@ -59,6 +59,40 @@ auto checkForErrors(const types::CfgMap& cfg1, const types::CfgMap& cfg2, const
return dict_count == 2; // All good if both are dictionaries (for now);
}

void mergeLeft(types::CfgMap& lhs, const types::CfgMap& rhs, const bool is_overlay) {
std::for_each(std::begin(rhs), std::end(rhs), [&](const auto& rhs_kv) {
const auto& rhs_key = rhs_kv.first;
const auto& rhs_val = rhs_kv.second;
// rhs key not found in lhs, so add it
if (!lhs.contains(rhs_key)) {
if (is_overlay) {
THROW_EXCEPTION(InvalidKeyException,
"Key '{}' not found in lhs, but is required for overlay merge.", rhs_key);
}
lhs.emplace(rhs_key, rhs_val);
return;
}

// If this is an overlay and the types don't match, throw an exception
if (is_overlay && (lhs.at(rhs_key)->type != rhs_val->type)) {
THROW_EXCEPTION(
MismatchTypeException,
"Types at key '{}' must match for an overlay merge. lhs is '{}', rhs is '{}'.", rhs_key,
lhs.at(rhs_key)->type, rhs_val->type);
}

// Both children are struct-like, so recurse into them
if (isStructLike(lhs.at(rhs_key)) && isStructLike(rhs.at(rhs_key))) {
mergeLeft(dynamic_pointer_cast<types::ConfigStructLike>(lhs.at(rhs_key))->data,
dynamic_pointer_cast<types::ConfigStructLike>(rhs.at(rhs_key))->data, is_overlay);
return;
}

// Replace lhs value with rhs value
lhs.at(rhs_key) = rhs_val;
});
}

/* Merge dictionaries recursively and keep all nested keys combined between the two dictionaries.
* Any key/value pairs that already exist in the leaves of cfg1 will be overwritten by the same
* key/value pairs from cfg2. */
@@ -95,6 +129,40 @@ auto mergeNestedMaps(const types::CfgMap& cfg1, const types::CfgMap& cfg2) -> ty
return cfg_out;
}

auto compareNestedMaps(const types::CfgMap& lhs, const types::CfgMap& rhs) -> bool {
if (lhs.size() != rhs.size()) {
return false;
}

bool match = true;
std::for_each(std::begin(lhs), std::end(lhs), [&](const auto& lhs_kv) {
const auto& k = lhs_kv.first;
const auto& lhs_val = lhs_kv.second;
// Check that rhs contains the key
if (!rhs.contains(k)) {
match = false;
return;
}
const auto& rhs_val = rhs.at(k);
// Compare types
if (lhs_val->type != rhs_val->type) {
match = false;
return;
}
// If both are dictionaries, recurse into them
if (isStructLike(lhs_val) && isStructLike(rhs_val)) {
match =
match && compareNestedMaps(dynamic_pointer_cast<types::ConfigStructLike>(lhs_val)->data,
dynamic_pointer_cast<types::ConfigStructLike>(rhs_val)->data);
}
// Otherwise, compare the values
match = match && (*lhs_val == *rhs_val);
return;
});

return match;
}

auto structFromReference(std::shared_ptr<types::ConfigReference>& ref,
const std::shared_ptr<types::ConfigProto>& proto)
-> std::shared_ptr<types::ConfigStruct> {
8 changes: 8 additions & 0 deletions src/config_reader.cpp
Original file line number Diff line number Diff line change
@@ -197,4 +197,12 @@ void Reader::getValue(const std::string& key, Reader& reader) const {
reader = Reader(struct_like->data, key);
}

void Reader::merge(const Reader& other) {
config::helpers::mergeLeft(cfg_data_, other.getCfgMap());
}

void Reader::applyOverlay(const Reader& overlay) {
config::helpers::mergeLeft(cfg_data_, overlay.getCfgMap(), true);
}

} // namespace flexi_cfg
20 changes: 20 additions & 0 deletions tests/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -124,3 +124,23 @@ target_include_directories(math_test PRIVATE

add_clang_format(math_test)
gtest_discover_tests(math_test)

################################################################################
add_executable(
config_merge_test
config_merge_test.cpp
)

target_link_libraries(
config_merge_test
flexi_cfg
taocpp::pegtl
gtest_main
)

target_include_directories(config_merge_test PRIVATE
${PROJECT_SOURCE_DIR}/include/
)

add_clang_format(config_merge_test)
gtest_discover_tests(config_merge_test)
125 changes: 125 additions & 0 deletions tests/config_helpers_test.cpp
Original file line number Diff line number Diff line change
@@ -617,3 +617,128 @@ TEST(ConfigHelpers, resolveVarRefs) {
flexi_cfg::config::CyclicReferenceException);
}
}

TEST(ConfigHelpers, compareNestedMaps) {
{
const std::string key = "key";
const std::vector<std::string> inner_keys = {"key1", "key2"};

// cfg1 is:
// struct key
// key1 = ""
// key2 = ""
flexi_cfg::config::types::CfgMap cfg1_inner = {
{inner_keys[0], std::make_shared<flexi_cfg::config::types::ConfigValue>("", kValue)},
{inner_keys[1], std::make_shared<flexi_cfg::config::types::ConfigValue>("", kValue)},
};
auto cfg1_struct = std::make_shared<flexi_cfg::config::types::ConfigStruct>(key, 0);
cfg1_struct->data = std::move(cfg1_inner);
flexi_cfg::config::types::CfgMap cfg1 = {{cfg1_struct->name, std::move(cfg1_struct)}};

// cfg2 is:
// struct key
// key1 = ""
// key2 = ""
flexi_cfg::config::types::CfgMap cfg2_inner = {
{inner_keys[0], std::make_shared<flexi_cfg::config::types::ConfigValue>("", kValue)},
{inner_keys[1], std::make_shared<flexi_cfg::config::types::ConfigValue>("", kValue)},
};
auto cfg2_struct = std::make_shared<flexi_cfg::config::types::ConfigStruct>(key, 0);
cfg2_struct->data = std::move(cfg2_inner);
flexi_cfg::config::types::CfgMap cfg2 = {{cfg2_struct->name, std::move(cfg2_struct)}};

EXPECT_TRUE(flexi_cfg::config::helpers::compareNestedMaps(cfg1, cfg2));
}

{
const std::string key = "key";
const std::vector<std::string> inner_keys = {"key1"};

// cfg1 is:
// struct key
// key1 = ""
flexi_cfg::config::types::CfgMap cfg1_inner = {
{inner_keys[0], std::make_shared<flexi_cfg::config::types::ConfigValue>("", kValue)},
};
auto cfg1_struct = std::make_shared<flexi_cfg::config::types::ConfigStruct>(key, 0);
cfg1_struct->data = std::move(cfg1_inner);
flexi_cfg::config::types::CfgMap cfg1 = {{cfg1_struct->name, std::move(cfg1_struct)}};

// cfg2 is:
// struct key
// key1 = "not_equal"
flexi_cfg::config::types::CfgMap cfg2_inner = {
{inner_keys[0],
std::make_shared<flexi_cfg::config::types::ConfigValue>("not_equal", kValue)},
};
auto cfg2_struct = std::make_shared<flexi_cfg::config::types::ConfigStruct>(key, 0);
cfg2_struct->data = std::move(cfg2_inner);
flexi_cfg::config::types::CfgMap cfg2 = {{cfg2_struct->name, std::move(cfg2_struct)}};

EXPECT_FALSE(flexi_cfg::config::helpers::compareNestedMaps(cfg1, cfg2));
}

{
const std::string key = "key";
const std::vector<std::string> inner_keys = {"key1", "key2"};

// cfg1 is:
// struct key
// key1 = ""
flexi_cfg::config::types::CfgMap cfg1_inner = {
{inner_keys[0], std::make_shared<flexi_cfg::config::types::ConfigValue>("", kValue)},
};
auto cfg1_struct = std::make_shared<flexi_cfg::config::types::ConfigStruct>(key, 0);
cfg1_struct->data = std::move(cfg1_inner);
flexi_cfg::config::types::CfgMap cfg1 = {{cfg1_struct->name, std::move(cfg1_struct)}};

// cfg2 is:
// struct key
// key2 = ""
flexi_cfg::config::types::CfgMap cfg2_inner = {
{inner_keys[1], std::make_shared<flexi_cfg::config::types::ConfigValue>("", kValue)},
};
auto cfg2_struct = std::make_shared<flexi_cfg::config::types::ConfigStruct>(key, 0);
cfg2_struct->data = std::move(cfg2_inner);
flexi_cfg::config::types::CfgMap cfg2 = {{cfg2_struct->name, std::move(cfg2_struct)}};

EXPECT_FALSE(flexi_cfg::config::helpers::compareNestedMaps(cfg1, cfg2));
}

{
const std::string key = "key";
const std::vector<std::string> inner_keys = {"key1"};

// cfg1 is:
// struct key
// struct nested
// key1 = "1"
flexi_cfg::config::types::CfgMap cfg1_lvl2 = {
{inner_keys[0], std::make_shared<flexi_cfg::config::types::ConfigValue>("1", kValue)}};
auto cfg1_inner = std::make_shared<flexi_cfg::config::types::ConfigStruct>(
"nested", 1 /* depth doesn't matter */);
cfg1_inner->data = std::move(cfg1_lvl2);
flexi_cfg::config::types::CfgMap cfg1_lvl1 = {{cfg1_inner->name, std::move(cfg1_inner)}};
auto cfg1_outer =
std::make_shared<flexi_cfg::config::types::ConfigStruct>(key, 0 /* depth doesn't matter */);
cfg1_outer->data = std::move(cfg1_lvl1);
flexi_cfg::config::types::CfgMap cfg1 = {{cfg1_outer->name, std::move(cfg1_outer)}};

// cfg2 is:
// struct key
// struct nested
// key1 = "2"
flexi_cfg::config::types::CfgMap cfg2_lvl2 = {
{inner_keys[0], std::make_shared<flexi_cfg::config::types::ConfigValue>("2", kValue)}};
auto cfg2_inner = std::make_shared<flexi_cfg::config::types::ConfigStruct>(
"nested", 1 /* depth doesn't matter */);
cfg2_inner->data = std::move(cfg2_lvl2);
flexi_cfg::config::types::CfgMap cfg2_lvl1 = {{cfg2_inner->name, std::move(cfg2_inner)}};
auto cfg2_outer =
std::make_shared<flexi_cfg::config::types::ConfigStruct>(key, 0 /* depth doesn't matter */);
cfg2_outer->data = std::move(cfg2_lvl1);
flexi_cfg::config::types::CfgMap cfg2 = {{cfg2_outer->name, std::move(cfg2_outer)}};

EXPECT_FALSE(flexi_cfg::config::helpers::compareNestedMaps(cfg1, cfg2));
}
}
Loading