Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dnsdist: add pooling support for RemoteLoggerInterface #15123

Merged
merged 25 commits into from
Feb 11, 2025
+713 −106
Merged

dnsdist: add pooling support for RemoteLoggerInterface #15123

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
965f0bb
dnsdist: add pooling support for RemoteLoggerInterface
esensar Feb 5, 2025
4282868
Fix typo in `checkAllParametersConsumed` for `newFrameStreamUnixLogger`
esensar Feb 5, 2025
adba4d1
Fix formatting in `dnsdist-lua-bindings-protobuf.cc`
esensar Feb 5, 2025
95fb88b
Fix typo in `dnsdist` makefile
esensar Feb 5, 2025
d1349d2
Remove `pick` from `dnsdist-lua-bindings-protobuf.cc`
esensar Feb 5, 2025
4e9477c
Add `remote_logger_pool` to meson
esensar Feb 5, 2025
fd65e60
Fix clang-tidy warnings
esensar Feb 5, 2025
2a62f7f
Reserve room in loggers vector before adding loggers
esensar Feb 5, 2025
f36ee29
Prevent erasing invalid value from options map
esensar Feb 5, 2025
73491fb
Add YAML support for multiple logger connections
esensar Feb 5, 2025
729b7ee
Fix `DnstapLogAction` and `DnstapLogResponseAction` in YAML config
esensar Feb 5, 2025
9cbf0bd
Add a basic lock in `RemoteLoggerInterface::queueData` for thread safety
esensar Feb 6, 2025
41a1eab
Shorten lock duration in `RemoteLoggerPool::queueData`
esensar Feb 6, 2025
6692e56
Replace locks in `remote_logger_pool` with an atomic counter
esensar Feb 7, 2025
8d43673
Change `RemoteLoggerPool::toString` to describe just general stats
esensar Feb 7, 2025
94e3fdc
Implement `RemoteLoggerPool::name`
esensar Feb 7, 2025
9de304d
Update docs for `connectionCount` parameter
esensar Feb 7, 2025
3a05381
Add `RemotePoolLogger` tests for TCP based logger
esensar Feb 7, 2025
42e52b7
Use integer placeholder in config template in `test_Dnstap.py`
esensar Feb 7, 2025
74c4614
Fix listener naming in `RemotePoolLogger` TCP tests
esensar Feb 7, 2025
ed7d865
Fix naming in listener for TcpLogger test
esensar Feb 7, 2025
b4f67d1
Fix remaining listener naming in `RemotePoolLogger` TCP tests
esensar Feb 7, 2025
5561909
Fix remaining naming in listener for TcpLogger test
esensar Feb 7, 2025
d7c88c9
Add `RemotePoolLogger` tests for unix socket based logger
esensar Feb 10, 2025
aaf5f4b
Add `RemotePoolLogger` tests for remote protobuf logger
esensar Feb 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pdns/dnsdistdist/Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,7 @@ dnsdist_SOURCES = \
proxy-protocol.cc proxy-protocol.hh \
qtype.cc qtype.hh \
remote_logger.cc remote_logger.hh \
remote_logger_pool.cc remote_logger_pool.hh \
sholder.hh \
snmp-agent.cc snmp-agent.hh \
sstuff.hh \
Expand Down
41 changes: 35 additions & 6 deletions pdns/dnsdistdist/dnsdist-configuration-yaml.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
#include "fstrm_logger.hh"
#include "iputils.hh"
#include "remote_logger.hh"
#include "remote_logger_pool.hh"
#include "xsk.hh"

#include "rust/cxx.h"
Expand Down Expand Up @@ -1434,9 +1435,12 @@ std::shared_ptr<DNSActionWrapper> getDnstapLogAction(const DnstapLogActionConfig
if (!logger && !(dnsdist::configuration::yaml::s_inClientMode || dnsdist::configuration::yaml::s_inConfigCheckMode)) {
throw std::runtime_error("Unable to find the dnstap logger named '" + std::string(config.logger_name) + "'");
}
boost::optional<dnsdist::actions::DnstapAlterFunction> alterFuncHolder;
dnsdist::actions::DnstapAlterFunction alterFunc;
dnsdist::configuration::yaml::getLuaFunctionFromConfiguration(alterFunc, config.alter_function_name, config.alter_function_code, config.alter_function_file, "dnstap log action");
auto action = dnsdist::actions::getDnstapLogAction(std::string(config.identity), std::move(logger), std::move(alterFunc));
if (dnsdist::configuration::yaml::getLuaFunctionFromConfiguration(alterFunc, config.alter_function_name, config.alter_function_code, config.alter_function_file, "dnstap log action")) {
alterFuncHolder = std::move(alterFunc);
}
auto action = dnsdist::actions::getDnstapLogAction(std::string(config.identity), std::move(logger), alterFuncHolder ? std::move(*alterFuncHolder) : std::optional<dnsdist::actions::DnstapAlterFunction>());
return newDNSActionWrapper(std::move(action), config.name);
#endif
}
Expand All @@ -1450,9 +1454,12 @@ std::shared_ptr<DNSResponseActionWrapper> getDnstapLogResponseAction(const Dnsta
if (!logger && !(dnsdist::configuration::yaml::s_inClientMode || dnsdist::configuration::yaml::s_inConfigCheckMode)) {
throw std::runtime_error("Unable to find the dnstap logger named '" + std::string(config.logger_name) + "'");
}
boost::optional<dnsdist::actions::DnstapAlterResponseFunction> alterFuncHolder;
dnsdist::actions::DnstapAlterResponseFunction alterFunc;
dnsdist::configuration::yaml::getLuaFunctionFromConfiguration(alterFunc, config.alter_function_name, config.alter_function_code, config.alter_function_file, "dnstap log response action");
auto action = dnsdist::actions::getDnstapLogResponseAction(std::string(config.identity), std::move(logger), std::move(alterFunc));
if (dnsdist::configuration::yaml::getLuaFunctionFromConfiguration(alterFunc, config.alter_function_name, config.alter_function_code, config.alter_function_file, "dnstap log response action")) {
alterFuncHolder = std::move(alterFunc);
}
auto action = dnsdist::actions::getDnstapLogResponseAction(std::string(config.identity), std::move(logger), alterFuncHolder ? std::move(*alterFuncHolder) : std::optional<dnsdist::actions::DnstapAlterResponseFunction>());
return newDNSResponseActionWrapper(std::move(action), config.name);
#endif
}
Expand Down Expand Up @@ -1533,7 +1540,18 @@ void registerProtobufLogger(const ProtobufLoggerConfiguration& config)
dnsdist::configuration::yaml::registerType<RemoteLoggerInterface>(object, config.name);
return;
}
auto object = std::shared_ptr<RemoteLoggerInterface>(std::make_shared<RemoteLogger>(ComboAddress(std::string(config.address)), config.timeout, config.max_queued_entries * 100, config.reconnect_wait_time, false));
std::shared_ptr<RemoteLoggerInterface> object;
if (config.connection_count > 1) {
std::vector<std::shared_ptr<RemoteLoggerInterface>> loggers;
loggers.reserve(config.connection_count);
for (uint64_t i = 0; i < config.connection_count; i++) {
loggers.push_back(std::make_shared<RemoteLogger>(ComboAddress(std::string(config.address)), config.timeout, config.max_queued_entries * 100, config.reconnect_wait_time, dnsdist::configuration::yaml::s_inClientMode));
}
object = std::shared_ptr<RemoteLoggerInterface>(std::make_shared<RemoteLoggerPool>(std::move(loggers)));
}
else {
object = std::shared_ptr<RemoteLoggerInterface>(std::make_shared<RemoteLogger>(ComboAddress(std::string(config.address)), config.timeout, config.max_queued_entries * 100, config.reconnect_wait_time, dnsdist::configuration::yaml::s_inClientMode));
}
dnsdist::configuration::yaml::registerType<RemoteLoggerInterface>(object, config.name);
#endif
}
Expand Down Expand Up @@ -1569,7 +1587,18 @@ void registerDnstapLogger(const DnstapLoggerConfiguration& config)
options["queueNotifyThreshold"] = config.queue_notify_threshold;
options["reopenInterval"] = config.reopen_interval;

auto object = std::shared_ptr<RemoteLoggerInterface>(std::make_shared<FrameStreamLogger>(family, std::string(config.address), false, options));
std::shared_ptr<RemoteLoggerInterface> object;
if (config.connection_count > 1) {
std::vector<std::shared_ptr<RemoteLoggerInterface>> loggers;
loggers.reserve(config.connection_count);
for (uint64_t i = 0; i < config.connection_count; i++) {
loggers.push_back(std::make_shared<FrameStreamLogger>(family, std::string(config.address), !dnsdist::configuration::yaml::s_inClientMode, options));
}
object = std::shared_ptr<RemoteLoggerInterface>(std::make_shared<RemoteLoggerPool>(std::move(loggers)));
}
else {
object = std::shared_ptr<RemoteLoggerInterface>(std::make_shared<FrameStreamLogger>(family, std::string(config.address), !dnsdist::configuration::yaml::s_inClientMode, options));
}
dnsdist::configuration::yaml::registerType<RemoteLoggerInterface>(object, config.name);
#endif
}
Expand Down
Loading
Loading