diff --git a/presto-native-execution/presto_cpp/main/CMakeLists.txt b/presto-native-execution/presto_cpp/main/CMakeLists.txt index abe71dd6f3fa..1508eb90db2e 100644 --- a/presto-native-execution/presto_cpp/main/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/CMakeLists.txt @@ -76,7 +76,8 @@ target_link_libraries( ${FOLLY_WITH_DEPENDENCIES} ${GLOG} ${GFLAGS_LIBRARIES} - pthread) + pthread + velox_dynamic_function_loader) # Enabling Parquet causes build errors with missing symbols on MacOS. This is # likely due to a conflict between Arrow Thrift from velox_hive_connector and @@ -89,6 +90,7 @@ set_property(TARGET presto_server_lib PROPERTY JOB_POOL_LINK presto_link_job_pool) add_executable(presto_server PrestoMain.cpp) +target_link_options(presto_server BEFORE PUBLIC "-Wl,-export-dynamic") # Moving velox_hive_connector and velox_tpch_connector to presto_server_lib # results in multiple link errors similar to the one below only on GCC. diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.cpp b/presto-native-execution/presto_cpp/main/PrestoServer.cpp index 1f316144bbfc..8526e9a4325f 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoServer.cpp @@ -44,6 +44,7 @@ #include "velox/common/base/StatsReporter.h" #include "velox/common/caching/CacheTTLController.h" #include "velox/common/caching/SsdCache.h" +#include "velox/common/dynamic_registry/DynamicLibraryLoader.h" #include "velox/common/file/FileSystems.h" #include "velox/common/memory/MmapAllocator.h" #include "velox/common/memory/SharedArbitrator.h" @@ -392,6 +393,7 @@ void PrestoServer::run() { registerRemoteFunctions(); registerVectorSerdes(); registerPrestoPlanNodeSerDe(); + registerDynamicFunctions(); const auto numExchangeHttpClientIoThreads = std::max( systemConfig->exchangeHttpClientNumIoThreadsHwMultiplier() * @@ -1593,5 +1595,28 @@ protocol::NodeStatus PrestoServer::fetchNodeStatus() { return nodeStatus; } +void PrestoServer::registerDynamicFunctions() { + auto systemConfig = SystemConfig::instance(); + if (!systemConfig->pluginDir().empty()) { + // if it is a valid directory, traverse and call dynamic function loader + const fs::path path(systemConfig->pluginDir()); + PRESTO_STARTUP_LOG(INFO) << path; + std::error_code + ec; // For using the non-throwing overloads of functions below. + if (fs::is_directory(path, ec)) { + using recursiveDirectoryIterator = + std::filesystem::recursive_directory_iterator; + std::set extensions{".so", ".dylib"}; + for (const auto& dirEntry : recursiveDirectoryIterator(path)) { + // skip non shared library files and directories from loading + auto dirEntryPath = dirEntry.path(); + if (!fs::is_directory(dirEntry, ec) && + extensions.find(dirEntryPath.extension()) != extensions.end()) { + facebook::velox::loadDynamicLibrary(dirEntryPath.c_str()); + } + } + } + } +} } // namespace facebook::presto diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.h b/presto-native-execution/presto_cpp/main/PrestoServer.h index 615d6bfb910d..0b4b3cf119b6 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.h +++ b/presto-native-execution/presto_cpp/main/PrestoServer.h @@ -190,6 +190,8 @@ class PrestoServer { VeloxPlanValidator* getVeloxPlanValidator(); + virtual void registerDynamicFunctions(); + /// Invoked to get the list of filters passed to the http server. std::vector> getHttpServerFilters(); diff --git a/presto-native-execution/presto_cpp/main/common/Configs.cpp b/presto-native-execution/presto_cpp/main/common/Configs.cpp index a3227719eb9e..4b2f9c60c9de 100644 --- a/presto-native-execution/presto_cpp/main/common/Configs.cpp +++ b/presto-native-execution/presto_cpp/main/common/Configs.cpp @@ -240,6 +240,7 @@ SystemConfig::SystemConfig() { STR_PROP(kCacheVeloxTtlCheckInterval, "1h"), BOOL_PROP(kEnableRuntimeMetricsCollection, false), BOOL_PROP(kPlanValidatorFailOnNestedLoopJoin, false), + STR_PROP(kPluginDir, ""), }; } @@ -763,6 +764,10 @@ bool SystemConfig::enableRuntimeMetricsCollection() const { return optionalProperty(kEnableRuntimeMetricsCollection).value(); } +std::string SystemConfig::pluginDir() const { + return optionalProperty(kPluginDir).value(); +} + NodeConfig::NodeConfig() { registeredProps_ = std::unordered_map>{ diff --git a/presto-native-execution/presto_cpp/main/common/Configs.h b/presto-native-execution/presto_cpp/main/common/Configs.h index fcd17794c1c6..002b7986e454 100644 --- a/presto-native-execution/presto_cpp/main/common/Configs.h +++ b/presto-native-execution/presto_cpp/main/common/Configs.h @@ -648,6 +648,8 @@ class SystemConfig : public ConfigBase { static constexpr std::string_view kInternalCommunicationJwtExpirationSeconds{ "internal-communication.jwt.expiration-seconds"}; + /// Optional string containing the path to the plugin directory + static constexpr std::string_view kPluginDir{"plugin.dir"}; /// Below are the Presto properties from config.properties that get converted /// to their velox counterparts in BaseVeloxQueryConfig and used solely from /// BaseVeloxQueryConfig. @@ -899,6 +901,8 @@ class SystemConfig : public ConfigBase { bool enableRuntimeMetricsCollection() const; bool prestoNativeSidecar() const; + + std::string pluginDir() const; }; /// Provides access to node properties defined in node.properties file.