From 1da12085d7c2062a00d6826dbafebfc4abd8af81 Mon Sep 17 00:00:00 2001 From: Arran Cudbard-Bell Date: Tue, 9 Feb 2016 16:18:53 -0800 Subject: [PATCH] Add triggers for min/max reserve time for connection pools --- mibs/FREERADIUS-NOTIFICATION-MIB.mib | 4 +-- raddb/trigger.conf | 12 +++++++++ src/include/connection.h | 10 +++++++- src/main/connection.c | 37 ++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 3 deletions(-) diff --git a/mibs/FREERADIUS-NOTIFICATION-MIB.mib b/mibs/FREERADIUS-NOTIFICATION-MIB.mib index f711d6eb69cc..4b4a8994bbf8 100644 --- a/mibs/FREERADIUS-NOTIFICATION-MIB.mib +++ b/mibs/FREERADIUS-NOTIFICATION-MIB.mib @@ -131,7 +131,7 @@ serverModuleConnectionFail NOTIFICATION-TYPE DESCRIPTION "Notification that the module has failed to open a new connection" ::= { serverModuleGeneric 4 } -serverModuleConnectionReservedPeriodHigh NOTIFICATION-TYPE +serverModuleConnectionReservedPeriodMax NOTIFICATION-TYPE OBJECTS { radiusdModuleName, radiusdModuleInstance, radiusdConnectionPoolServer, @@ -140,7 +140,7 @@ serverModuleConnectionReservedPeriodHigh NOTIFICATION-TYPE DESCRIPTION "Notification that the period a connection was reserved for exceeded the configured maximum" ::= { serverModuleGeneric 5 } -serverModuleConnectionReservedPeriodLow NOTIFICATION-TYPE +serverModuleConnectionReservedPeriodMin NOTIFICATION-TYPE OBJECTS { radiusdModuleName, radiusdModuleInstance, radiusdConnectionPoolServer, diff --git a/raddb/trigger.conf b/raddb/trigger.conf index a9065b169ad4..7cb3ca65a80d 100644 --- a/raddb/trigger.conf +++ b/raddb/trigger.conf @@ -221,6 +221,12 @@ trigger { # The module has been HUP'd via radmin hup = "${snmptrap}::serverModuleHup ${args}" + + # Connection was released too quickly + min = "${snmptrap}::serverModuleConnectionReservedPeriodMin ${args}" + + # Connection was held for too long + max = "${snmptrap}::serverModuleConnectionReservedPeriodMax ${args}" } # The SQL module @@ -239,6 +245,12 @@ trigger { # The module has been HUP'd via radmin hup = "${snmptrap}::serverModuleHup ${args}" + + # Connection was released too quickly + min = "${snmptrap}::serverModuleConnectionReservedPeriodMin ${args}" + + # Connection was held for too long + max = "${snmptrap}::serverModuleConnectionReservedPeriodMax ${args}" } # You can also use connection pool's start/stop/open/close triggers diff --git a/src/include/connection.h b/src/include/connection.h index 6eb6b692d77a..1127f6a2c632 100644 --- a/src/include/connection.h +++ b/src/include/connection.h @@ -27,6 +27,7 @@ RCSIDH(connection_h, "$Id$") #include +#include #ifdef __cplusplus extern "C" { @@ -47,7 +48,14 @@ typedef struct fr_connection_pool_state { struct timeval last_released; //!< Last time a connection was released. struct timeval last_closed; //!< Last time a connection was closed. - int next_delay; //!< The next delay time. cleanup. Initialized to +#ifdef WITH_STATS + fr_stats_t held_stats; //!< How long connections were held for. +#endif + + time_t last_held_min; //!< Last time we warned about a low latency event. + time_t last_held_max; //!< Last time we warned about a high latency event. + + uint32_t next_delay; //!< The next delay time. cleanup. Initialized to //!< cleanup_interval, and decays from there. uint64_t count; //!< Number of connections spawned over the lifetime diff --git a/src/main/connection.c b/src/main/connection.c index 3b6bc720a5da..e784ebacfec5 100644 --- a/src/main/connection.c +++ b/src/main/connection.c @@ -128,6 +128,11 @@ struct fr_connection_pool_t { //!< fired by the connection pool code. VALUE_PAIR *trigger_args; //!< Arguments to make available in connection pool triggers. + struct timeval held_trigger_min; //!< If a connection is held for less than the specified + //!< period, fire a trigger. + struct timeval held_trigger_max; //!< If a connection is held for longer than the specified + //!< period, fire a trigger. + fr_connection_create_t create; //!< Function used to create new connections. fr_connection_alive_t alive; //!< Function used to check status of connections. fr_connection_pool_reconnect_t reconnect; //!< Called during connection pool reconnect. @@ -146,6 +151,8 @@ static const CONF_PARSER connection_config[] = { { FR_CONF_OFFSET("cleanup_interval", PW_TYPE_INTEGER, fr_connection_pool_t, cleanup_interval), .dflt = "30" }, { FR_CONF_OFFSET("idle_timeout", PW_TYPE_INTEGER, fr_connection_pool_t, idle_timeout), .dflt = "60" }, { FR_CONF_OFFSET("connect_timeout", PW_TYPE_TIMEVAL, fr_connection_pool_t, connect_timeout), .dflt = "3.0" }, + { FR_CONF_OFFSET("held_trigger_min", PW_TYPE_TIMEVAL, fr_connection_pool_t, held_trigger_min), .dflt = "0.0" }, + { FR_CONF_OFFSET("held_trigger_max", PW_TYPE_TIMEVAL, fr_connection_pool_t, held_trigger_max), .dflt = "0.5" }, { FR_CONF_OFFSET("retry_delay", PW_TYPE_INTEGER, fr_connection_pool_t, retry_delay), .dflt = "1" }, { FR_CONF_OFFSET("spread", PW_TYPE_BOOLEAN, fr_connection_pool_t, spread), .dflt = "no" }, CONF_PARSER_TERMINATOR @@ -1335,6 +1342,8 @@ void *fr_connection_get(fr_connection_pool_t *pool, REQUEST *request) void fr_connection_release(fr_connection_pool_t *pool, REQUEST *request, void *conn) { fr_connection_t *this; + struct timeval held; + bool trigger_min = false, trigger_max = false; this = fr_connection_find(pool, conn); if (!this) return; @@ -1347,6 +1356,31 @@ void fr_connection_release(fr_connection_pool_t *pool, REQUEST *request, void *c gettimeofday(&this->last_released, NULL); pool->state.last_released = this->last_released; + /* + * This is done inside the mutex to ensure + * updates are atomic. + */ + fr_timeval_subtract(&held, &this->last_released, &this->last_reserved); + + /* + * Check we've not exceeded out trigger limits + */ + if ((pool->held_trigger_min.tv_sec || pool->held_trigger_min.tv_usec) && + (fr_timeval_cmp(&held, &pool->held_trigger_min) < 0) && + (pool->state.last_held_min != this->last_released.tv_sec)) { + trigger_min = true; + pool->state.last_held_min = this->last_released.tv_sec; + } + + if ((pool->held_trigger_max.tv_sec || pool->held_trigger_min.tv_usec) && + (fr_timeval_cmp(&held, &pool->held_trigger_max) > 0) && + (pool->state.last_held_max != this->last_released.tv_sec)) { + trigger_max = true; + pool->state.last_held_max = this->last_released.tv_sec; + } + + fr_stats_bins(&pool->state.held_stats, &this->last_reserved, &this->last_released); + /* * Insert the connection in the heap. * @@ -1368,6 +1402,9 @@ void fr_connection_release(fr_connection_pool_t *pool, REQUEST *request, void *c * connections, go manage the pool && clean some up. */ fr_connection_pool_check(pool); + + if (trigger_min) fr_connection_trigger_exec(pool, "min"); + if (trigger_max) fr_connection_trigger_exec(pool, "max"); } /** Reconnect a suspected inviable connection