Add triggers for min/max reserve time for connection pools

mibs/FREERADIUS-NOTIFICATION-MIB.mib

@@ -131,7 +131,7 @@ serverModuleConnectionFail NOTIFICATION-TYPE
        DESCRIPTION "Notification that the module has failed to open a new connection"
        ::= { serverModuleGeneric 4 }
 
-serverModuleConnectionReservedPeriodHigh NOTIFICATION-TYPE
+serverModuleConnectionReservedPeriodMax NOTIFICATION-TYPE
        OBJECTS { radiusdModuleName,
 		 radiusdModuleInstance,
 		 radiusdConnectionPoolServer,
@@ -140,7 +140,7 @@ serverModuleConnectionReservedPeriodHigh NOTIFICATION-TYPE
        DESCRIPTION "Notification that the period a connection was reserved for exceeded the configured maximum"
        ::= { serverModuleGeneric 5 }
 
-serverModuleConnectionReservedPeriodLow NOTIFICATION-TYPE
+serverModuleConnectionReservedPeriodMin NOTIFICATION-TYPE
        OBJECTS { radiusdModuleName,
 		 radiusdModuleInstance,
 		 radiusdConnectionPoolServer,

raddb/trigger.conf

@@ -221,6 +221,12 @@ trigger {
 
 			# The module has been HUP'd via radmin
 			hup = "${snmptrap}::serverModuleHup ${args}"
+
+			# Connection was released too quickly
+			min = "${snmptrap}::serverModuleConnectionReservedPeriodMin ${args}"
+
+			# Connection was held for too long
+			max = "${snmptrap}::serverModuleConnectionReservedPeriodMax ${args}"
 		}
 
 		# The SQL module
@@ -239,6 +245,12 @@ trigger {
 
 			# The module has been HUP'd via radmin
 			hup = "${snmptrap}::serverModuleHup ${args}"
+
+			# Connection was released too quickly
+			min = "${snmptrap}::serverModuleConnectionReservedPeriodMin ${args}"
+
+			# Connection was held for too long
+			max = "${snmptrap}::serverModuleConnectionReservedPeriodMax ${args}"
 		}
 
 		# You can also use connection pool's start/stop/open/close triggers

src/include/connection.h

@@ -27,6 +27,7 @@
 RCSIDH(connection_h, "$Id$")
 
 #include <freeradius-devel/radiusd.h>
+#include <freeradius-devel/stats.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -47,7 +48,14 @@ typedef struct fr_connection_pool_state {
 	struct timeval	last_released;		//!< Last time a connection was released.
 	struct timeval	last_closed;		//!< Last time a connection was closed.
 
-	int		next_delay;    	 	//!< The next delay time.  cleanup.  Initialized to
+#ifdef WITH_STATS
+	fr_stats_t	held_stats;		//!< How long connections were held for.
+#endif
+
+	time_t		last_held_min;		//!< Last time we warned about a low latency event.
+	time_t		last_held_max;		//!< Last time we warned about a high latency event.
+
+	uint32_t	next_delay;    	 	//!< The next delay time.  cleanup.  Initialized to
 						//!< cleanup_interval, and decays from there.
 
 	uint64_t	count;			//!< Number of connections spawned over the lifetime

src/main/connection.c

@@ -128,6 +128,11 @@ struct fr_connection_pool_t {
 						//!< fired by the connection pool code.
 	VALUE_PAIR	*trigger_args;		//!< Arguments to make available in connection pool triggers.
 
+	struct timeval	held_trigger_min;	//!< If a connection is held for less than the specified
+						//!< period, fire a trigger.
+	struct timeval	held_trigger_max;	//!< If a connection is held for longer than the specified
+						//!< period, fire a trigger.
+
 	fr_connection_create_t	create;		//!< Function used to create new connections.
 	fr_connection_alive_t	alive;		//!< Function used to check status of connections.
 	fr_connection_pool_reconnect_t reconnect;	//!< Called during connection pool reconnect.
@@ -146,6 +151,8 @@ static const CONF_PARSER connection_config[] = {
 	{ FR_CONF_OFFSET("cleanup_interval", PW_TYPE_INTEGER, fr_connection_pool_t, cleanup_interval), .dflt = "30" },
 	{ FR_CONF_OFFSET("idle_timeout", PW_TYPE_INTEGER, fr_connection_pool_t, idle_timeout), .dflt = "60" },
 	{ FR_CONF_OFFSET("connect_timeout", PW_TYPE_TIMEVAL, fr_connection_pool_t, connect_timeout), .dflt = "3.0" },
+	{ FR_CONF_OFFSET("held_trigger_min", PW_TYPE_TIMEVAL, fr_connection_pool_t, held_trigger_min), .dflt = "0.0" },
+	{ FR_CONF_OFFSET("held_trigger_max", PW_TYPE_TIMEVAL, fr_connection_pool_t, held_trigger_max), .dflt = "0.5" },
 	{ FR_CONF_OFFSET("retry_delay", PW_TYPE_INTEGER, fr_connection_pool_t, retry_delay), .dflt = "1" },
 	{ FR_CONF_OFFSET("spread", PW_TYPE_BOOLEAN, fr_connection_pool_t, spread), .dflt = "no" },
 	CONF_PARSER_TERMINATOR
@@ -1335,6 +1342,8 @@ void *fr_connection_get(fr_connection_pool_t *pool, REQUEST *request)
 void fr_connection_release(fr_connection_pool_t *pool, REQUEST *request, void *conn)
 {
 	fr_connection_t *this;
+	struct timeval	held;
+	bool trigger_min = false, trigger_max = false;
 
 	this = fr_connection_find(pool, conn);
 	if (!this) return;
@@ -1347,6 +1356,31 @@ void fr_connection_release(fr_connection_pool_t *pool, REQUEST *request, void *c
 	gettimeofday(&this->last_released, NULL);
 	pool->state.last_released = this->last_released;
 
+	/*
+	 *	This is done inside the mutex to ensure
+	 *	updates are atomic.
+	 */
+	fr_timeval_subtract(&held, &this->last_released, &this->last_reserved);
+
+	/*
+	 *	Check we've not exceeded out trigger limits
+	 */
+	if ((pool->held_trigger_min.tv_sec || pool->held_trigger_min.tv_usec) &&
+	    (fr_timeval_cmp(&held, &pool->held_trigger_min) < 0) &&
+	    (pool->state.last_held_min != this->last_released.tv_sec)) {
+	    	trigger_min = true;
+	    	pool->state.last_held_min = this->last_released.tv_sec;
+	}
+
+	if ((pool->held_trigger_max.tv_sec || pool->held_trigger_min.tv_usec) &&
+	    (fr_timeval_cmp(&held, &pool->held_trigger_max) > 0) &&
+	    (pool->state.last_held_max != this->last_released.tv_sec)) {
+	    	trigger_max = true;
+	    	pool->state.last_held_max = this->last_released.tv_sec;
+	}
+
+	fr_stats_bins(&pool->state.held_stats, &this->last_reserved, &this->last_released);
+
 	/*
 	 *	Insert the connection in the heap.
 	 *
@@ -1368,6 +1402,9 @@ void fr_connection_release(fr_connection_pool_t *pool, REQUEST *request, void *c
 	 *	connections, go manage the pool && clean some up.
 	 */
 	fr_connection_pool_check(pool);
+
+	if (trigger_min) fr_connection_trigger_exec(pool, "min");
+	if (trigger_max) fr_connection_trigger_exec(pool, "max");
 }
 
 /** Reconnect a suspected inviable connection