RANGER-4565: Enhance Ranger's performance tracing module to optionall…

…y collect statistical information
apache · Nov 28, 2023 · 04c93b3 · 04c93b3
1 parent 6c26045
commit 04c93b3
Show file tree

Hide file tree

Showing 4 changed files with 71 additions and 18 deletions.
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
@@ -86,6 +86,12 @@ public RangerBasePlugin(RangerPluginConfig pluginConfig) {
 		this.pluginConfig  = pluginConfig;
 		this.pluginContext = new RangerPluginContext(pluginConfig);
 
+		boolean usePerfDataRecorder  = pluginConfig.getBoolean("ranger.perf.aggregate.data", false);
+		int     perfDataDumpInterval = pluginConfig.getInt("ranger.perf.aggregate.data.dump.interval", 0);
+		boolean usePerfDataLock  = pluginConfig.getBoolean("ranger.perf.aggregate.data.lock.enabled", false);
+
+		PerfDataRecorder.initialize(usePerfDataRecorder, perfDataDumpInterval, usePerfDataLock, null);
+
 		Set<String> superUsers         = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".super.users"));
 		Set<String> superGroups        = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".super.groups"));
 		Set<String> auditExcludeUsers  = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".audit.exclude.users"));

diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
@@ -37,13 +37,27 @@ public class PerfDataRecorder {
 	private static final Logger PERF = RangerPerfTracer.getPerfLogger(PerfDataRecorder.class);
 
 	private static volatile PerfDataRecorder instance;
-	private Map<String, PerfStatistic> perfStatistics = new HashMap<>();
+	final private Map<String, PerfStatistic> perfStatistics = Collections.synchronizedMap(new HashMap<>());
+	private RangerReadWriteLock lock = null;
 
 	public static void initialize(List<String> names) {
-		if (instance == null) {
-			synchronized (PerfDataRecorder.class) {
-				if (instance == null) {
-					instance = new PerfDataRecorder(names);
+		initialize(true, 0, false, names);
+	}
+
+	public static void initialize(final boolean useRecorder, final int collectionIntervalInSeconds, final boolean usePerfDataLock, List<String> names) {
+		if (useRecorder) {
+			if (instance == null) {
+				synchronized (PerfDataRecorder.class) {
+					if (instance == null) {
+						instance = new PerfDataRecorder(names);
+						instance.lock = new RangerReadWriteLock(usePerfDataLock);
+						if (collectionIntervalInSeconds > 0) {
+							Thread statDumper = new StatisticsDumper(collectionIntervalInSeconds);
+							statDumper.setName("Perf-Statistics-Dumper");
+							statDumper.setDaemon(true);
+							statDumper.start();
+						}
+					}
 				}
 			}
 		}
@@ -61,7 +75,9 @@ public static void printStatistics() {
 
 	public static void clearStatistics() {
 		if (instance != null) {
-			instance.clear();
+			try (RangerReadWriteLock.RangerLock writeLock = instance.lock.getWriteLock()) {
+				instance.clear();
+			}
 		}
 	}
 
@@ -72,7 +88,11 @@ public static void recordStatistic(String tag, long cpuTime, long elapsedTime) {
 	}
 
 	private void dumpStatistics() {
-		List<String> tags = new ArrayList<>(perfStatistics.keySet());
+		List<String> tags;
+
+		try (RangerReadWriteLock.RangerLock readLock = lock.getReadLock()) {
+			tags = new ArrayList<>(perfStatistics.keySet());
+		}
 
 		Collections.sort(tags);
 
@@ -111,20 +131,24 @@ private void clear() {
 	}
 
 	private void record(String tag, long cpuTime, long elapsedTime) {
-		PerfStatistic perfStatistic = perfStatistics.get(tag);
+		try (RangerReadWriteLock.RangerLock writeLock = lock.getWriteLock()) {
+
+			PerfStatistic perfStatistic = perfStatistics.get(tag);
 
-		if (perfStatistic == null) {
-			synchronized (PerfDataRecorder.class) {
-				perfStatistic = perfStatistics.get(tag);
+			if (perfStatistic == null) {
+				synchronized (PerfDataRecorder.class) {
+					perfStatistic = perfStatistics.get(tag);
 
-				if(perfStatistic == null) {
-					perfStatistic = new PerfStatistic();
-					perfStatistics.put(tag, perfStatistic);
+					if (perfStatistic == null) {
+						perfStatistic = new PerfStatistic();
+						perfStatistics.put(tag, perfStatistic);
+					}
 				}
 			}
-		}
 
-		perfStatistic.addPerfDataItem(cpuTime, elapsedTime);
+			perfStatistic.addPerfDataItem(cpuTime, elapsedTime);
+
+		}
 	}
 
 	private PerfDataRecorder(List<String> names) {
@@ -206,4 +230,25 @@ public long getMaxTimeSpent() {
 			return maxTimeSpent.get();
 		}
 	}
+
+	private static class StatisticsDumper extends Thread {
+		final int collectionIntervalInSeconds;
+		StatisticsDumper(final int collectionIntervalInSeconds) {
+			this.collectionIntervalInSeconds = collectionIntervalInSeconds;
+		}
+		@Override
+		public void run() {
+			while (true) {
+				try {
+					sleep(collectionIntervalInSeconds * 1000);
+					printStatistics();
+					clearStatistics();
+				} catch (InterruptedException exception) {
+					printStatistics();
+					LOG.warn("Thread[" + this.getName() + "] was interrupted. Returning from thread. Performance statistics will NOT be dumped periodically!!");
+					break;
+				}
+			}
+		}
+	}
 }
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java
@@ -37,7 +37,7 @@ public void log() {
 		long reportingThreshold = threadInfo == null ? 0L : (1000000/1000 - 1); // just about a microsecond
 
 		if (elapsedTime > reportingThreshold) {
-			logger.debug("[PERF]:" + (threadInfo != null ? threadInfo.getThreadName() : "") + ":" + tag + data + ":" + getElapsedCpuTime() + ":" + getElapsedUserTime());
+			PerfDataRecorder.recordStatistic(tag, (getElapsedCpuTime()+500)/1000, (getElapsedUserTime() + 500)/1000);
 		}
 	}
 

diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java
@@ -55,7 +55,9 @@ public static RangerPerfTracer getPerfTracer(Logger logger, String tag) {
 			int indexOfTagEndMarker = StringUtils.indexOf(tag, tagEndMarker);
 			if (indexOfTagEndMarker != -1) {
 				realTag = StringUtils.substring(tag, 0, indexOfTagEndMarker);
-				data = StringUtils.substring(tag, indexOfTagEndMarker);
+				if (!PerfDataRecorder.collectStatistics()) {
+					data = StringUtils.substring(tag, indexOfTagEndMarker);
+				}
 			} else {
 				realTag = tag;
 			}