RANGER-4837: Fix flaky test by iterating over datewise subfolders for…

… counting orc logs (#337)
apache · Jul 15, 2024 · c9361dc · c9361dc
1 parent bc091c5
commit c9361dc
Showing 1 changed file with 39 additions and 21 deletions.
diff --git a/security-admin/src/test/java/org/apache/ranger/audit/TestAuditQueue.java b/security-admin/src/test/java/org/apache/ranger/audit/TestAuditQueue.java
@@ -27,6 +27,7 @@
 import java.nio.file.Paths;
 import java.time.LocalDate;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Properties;
 
@@ -735,7 +736,7 @@ public void testAuditFileQueueSpoolORC(){
 		try {
 			Thread.sleep(40000);
 		} catch (InterruptedException e) {
-			System.out.println(e);
+			logger.error(e.getMessage());
 		}
 		queue.waitToComplete();
 		assertTrue("File created", logFile.exists());
@@ -745,7 +746,9 @@ public void testAuditFileQueueSpoolORC(){
 	@Test
 	public void testAuditFileQueueSpoolORCRollover(){
 		String appType = "test";
-		int messageToSend = 100000;
+		int messageToSend = 1000;
+		int preRolloverMessagesCount = (int)(0.8*messageToSend);
+		int postRolloverMessagesCount = messageToSend - preRolloverMessagesCount;
 		String spoolFolderName = "target/spool";
 		String logFolderName = "target/testAuditFileQueueSpoolORC";
 		try {
@@ -760,11 +763,7 @@ public void testAuditFileQueueSpoolORCRollover(){
 		}
 		assertTrue(Files.notExists(Paths.get(spoolFolderName)));
 		assertTrue(Files.notExists(Paths.get(logFolderName)));
-		String subdir = appType + "/" + LocalDate.now().toString().replace("-","");
 		File logFolder = new File(logFolderName);
-		File logSubfolder = new File(logFolder, subdir);
-		String logFileName = "test_ranger_audit.orc";
-		File logFile = new File(logSubfolder, logFileName);
 		Properties props = new Properties();
 		props.put(AuditProviderFactory.AUDIT_IS_ENABLED_PROP, "true");
 		String hdfsPropPrefix = AuditProviderFactory.AUDIT_DEST_BASE + ".hdfs";
@@ -774,6 +773,7 @@ public void testAuditFileQueueSpoolORCRollover(){
 				"%app-type%_ranger_audit.orc");
 		String orcPrefix = hdfsPropPrefix + ".orc";
 		props.put(orcPrefix+".compression","snappy");
+		//large numbers used here to ensure that file rollover happens because of file rollover seconds and not orc file /related props
 		props.put(orcPrefix+".buffersize",""+100000000000000L);
 		props.put(orcPrefix+".stripesize",""+100000000000000L);
 		props.put(hdfsPropPrefix + ".batch.queuetype","filequeue");
@@ -782,35 +782,53 @@ public void testAuditFileQueueSpoolORCRollover(){
 		String fileSpoolPrefix = filequeuePrefix + ".filespool";
 		props.put(fileSpoolPrefix+".dir",spoolFolderName);
 		props.put(fileSpoolPrefix+".buffer.size",""+100000000000000L);
-		props.put(fileSpoolPrefix+".file.rollover.sec",""+300);
+		props.put(fileSpoolPrefix+".file.rollover.sec",""+5);
 		AuditProviderFactory factory = new AuditProviderFactory();
 		factory.init(props, appType);
 		AuditHandler queue = factory.getAuditProvider();
-		for (int i = 0; i < messageToSend; i++) {
+		for (int i = 0; i < preRolloverMessagesCount; i++) {
 			queue.log(createEvent());
 			try {
-				Thread.sleep(5);
+				Thread.sleep(10);
 			} catch (InterruptedException e) {
-				System.out.println(e);
+				logger.error(e.getMessage());
 			}
 		}
+		//wait for rollover to happen
 		try {
-			Thread.sleep(40000);
+			Thread.sleep(10000);
 		} catch (InterruptedException e) {
-			System.out.println(e);
+			logger.error(e.getMessage());
+		}
+		//send some more logs
+		for (int i = 0; i < postRolloverMessagesCount; i++) {
+			queue.log(createEvent());
+			try {
+				Thread.sleep(10);
+			} catch (InterruptedException e) {
+				logger.error(e.getMessage());
+			}
 		}
 		queue.waitToComplete();
-		assertTrue("File created", logFile.exists());
-		File[] listOfFiles = logSubfolder.listFiles();
 		int totalLogsOrc = 0;
-		if (listOfFiles != null){
-			for(File f : listOfFiles){
-				if (f.getName().endsWith(".orc")){
-					totalLogsOrc += getOrcFileRowCount(f.getPath());
+		File appSubFolder = new File(logFolder,appType);
+		String[] datewiseSubfolders = appSubFolder.list();
+		logger.info("subfolder list="+ Arrays.toString(datewiseSubfolders));
+		if (datewiseSubfolders != null) {
+			for (String dateSubfolder : datewiseSubfolders){
+				File logSubfolder = new File(appSubFolder, dateSubfolder);
+				File[] listOfFiles = logSubfolder.listFiles();
+				if (listOfFiles != null){
+					for(File f : listOfFiles){
+						if (f.getName().endsWith(".orc")){
+							logger.info("Reading orc file:"+f.getName());
+							totalLogsOrc += getOrcFileRowCount(f.getPath());
+						}
+					}
 				}
 			}
 		}
-		System.out.println("Number of logs in orc="+totalLogsOrc);
+		logger.info("Number of logs in orc="+totalLogsOrc);
 		long totalLogsArchive = 0;
 
 		try {
@@ -826,7 +844,7 @@ public void testAuditFileQueueSpoolORCRollover(){
 		} catch (IOException e) {
 			throw new RuntimeException(e);
 		}
-		System.out.println("Number of logs in archive:"+totalLogsArchive);
+		logger.info("Number of logs in archive:"+totalLogsArchive);
 		assertEquals(totalLogsOrc, totalLogsArchive);
 
 		long notYetConvertedToORCLogsCount = 0;
@@ -848,7 +866,7 @@ public void testAuditFileQueueSpoolORCRollover(){
 		catch (IOException e){
 			throw new RuntimeException(e);
 		}
-		System.out.println("Number of logs not converted to ORC:"+notYetConvertedToORCLogsCount);
+		logger.info("Number of logs not converted to ORC:"+notYetConvertedToORCLogsCount);
 		assertEquals(messageToSend, notYetConvertedToORCLogsCount+totalLogsArchive);
 	}