Rate limiting

src/main/java/de/uoc/dh/idh/autodone/config/AutodoneConfig.java

@@ -36,6 +36,12 @@ public class AutodoneConfig {
 
 	public static final int AUTODONE_THREADPOOL;
 
+	//
+
+	public static final int AUTODONE_DOWNLOADRATELIMIT;
+
+	public static final int AUTODONE_DOWNLOADTHREADPOOL;
+
 	//
 
 	static {
@@ -53,6 +59,8 @@ public class AutodoneConfig {
 		AUTODONE_PAGINATION = getEnvironment().getProperty("autodone.pagination", int.class);
 		AUTODONE_SCHEDULING = getEnvironment().getProperty("autodone.scheduling", int.class);
 		AUTODONE_THREADPOOL = getEnvironment().getProperty("autodone.threadpool", int.class);
+		AUTODONE_DOWNLOADRATELIMIT = getEnvironment().getProperty("autodone.downloadratelimit", int.class);
+		AUTODONE_DOWNLOADTHREADPOOL = getEnvironment().getProperty("autodone.downloadthreadpool", int.class);
 	}
 
 }

src/main/java/de/uoc/dh/idh/autodone/controller/ImportController.java

@@ -21,6 +21,8 @@
 import de.uoc.dh.idh.autodone.entities.GroupEntity;
 import de.uoc.dh.idh.autodone.services.GroupService;
 import de.uoc.dh.idh.autodone.services.ImportService;
+import de.uoc.dh.idh.autodone.services.MediaDownloadQueue;
+import de.uoc.dh.idh.autodone.services.MediaDownloadTask;
 import de.uoc.dh.idh.autodone.services.StatusService;
 import jakarta.servlet.http.HttpSession;
 
@@ -40,6 +42,9 @@ public class ImportController {
 	@Autowired()
 	private StatusService statusService;
 
+	@Autowired()
+    private MediaDownloadQueue mediaDownloadQueue;
+
 	//
 
 	@GetMapping()
@@ -83,6 +88,13 @@ public String post(@RequestBody() MultipartFile file, @RequestParam() Map<String
 			httpSession.removeAttribute("import");
 
 			var save = groupService.save(mapFields(form, group, FORCE));
+
+			for (var status : save.status) {
+				if (status.media.size() > 0) {
+					mediaDownloadQueue.addTask(new MediaDownloadTask(status.media.get(0), status));
+				}
+			}
+
 			return "redirect:/group?uuid=" + save.uuid;
 		}
 	}

src/main/java/de/uoc/dh/idh/autodone/services/ImportService.java

@@ -1,26 +1,16 @@
 package de.uoc.dh.idh.autodone.services;
 
-import static de.uoc.dh.idh.autodone.config.AutodoneConfig.AUTODONE_IMG_FORMAT;
-import static de.uoc.dh.idh.autodone.config.AutodoneConfig.AUTODONE_IMG_SIZE_X;
-import static de.uoc.dh.idh.autodone.config.AutodoneConfig.AUTODONE_IMG_SIZE_Y;
 import static de.uoc.dh.idh.autodone.config.AutodoneConfig.AUTODONE_IMPORT_SKIP;
 import static de.uoc.dh.idh.autodone.config.AutodoneConfig.AUTODONE_IMPORT_SNIP;
 import static de.uoc.dh.idh.autodone.utils.ObjectUtils.mapFields;
-import static java.awt.Image.SCALE_FAST;
-import static java.awt.image.BufferedImage.TYPE_INT_ARGB;
 import static java.time.Instant.now;
 import static java.time.LocalDateTime.ofInstant;
 import static java.time.ZoneOffset.UTC;
 import static java.util.Map.of;
-import static javax.imageio.ImageIO.read;
-import static javax.imageio.ImageIO.write;
 
-import java.awt.image.BufferedImage;
 import java.io.BufferedReader;
-import java.io.ByteArrayOutputStream;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.net.URL;
 import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -114,20 +104,19 @@ public StatusEntity importStatus(String line, int number) throws Exception {
 
 			if (columns.length > 3) {
 				try {
-					status.media.add(mapFields(of("status", status), importMedia(columns[3])));
-
-					if (status.media.get(0).description != null) {
-						status.exceptions.add(new ParseException("Image scaled down (4th column)", number));
-						status.media.get(0).description = null;
-					}
+					MediaEntity media = new MediaEntity();
+					media.url = columns[3];
 
 					if (columns.length > 4) {
 						if (columns[4].length() > 1500) {
 							status.exceptions.add(new ParseException("Image caption too long (5th column)", number));
 						} else {
-							status.media.get(0).description = columns[4];
+							media.description = columns[4];
 						}
 					}
+
+					status.media.add(mapFields(of("status", status), media));
+
 				} catch (Exception exception) {
 					status.exceptions.add(new ParseException("Image not usable (4th column)", number));
 				}
@@ -138,42 +127,4 @@ public StatusEntity importStatus(String line, int number) throws Exception {
 	}
 
 	//
-
-	public MediaEntity importMedia(String url) throws Exception {
-		var media = new MediaEntity();
-		var request = new URL(url).openConnection();
-
-		media.contentType = request.getContentType();
-
-		if (request.getContentLength() < 1024000) {
-			media.file = request.getInputStream().readAllBytes();
-		} else {
-			var buffer = new ByteArrayOutputStream();
-			var source = read(request.getInputStream());
-
-			var scaleX = source.getWidth();
-			var scaleY = source.getHeight();
-
-			if (scaleX > AUTODONE_IMG_SIZE_X) {
-				scaleX = AUTODONE_IMG_SIZE_X;
-				scaleY = (scaleX * source.getHeight()) / source.getWidth();
-			}
-
-			if (scaleY > AUTODONE_IMG_SIZE_Y) {
-				scaleY = AUTODONE_IMG_SIZE_Y;
-				scaleX = (scaleY * source.getWidth()) / source.getHeight();
-			}
-
-			var target = new BufferedImage(scaleX, scaleY, TYPE_INT_ARGB);
-			var scaled = source.getScaledInstance(scaleX, scaleY, SCALE_FAST);
-			target.getGraphics().drawImage(scaled, 0, 0, null, null);
-			write(target, AUTODONE_IMG_FORMAT, buffer);
-
-			media.description = "Scaled down from original";
-			media.file = buffer.toByteArray();
-		}
-
-		return media;
-	}
-
 }

src/main/java/de/uoc/dh/idh/autodone/services/MediaDownloadExecutor.java

@@ -0,0 +1,133 @@
+package de.uoc.dh.idh.autodone.services;
+
+import static de.uoc.dh.idh.autodone.config.AutodoneConfig.AUTODONE_IMG_FORMAT;
+import static de.uoc.dh.idh.autodone.config.AutodoneConfig.AUTODONE_IMG_SIZE_X;
+import static de.uoc.dh.idh.autodone.config.AutodoneConfig.AUTODONE_IMG_SIZE_Y;
+import static de.uoc.dh.idh.autodone.config.AutodoneConfig.AUTODONE_DOWNLOADRATELIMIT;
+import static de.uoc.dh.idh.autodone.config.AutodoneConfig.AUTODONE_DOWNLOADTHREADPOOL;
+
+
+import java.util.concurrent.PriorityBlockingQueue;
+
+
+import java.io.IOException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import jakarta.annotation.PostConstruct;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import static java.awt.Image.SCALE_FAST;
+import static java.awt.image.BufferedImage.TYPE_INT_ARGB;
+import static javax.imageio.ImageIO.read;
+import static javax.imageio.ImageIO.write;
+
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayOutputStream;
+import java.net.URL;
+
+import de.uoc.dh.idh.autodone.entities.MediaEntity;
+
+
+@Component
+public class MediaDownloadExecutor {
+    private final MediaDownloadQueue queue;
+    private final ExecutorService executorService;
+
+    @Autowired()
+    private MediaService mediaService;
+
+
+    @Autowired()
+    public MediaDownloadExecutor(MediaDownloadQueue queue) {
+        this.queue = queue;
+        this.executorService = Executors.newFixedThreadPool(AUTODONE_DOWNLOADTHREADPOOL);
+        System.out.println("Created executor service with threads: " + AUTODONE_DOWNLOADTHREADPOOL);
+    }
+
+    @PostConstruct
+    public void start() {
+        for (int i = 0; i < ((ThreadPoolExecutor) executorService).getCorePoolSize(); i++) {
+            executorService.submit(this::processTasks);
+        }
+    }
+
+    private void processTasks() {
+        try {
+            System.out.println("Thread started with id: " + Thread.currentThread().getId());
+            while (true) {
+                MediaDownloadTask task = queue.takeTask();
+
+                MediaEntity downloadMedia = importMedia(task.getMedia().getUrl());
+
+                MediaEntity media = mediaService.getAny(task.getMedia().getUuid());
+
+                media.file = downloadMedia.file;
+
+
+                media.contentType = downloadMedia.contentType;
+                if (downloadMedia.description != null) {
+                    media.description = media.description + " " + downloadMedia.description;
+                }
+
+                mediaService.save(media);
+
+                TimeUnit.MILLISECONDS.sleep(AUTODONE_DOWNLOADRATELIMIT);
+
+            }
+        } catch (InterruptedException e) {
+            System.out.println("Thread interrupted");
+            Thread.currentThread().interrupt();
+        }
+    }
+
+    public void stop() {
+        executorService.shutdownNow();
+    }
+
+    public MediaEntity importMedia(String url) {
+        try {
+            var media = new MediaEntity();
+            var request = new URL(url).openConnection();
+
+            media.contentType = request.getContentType();
+
+            if (request.getContentLength() < 1024000) {
+                media.file = request.getInputStream().readAllBytes();
+            } else {
+                var buffer = new ByteArrayOutputStream();
+                var source = read(request.getInputStream());
+
+                var scaleX = source.getWidth();
+                var scaleY = source.getHeight();
+
+                if (scaleX > AUTODONE_IMG_SIZE_X) {
+                    scaleX = AUTODONE_IMG_SIZE_X;
+                    scaleY = (scaleX * source.getHeight()) / source.getWidth();
+                }
+
+                if (scaleY > AUTODONE_IMG_SIZE_Y) {
+                    scaleY = AUTODONE_IMG_SIZE_Y;
+                    scaleX = (scaleY * source.getWidth()) / source.getHeight();
+                }
+
+                var target = new BufferedImage(scaleX, scaleY, TYPE_INT_ARGB);
+                var scaled = source.getScaledInstance(scaleX, scaleY, SCALE_FAST);
+                target.getGraphics().drawImage(scaled, 0, 0, null, null);
+                write(target, AUTODONE_IMG_FORMAT, buffer);
+
+                media.description = "(Scaled down from original)";
+                media.file = buffer.toByteArray();
+            }
+
+            return media;
+
+        } catch (IOException e) {
+            System.out.println("Io Exception" + e.getMessage());
+            return null;
+        }
+    }
+}

src/main/java/de/uoc/dh/idh/autodone/services/MediaDownloadQueue.java

@@ -0,0 +1,27 @@
+package de.uoc.dh.idh.autodone.services;
+
+import org.springframework.stereotype.Component;
+
+import java.util.concurrent.PriorityBlockingQueue;
+
+@Component
+public class MediaDownloadQueue {
+    private final PriorityBlockingQueue<MediaDownloadTask> queue = new PriorityBlockingQueue<>();
+
+    public void addTask(MediaDownloadTask task) {
+        queue.add(task);
+    }
+
+    public MediaDownloadTask takeTask() throws InterruptedException {
+        MediaDownloadTask task = queue.take();
+        return task;
+    }
+
+    public void clear() {
+        queue.clear();
+    }
+
+    public PriorityBlockingQueue<MediaDownloadTask> getQueue() {
+        return queue;
+    }
+}

src/main/java/de/uoc/dh/idh/autodone/services/MediaDownloadTask.java

@@ -0,0 +1,27 @@
+package de.uoc.dh.idh.autodone.services;
+
+import de.uoc.dh.idh.autodone.entities.MediaEntity;
+import de.uoc.dh.idh.autodone.entities.StatusEntity;
+
+public class MediaDownloadTask implements Comparable<MediaDownloadTask> {
+    private final MediaEntity media;
+    private final StatusEntity status;
+
+    public MediaDownloadTask(MediaEntity media, StatusEntity status) {
+        this.media = media;
+        this.status = status;
+    }
+
+    public MediaEntity getMedia() {
+        return media;
+    }
+
+    public StatusEntity getStatus() {
+        return status;
+    }
+
+    @Override
+    public int compareTo(MediaDownloadTask other) {
+        return this.status.getDate().compareTo(other.getStatus().getDate());
+    }
+}

src/main/resources/config/application.properties

@@ -26,6 +26,9 @@ autodone.pagination=20
 autodone.scheduling=30
 autodone.threadpool=5
 
+autodone.downloadratelimit=1000
+autodone.downloadthreadpool=2
+
 ############
 # MASTODON #
 ############

src/main/resources/templates/forms/media.html

@@ -43,6 +43,20 @@
             <label class="form-label">Description for the attached Media</label>
           </div>
         </div>
+        <div th:if="*{file}">
+          <label class="form-label fw-bold" for="downloadStatus">Download Status</label>
+          <div class="form-floating">
+            <p class="form-control-plaintext">File available</p>
+            <label class="form-label">File Download Status</label>
+          </div>
+        </div>
+        <div th:unless="*{file}">
+          <label class="form-label fw-bold" for="downloadStatus">Download Status</label>
+          <div class="form-floating">
+            <p class="form-control-plaintext">File not available</p>
+            <label class="form-label">File Download Status</label>
+          </div>
+        </div>
       </div>
     </div>
   </div>