Merge branch 'master' of github.com:internetarchive/heritrix3

internetarchive · Jun 21, 2021 · 33a01a3 · 33a01a3
2 parents 6d0ae8f + 31e54cc
commit 33a01a3
Show file tree

Hide file tree

Showing 5 changed files with 32 additions and 225 deletions.
diff --git a/commons/pom.xml b/commons/pom.xml
@@ -121,12 +121,6 @@
 			<artifactId>junit</artifactId>
 			<scope>compile</scope>
 		</dependency>
-		<dependency>
-			<groupId>it.unimi.dsi</groupId>
-			<artifactId>mg4j</artifactId>
-			<version>1.0.1</version>
-			<scope>compile</scope>
-		</dependency>
 		<dependency>
 			<groupId>com.anotherbigidea</groupId>
 			<artifactId>javaswf</artifactId>

diff --git a/commons/src/main/java/org/archive/io/CrawlerJournal.java b/commons/src/main/java/org/archive/io/CrawlerJournal.java
@@ -19,7 +19,6 @@
 package org.archive.io;
 
 import it.unimi.dsi.fastutil.io.FastBufferedOutputStream;
-import it.unimi.dsi.mg4j.util.MutableString;
 
 import java.io.Closeable;
 import java.io.File;
@@ -121,24 +120,6 @@ public synchronized void writeLine(String... strs) {
         }
     }
 
-    /**
-     * Write a line. 
-     * 
-     * @param mstring MutableString to write
-     */
-    public synchronized void writeLine(MutableString mstring) {
-        if (this.out == null) {
-            return;
-        }
-        try {
-            mstring.write(out);
-            this.out.write("\n");
-            noteLine();
-        } catch (IOException e) {
-            LOGGER.log(Level.SEVERE,"problem writing journal line: "+mstring, e);
-        }
-    }
-
     /**
      * Count and note a line
      * 

diff --git a/dist/src/main/licenses/mg4j-1.0.1.LICENSE b/dist/src/main/licenses/mg4j-1.0.1.LICENSE
diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java
@@ -180,9 +180,17 @@ protected Representation post(Representation entity, Variant variant) throws Res
                     cancel = true; 
                 }
             }
-            if(!cancel) {
-                System.exit(0); 
-            }
+			if (!cancel) {
+				Flash.addFlash(getResponse(), "Shutting down ... bye");
+				new Thread(() -> {
+					try {
+						Thread.sleep(1000);
+					} catch (InterruptedException e) {
+						Thread.currentThread().interrupt();
+					}
+					System.exit(0);
+				}).start();
+			}
         } else if ("gc".equals(action)) {
             System.gc();
         }

diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java
@@ -18,13 +18,9 @@
  */
 package org.archive.modules.fetcher;
 
-import it.unimi.dsi.mg4j.util.MutableString;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.Reader;
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
@@ -126,36 +122,29 @@ public void saveCookies(String saveCookiesFile) {
             return;
         }
 
-        FileOutputStream out = null;
-        try {
-            out = new FileOutputStream(new File(saveCookiesFile));
+        try (BufferedWriter out = Files.newBufferedWriter(Paths.get(saveCookiesFile))) {
             String tab ="\t";
-            out.write("# Heritrix Cookie File\n".getBytes());
-            out.write("# This file is the Netscape cookies.txt format\n\n".getBytes());
+            out.write("# Heritrix Cookie File\n");
+            out.write("# This file is the Netscape cookies.txt format\n\n");
             for (Cookie cookie: new ArrayList<Cookie>(getCookies())) {
-                // Guess an initial size
-                MutableString line = new MutableString(1024 * 2);
-                line.append(cookie.getDomain());
-                line.append(tab);
-                // XXX line.append(cookie.isDomainAttributeSpecified() ? "TRUE" : "FALSE");
-                line.append("TRUE");
-                line.append(tab);
-                line.append(cookie.getPath() != null ? cookie.getPath() : "/");
-                line.append(tab);
-                line.append(cookie.isSecure() ? "TRUE" : "FALSE");
-                line.append(tab);
-                line.append(cookie.getExpiryDate() != null ? cookie.getExpiryDate().getTime() / 1000 : -1);
-                line.append(tab);
-                line.append(cookie.getName());
-                line.append(tab);
-                line.append(cookie.getValue() != null ? cookie.getValue() : "");
-                line.append("\n");
-                out.write(line.toString().getBytes());
+                out.write(cookie.getDomain());
+                out.write(tab);
+                // XXX out.write(cookie.isDomainAttributeSpecified() ? "TRUE" : "FALSE");
+                out.write("TRUE");
+                out.write(tab);
+                out.write(cookie.getPath() != null ? cookie.getPath() : "/");
+                out.write(tab);
+                out.write(cookie.isSecure() ? "TRUE" : "FALSE");
+                out.write(tab);
+                out.write(Long.toString(cookie.getExpiryDate() != null ? cookie.getExpiryDate().getTime() / 1000 : -1));
+                out.write(tab);
+                out.write(cookie.getName());
+                out.write(tab);
+                out.write(cookie.getValue() != null ? cookie.getValue() : "");
+                out.write("\n");
             }
         } catch (IOException e) {
             logger.log(Level.SEVERE, "Unable to write " + saveCookiesFile, e);
-        } finally {
-            IOUtils.closeQuietly(out);
         }
     }