Autofill HDFS URI in DeltaTableDialog (#737)

* Autofill HDFS URI in DeltaTableDialog * Use login shell to run commands on remote hosts so that that hilview processes can pick up environment variables. Co-authored-by: Bin Wang <[email protected]>
vmware-archive · Aug 9, 2021 · 11dbb58 · 11dbb58
1 parent 27f530d
commit 11dbb58
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 6 deletions.
diff --git a/bin/hillviewCommon.py b/bin/hillviewCommon.py
@@ -74,7 +74,7 @@ def run_remote_shell_command(self, command, verbose=True):
             logger.info(message)
         f.close()
 
-        command = "ssh " + self.uh() + " bash -s < " + file.name
+        command = "ssh " + self.uh() + " bash -l -s < " + file.name
         exitcode = subprocess.call(command, shell=True)
         if exitcode != 0:
             error = "Exit code returned: " + str(exitcode)

diff --git a/web/src/main/java/org/hillview/targets/InitialObjectTarget.java b/web/src/main/java/org/hillview/targets/InitialObjectTarget.java
@@ -17,7 +17,10 @@
 
 package org.hillview.targets;
 
+import com.google.gson.JsonPrimitive;
 import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.hillview.*;
 import org.hillview.dataset.RemoteDataSet;
 import org.hillview.dataset.api.*;
@@ -106,6 +109,34 @@ public void getUIConfig(RpcRequest request, RpcRequestContext context) {
         this.returnResult(Configuration.instance.getAsJson(), request, context);
     }
 
+    @HillviewRpc
+    public void getHDFSUri(RpcRequest request, RpcRequestContext context) {
+        final String hadoopHome = System.getenv("HADOOP_HOME");
+        String hdfsUri;
+
+        if (hadoopHome == null) {
+            HillviewLogger.instance.warn("Could not find HADOOP_HOME");
+            hdfsUri = "";
+        } else {
+            try {
+                final Path hadoopConfigDir = new Path(Paths.get(hadoopHome, "etc", "hadoop").toUri());
+                org.apache.hadoop.conf.Configuration configuration
+                        = new org.apache.hadoop.conf.Configuration();
+                configuration.addResource(new Path(hadoopConfigDir, "core-site.xml"));
+                configuration.addResource(new Path(hadoopConfigDir, "hdfs-site.xml"));
+                hdfsUri = FileSystem.getDefaultUri(configuration).toString();
+            } catch (Exception exception) {
+                HillviewLogger.instance.warn(
+                        "Found HADOOP_HOME but failed to load config",
+                        "HADOOP_HOME: {0}, error message: {1}",
+                        hadoopHome, exception.getMessage());
+                hdfsUri = "";
+            }
+        }
+
+        this.returnResult(new JsonValue(new JsonPrimitive(hdfsUri)), request, context);
+    }
+
     @HillviewRpc
     public void openingBookmark(RpcRequest request, RpcRequestContext context) throws IOException {
         String bookmarkFile = request.parseArgs(String.class);

diff --git a/web/src/main/webapp/loadView.ts b/web/src/main/webapp/loadView.ts
@@ -152,9 +152,13 @@ export class LoadView extends RemoteObject implements IDataView {
             }, {
                 text: "Delta table...",
                 action: () => {
-                    const dialog = new DeltaTableDialog();
-                    dialog.setAction(() => this.init.loadDeltaTable(dialog.getDeltaTableDescription(), this.page))
-                    dialog.show();
+                    const rr = this.createStreamingRpcRequest<string>("getHDFSUri", null);
+                    const observer = new GenericReceiver("get HDFS URI", this.page, rr, hdfsUri => {
+                        const dialog = new DeltaTableDialog(hdfsUri);
+                        dialog.setAction(() => this.init.loadDeltaTable(dialog.getDeltaTableDescription(), this.page))
+                        dialog.show();
+                    });
+                    rr.invoke(observer);
                 },
                 help: "A delta lake table"
             });
@@ -482,7 +486,7 @@ class OrcFileDialog extends Dialog {
 }
 
 class DeltaTableDialog extends Dialog {
-    constructor() {
+    constructor(hdfsUri: string) {
         super(
             "Load a Delta Lake table",
             "Load a Delta Lake table from local filesystem, hdfs, or s3"
@@ -491,7 +495,7 @@ class DeltaTableDialog extends Dialog {
             "deltaTablePath",
             "Delta table path",
             FieldKind.String,
-            "/delta-table",
+            hdfsUri,
             "Path to the delta table to load."
         );
         path.required = true;