diff --git a/.gitignore b/.gitignore index 4b7d1d6a1..e47cdae58 100644 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,7 @@ repository/*.jar # data which is too big to put into git data/ontime/On_Time_On_Time* +data/ontime_big/*.gz data/ontime/2016_*.csv data/ontime/*.orc data/ontime_orc/* diff --git a/bin/deploy-greenplum.py b/bin/deploy-greenplum.py index b48b2a5b5..845105a39 100755 --- a/bin/deploy-greenplum.py +++ b/bin/deploy-greenplum.py @@ -23,6 +23,7 @@ from argparse import ArgumentParser from jproperties import Properties import os +import tempfile from hillviewCommon import ClusterConfiguration, get_config, get_logger, execute_command def main(): @@ -34,8 +35,8 @@ def main(): execute_command("./package-binaries.sh") web = config.get_webserver() web.copy_file_to_remote("../hillview-bin.zip", ".", "") - web.copy_file_to_remote("config-greenplum.json", ".", "") web.run_remote_shell_command("unzip -o hillview-bin.zip") + web.copy_file_to_remote("config-greenplum.json", "bin", "") web.run_remote_shell_command("cd bin; ./upload-data.py -d . -s dump-greenplum.sh config-greenplum.json") web.run_remote_shell_command("cd bin; ./redeploy.sh -s config-greenplum.json") web.copy_file_to_remote("../repository/PROGRESS_DATADIRECT_JDBC_DRIVER_PIVOTAL_GREENPLUM_5.1.4.000275.jar", @@ -45,10 +46,12 @@ def main(): p = Properties() p.load(f, "utf-8") p["greenplumDumpScript"] = config.service_folder + "/dump-greenplum.sh" - with open("hillview.properties", "wb") as f: - p.store(f, encoding="utf-8") - web.copy_file_to_remote("hillview.properties", config.service_folder, "") - os.remove("hillview.properties") + p["hideDemoMenu"] = "true" + tmp = tempfile.NamedTemporaryFile(mode="w", delete=False) + p.store(tmp, encoding="utf-8") + tmp.close() + web.copy_file_to_remote(tmp.name, config.service_folder + "/hillview.properties", "") + os.remove(tmp.name) if __name__ == "__main__": main() diff --git a/bin/dump-greenplum.sh b/bin/dump-greenplum.sh index b7af229b4..82a454c60 100644 --- a/bin/dump-greenplum.sh +++ b/bin/dump-greenplum.sh @@ -25,4 +25,5 @@ DIR=$1 PREFIX="file" mkdir -p ${DIR} || exit 1 -echo "$(${DIR}/${PREFIX}${GP_SEGMENT_ID} \ No newline at end of file +#cat ${DIR}/${PREFIX}${GP_SEGMENT_ID} +split -l 500000 -a 3 - ${DIR}/${PREFIX}${GP_SEGMENT_ID} { + + private final Schema schema; + + public LoadCsvColumnsSketch(Schema schema) { + this.schema = schema; + } + + @Nullable + @Override + public ControlMessage.StatusList create(@Nullable ITable data) { + HillviewLogger.instance.info("Loading CSV columns for table", + "Columns are {0}", this.schema.toString()); + Converters.checkNull(data); + CsvFileLoader.Config config = new CsvFileLoader.Config(); + config.hasHeaderRow = true; + CsvFileLoader loader = new CsvFileLoader( + // The data will be in the same source file (data.getSourceFile()) which was used + // initially to load the table. + Converters.checkNull(data.getSourceFile()), config, new LazySchema(this.schema)); + ITable loaded = loader.load(); + Converters.checkNull(loaded); + for (String c: this.schema.getColumnNames()) { + IColumn ld = loaded.getLoadedColumn(c); + LazyColumn lc = data.getColumn(c).as(LazyColumn.class); + Converters.checkNull(lc); + if (lc.sizeInRows() != ld.sizeInRows()) + throw new RuntimeException("Loaded column has different size from original column:" + + " file=" + data.getSourceFile() + + " loaded=" + ld.toString() + " size=" + ld.sizeInRows() + + " original=" + lc.toString() + " size=" + lc.sizeInRows()); + Converters.checkNull(lc).setData(ld); + } + return new ControlMessage.StatusList(new ControlMessage.Status("OK")); + } +} diff --git a/platform/src/main/java/org/hillview/storage/FileSetDescription.java b/platform/src/main/java/org/hillview/storage/FileSetDescription.java index 4ce91948c..b57a2d311 100644 --- a/platform/src/main/java/org/hillview/storage/FileSetDescription.java +++ b/platform/src/main/java/org/hillview/storage/FileSetDescription.java @@ -68,6 +68,7 @@ public class FileSetDescription implements IJson { public String cookie = null; /** * Used for testing: allows reading the same data multiple times. + * 0 is the same as 1. */ public int repeat = 1; /** @@ -84,6 +85,11 @@ public class FileSetDescription implements IJson { * useful for temporary files. */ public boolean deleteAfterLoading; + /** + * Actual name of the dataset. + */ + @Nullable + public String name; @SuppressWarnings("unused") public String getBasename() { @@ -118,24 +124,35 @@ class FileReference implements IFileReference { public ITable load() { TextFileLoader loader; switch (FileSetDescription.this.fileKind) { - case "csv": + case "lazycsv": + /* { + For now treated as equivalent to CSV. This is used for the + csv files dumped by the greenplum database. + + CsvFileLoader.Config config = new CsvFileLoader.Config(); + config.allowFewerColumns = true; + config.hasHeaderRow = FileSetDescription.this.headerRow; + loader = new LazyCsvFileLoader( + this.pathname, config, FileSetDescription.this.getSchema()); + break; + } + */ + case "csv": { CsvFileLoader.Config config = new CsvFileLoader.Config(); config.allowFewerColumns = true; config.hasHeaderRow = FileSetDescription.this.headerRow; loader = new CsvFileLoader( this.pathname, config, FileSetDescription.this.getSchema()); break; + } case "orc": - loader = new OrcFileLoader( - this.pathname, FileSetDescription.this.getSchema(), true); + loader = new OrcFileLoader(this.pathname, FileSetDescription.this.getSchema(), true); break; case "parquet": - loader = new ParquetFileLoader( - this.pathname, true); + loader = new ParquetFileLoader(this.pathname, true); break; case "json": - loader = new JsonFileLoader( - this.pathname, FileSetDescription.this.getSchema()); + loader = new JsonFileLoader(this.pathname, FileSetDescription.this.getSchema()); break; case "hillviewlog": loader = new HillviewLogs.LogFileLoader(this.pathname); diff --git a/platform/src/main/java/org/hillview/storage/LazyCsvFileLoader.java b/platform/src/main/java/org/hillview/storage/LazyCsvFileLoader.java new file mode 100644 index 000000000..c09b91746 --- /dev/null +++ b/platform/src/main/java/org/hillview/storage/LazyCsvFileLoader.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2020 VMware Inc. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.hillview.storage; + +import org.hillview.table.ColumnDescription; +import org.hillview.table.LazySchema; +import org.hillview.table.Schema; +import org.hillview.table.Table; +import org.hillview.table.api.IColumn; +import org.hillview.table.api.ITable; +import org.hillview.table.columns.LazyColumn; +import org.hillview.utils.Converters; +import org.hillview.utils.HillviewLogger; + +import java.util.List; + +/** + * This is a special form of the CsvFileLoader, which only loads the first column of the + * schema when invoked. + * The other columns will be loaded later, as invoked explicitly. + */ +public class LazyCsvFileLoader extends TextFileLoader { + private final LazySchema schema; + CsvFileLoader loader; + + public LazyCsvFileLoader(String path, CsvFileLoader.Config configuration, LazySchema schema) { + super(path); + this.schema = schema; + this.allowFewerColumns = configuration.allowFewerColumns; + if (this.schema.isNull()) + throw new RuntimeException("Schema guessing not supported for lazy csv loading"); + Schema firstColumn = new Schema(Converters.checkNull( + this.schema.getSchema()).getColumnDescriptions().subList(0, 1)); + this.loader = new CsvFileLoader(path, configuration, new LazySchema(firstColumn)); + } + + @Override + public void prepareLoading() { + this.loader.prepareLoading(); + } + + public ITable loadFragment(int maxRows, boolean skip) { + ITable table = this.loader.loadFragment(maxRows, skip); + int rowCount = table.getNumOfRows(); + Schema schema = this.schema.getSchema(); + List desc = Converters.checkNull(schema).getColumnDescriptions(); + Table result = Table.createLazyTable(desc, rowCount, this.filename, new NoLoader()); + String firstColName = this.schema.getSchema().getColumnNames().get(0); + IColumn col0 = table.getLoadedColumn(firstColName); + LazyColumn fc = result.getColumn(firstColName).as(LazyColumn.class); + Converters.checkNull(fc).setData(col0); + return result; + } + + @Override + public void endLoading() { + this.loader.endLoading(); + } +} diff --git a/platform/src/main/java/org/hillview/storage/NoLoader.java b/platform/src/main/java/org/hillview/storage/NoLoader.java new file mode 100644 index 000000000..6b9b68dc5 --- /dev/null +++ b/platform/src/main/java/org/hillview/storage/NoLoader.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2020 VMware Inc. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.hillview.storage; + +import org.hillview.table.api.IColumn; +import org.hillview.table.api.IColumnLoader; + +import java.util.List; + +public class NoLoader implements IColumnLoader { + @Override + public List loadColumns(List names) { + throw new RuntimeException("Cannot load columns"); + } +} diff --git a/platform/src/main/java/org/hillview/storage/jdbc/JdbcDatabase.java b/platform/src/main/java/org/hillview/storage/jdbc/JdbcDatabase.java index 0c0342019..d6b4d2f1a 100644 --- a/platform/src/main/java/org/hillview/storage/jdbc/JdbcDatabase.java +++ b/platform/src/main/java/org/hillview/storage/jdbc/JdbcDatabase.java @@ -43,8 +43,10 @@ public class JdbcDatabase { private final JdbcConnection conn; @Nullable private Connection connection; + public final JdbcConnectionInformation connInfo; public JdbcDatabase(final JdbcConnectionInformation connInfo) { + this.connInfo = connInfo; this.conn = JdbcConnection.create(connInfo); this.connection = null; } @@ -197,6 +199,50 @@ public JsonGroups histogram( return JsonGroups.fromArray(data, nulls); } + static class ColumnInfo { + public final String name; + public final String sqlType; + + ColumnInfo(String name, String sqlType) { + this.name = name; + this.sqlType = sqlType; + } + + public String toString() { + return this.name + " " + this.sqlType; + } + } + + public static String sqlType(ContentsKind kind) { + String type = null; + switch (kind) { + case None: + type = "varchar(1)"; + break; + case Json: + case String: + type = "varchar"; + break; + case LocalDate: + case Date: + type = "timestamp"; + break; + case Integer: + type = "int"; + break; + case Double: + type = "double precision"; + break; + case Time: + type = "time"; + break; + case Interval: + case Duration: + throw new RuntimeException("Unsupported SQL type"); + } + return type; + } + public JsonGroups> histogram2D(ColumnDescription cd0, ColumnDescription cd1, IHistogramBuckets buckets0, IHistogramBuckets buckets1, diff --git a/platform/src/main/java/org/hillview/table/columns/LazyColumn.java b/platform/src/main/java/org/hillview/table/columns/LazyColumn.java index 4ed9c1c8d..b8694132b 100644 --- a/platform/src/main/java/org/hillview/table/columns/LazyColumn.java +++ b/platform/src/main/java/org/hillview/table/columns/LazyColumn.java @@ -114,8 +114,12 @@ synchronized private IColumn ensureLoaded() { List loaded = this.loader.loadColumns(toLoad); if (loaded.size() != 1) throw new RuntimeException("Expected 1 column to be loaded, not " + loaded.size()); - this.data = loaded.get(0); + this.setData(loaded.get(0)); assert this.data != null; return this.data; } + + synchronized public void setData(IColumn data) { + this.data = data; + } } diff --git a/platform/src/main/java/org/hillview/utils/UIConfig.java b/platform/src/main/java/org/hillview/utils/UIConfig.java new file mode 100644 index 000000000..f564d345e --- /dev/null +++ b/platform/src/main/java/org/hillview/utils/UIConfig.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2020 VMware Inc. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.hillview.utils; + +import org.hillview.dataset.api.IJsonSketchResult; + +/** + * Parameters controlling the UI display of Hillview. + * These are read from hillview.properties with the same names. + * Same as the TypeScript class with the same name. + */ +public class UIConfig implements IJsonSketchResult { + static final long serialVersionUID = 1; + + public UIConfig() {} + + public boolean enableSaveAs; + public boolean localDbMenu; + public boolean showTestMenu; + public boolean enableManagement; + public boolean privateIsCsv; + public boolean hideSuggestions; + public boolean hideDemoMenu; +} diff --git a/uiconfig.json b/uiconfig.json deleted file mode 100644 index 11ac3c445..000000000 --- a/uiconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "enableSaveAs": true, - "localDbMenu": true, - "showTestMenu": true, - "enableManagement": true, - "privateIsCsv": true, - "hideSuggestions": true -} diff --git a/web/src/main/java/org/hillview/Configuration.java b/web/src/main/java/org/hillview/Configuration.java new file mode 100644 index 000000000..066284956 --- /dev/null +++ b/web/src/main/java/org/hillview/Configuration.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2020 VMware Inc. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.hillview; + +import org.hillview.utils.HillviewLogger; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Properties; + +/** + * This class manages configuration of the global root node of a Hillview deployment. + */ +public class Configuration { + /** + * This file contains the global properties that control hillview. + * This file is read by the root node. + */ + static final String propertiesFile = "hillview.properties"; + + public static final Configuration instance = new Configuration(); + + // Global application properties + public final Properties properties; + + private Configuration() { + this.properties = new Properties(); + try (FileInputStream prop = new FileInputStream(propertiesFile)) { + this.properties.load(prop); + } catch (FileNotFoundException ex) { + HillviewLogger.instance.info("No properties file found", "{0}", propertiesFile); + } catch (IOException ex) { + HillviewLogger.instance.error("Error while loading properties from file", ex); + } + } + + public String getProperty(String propertyName, String defaultValue) { + return this.properties.getProperty(propertyName, defaultValue); + } + + public String getGreenplumDumpScript() { + return this.getProperty( + // The dump-greenplum.sh script will write its stdin to the specified file + "greenplumDumpScript", "/home/gpadmin/hillview/dump-greenplum.sh"); + } + + public String getGreenplumDumpDirectory() { + return this.getProperty("greenplumDumpDirectory", "/tmp"); + } + + public boolean getBooleanProperty(String prop) { + String value = this.getProperty(prop, "false"); + return !value.trim().toLowerCase().equals("false"); + } +} diff --git a/web/src/main/java/org/hillview/RpcObjectManager.java b/web/src/main/java/org/hillview/RpcObjectManager.java index 313bee3ef..06675a8bc 100644 --- a/web/src/main/java/org/hillview/RpcObjectManager.java +++ b/web/src/main/java/org/hillview/RpcObjectManager.java @@ -25,12 +25,8 @@ import javax.annotation.Nullable; import javax.websocket.Session; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; import java.util.HashMap; import java.util.List; -import java.util.Properties; import java.util.function.Consumer; /** @@ -44,12 +40,6 @@ * This is a singleton pattern. */ public final class RpcObjectManager { - /** - * This file contains the global properties that control hillview. - * This file is read by the root node. - */ - static final String propertiesFile = "hillview.properties"; - /** * Well-known id of the initial object. */ @@ -61,9 +51,6 @@ public final class RpcObjectManager { // the unique global instance. public static final RpcObjectManager instance; - // Global application properties - public final Properties properties; - // Map the session to the targetId object that is replying to the request, if any. private final HashMap sessionRequest = new HashMap(10); @@ -76,14 +63,6 @@ public final class RpcObjectManager { // Private constructor private RpcObjectManager() { this.objectLog = new RedoLog(); - this.properties = new Properties(); - try (FileInputStream prop = new FileInputStream(propertiesFile)) { - this.properties.load(prop); - } catch (FileNotFoundException ex) { - HillviewLogger.instance.info("No properties file found", "{0}", propertiesFile); - } catch (IOException ex) { - HillviewLogger.instance.error("Error while loading properties from file", ex); - } } synchronized void addSession(Session session, @Nullable RpcTarget target) { diff --git a/web/src/main/java/org/hillview/RpcTarget.java b/web/src/main/java/org/hillview/RpcTarget.java index 057a1669e..c15794606 100644 --- a/web/src/main/java/org/hillview/RpcTarget.java +++ b/web/src/main/java/org/hillview/RpcTarget.java @@ -128,7 +128,6 @@ void execute(RpcRequest request, RpcRequestContext context) { RpcReply reply = request.createReply(ex); if (context.session != null) RpcServer.sendReply(reply, context.session); - throw new RuntimeException(ex); } } diff --git a/web/src/main/java/org/hillview/targets/GreenplumFileDescriptionTarget.java b/web/src/main/java/org/hillview/targets/GreenplumFileDescriptionTarget.java new file mode 100644 index 000000000..48613061b --- /dev/null +++ b/web/src/main/java/org/hillview/targets/GreenplumFileDescriptionTarget.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2017 VMware Inc. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.hillview.targets; + +import org.hillview.*; +import org.hillview.dataset.api.IDataSet; +import org.hillview.dataset.api.IMap; +import org.hillview.maps.FalseMap; +import org.hillview.maps.LoadFilesMap; +import org.hillview.storage.IFileReference; +import org.hillview.storage.jdbc.JdbcDatabase; +import org.hillview.table.Schema; +import org.hillview.table.api.ITable; + +import javax.annotation.Nullable; + +/** + * This target loads files that are produced by dumping data from a Greenplum database. + */ +public class GreenplumFileDescriptionTarget extends FileDescriptionTarget { + static final long serialVersionUID = 1; + + private final JdbcDatabase database; + private final Schema schema; + private final String tmpTableName; + + GreenplumFileDescriptionTarget(IDataSet files, + HillviewComputation computation, + @Nullable String metadataDirectory, + String tmpTableName, + JdbcDatabase database, + Schema schema) { + super(files, computation, metadataDirectory); + this.tmpTableName = tmpTableName; + this.database = database; + this.schema = schema; + } + + @HillviewRpc + public void loadTable(RpcRequest request, RpcRequestContext context) { + IMap loader = new LoadFilesMap(); + this.runMap(this.files, loader, (d, c) -> new GreenplumTarget( + d, c, this.metadataDirectory, + this.tmpTableName, this.database, this.schema), request, context); + } + + @HillviewRpc + public void prune(RpcRequest request, RpcRequestContext context) { + this.runPrune(this.files, new FalseMap(), + (d, c) -> new GreenplumFileDescriptionTarget( + d, c, this.metadataDirectory, + this.tmpTableName, this.database, this.schema), request, context); + } +} diff --git a/web/src/main/java/org/hillview/targets/GreenplumStubTarget.java b/web/src/main/java/org/hillview/targets/GreenplumStubTarget.java new file mode 100644 index 000000000..cdef45fa6 --- /dev/null +++ b/web/src/main/java/org/hillview/targets/GreenplumStubTarget.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2020 VMware Inc. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.hillview.targets; + +import org.hillview.*; +import org.hillview.sketches.PrecomputedSketch; +import org.hillview.storage.jdbc.JdbcConnectionInformation; +import org.hillview.table.api.ITable; +import org.hillview.utils.Converters; +import org.hillview.utils.JsonInString; +import org.hillview.utils.Utilities; + +import java.sql.SQLException; +import java.util.List; + +/** + * This target is the first interface to a Greenplum database. + * It inherits some operations from SimpleDBTarget, in particular, + * getSummary. + */ +public class GreenplumStubTarget extends SimpleDBTarget { + static final String filePrefix = "file"; // Should match the prefix in the dump script + + public GreenplumStubTarget(JdbcConnectionInformation conn, HillviewComputation c, String dir) { + super(conn, c, dir); + } + + @HillviewRpc + public void initializeTable(RpcRequest request, RpcRequestContext context) throws SQLException { + String tmpTableName = request.parseArgs(String.class); + Converters.checkNull(this.schema); + /* + In this scheme, which does not seem to work, we only dump the first column now, + and we load the other ones later, when needed. + + List col = Utilities.list(this.schema.getColumnNames().get(0)); + GreenplumTarget.writeColumns(col, this.database, this.schema, tmpTableName); + */ + + // Create an external table that will be written into + String tableName = this.jdbc.table; + String query = "CREATE WRITABLE EXTERNAL WEB TABLE " + + tmpTableName + " (LIKE " + tableName + ") EXECUTE '" + + Configuration.instance.getGreenplumDumpScript() + " " + + Configuration.instance.getGreenplumDumpDirectory() + "/" + tmpTableName + + "' FORMAT 'CSV'"; + database.executeUpdate(query); + // This triggers the dumping of the data on the workers + query = "INSERT INTO " + tmpTableName + " SELECT * FROM " + tableName; + database.executeUpdate(query); + // Cleanup: remove temporary table and view + query = "DROP EXTERNAL TABLE " + tmpTableName; + database.executeUpdate(query); + + PrecomputedSketch sk = new PrecomputedSketch<>( + JsonInString.makeJsonString( + Configuration.instance.getGreenplumDumpDirectory() + "/" + tmpTableName + "/" + filePrefix + "*")); + this.runCompleteSketch(this.table, sk, request, context); + this.database.disconnect(); + } +} diff --git a/web/src/main/java/org/hillview/targets/GreenplumTarget.java b/web/src/main/java/org/hillview/targets/GreenplumTarget.java index 050a47e67..89b88b3ea 100644 --- a/web/src/main/java/org/hillview/targets/GreenplumTarget.java +++ b/web/src/main/java/org/hillview/targets/GreenplumTarget.java @@ -18,51 +18,125 @@ package org.hillview.targets; import org.hillview.*; -import org.hillview.sketches.PrecomputedSketch; -import org.hillview.storage.jdbc.JdbcConnectionInformation; +import org.hillview.dataset.api.ControlMessage; +import org.hillview.dataset.api.IDataSet; +import org.hillview.sketches.LoadCsvColumnsSketch; +import org.hillview.storage.jdbc.JdbcDatabase; +import org.hillview.table.ColumnDescription; +import org.hillview.table.Schema; import org.hillview.table.api.ITable; -import org.hillview.utils.JsonInString; +import org.hillview.utils.Converters; +import javax.annotation.Nullable; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; -@SuppressWarnings("SqlNoDataSourceInspection") -public class GreenplumTarget extends SimpleDBTarget { - static final String filePrefix = "file"; // Should match the prefix in the dump script +/** + * This class interfaces with a Greenplum database. It is very much like a TableTarget, except + * that for each operation is checks to see first whether the columns that will be operated on + * have been loaded, and if not loads them explicitly. + */ +public class GreenplumTarget extends TableTarget { + JdbcDatabase database; - /* - This is the expected contents of the dump-greenplum.sh script: -#!/bin/sh -DIR=$1 -PREFIX="file" -mkdir -p ${DIR} || exit 1 -echo "$(${DIR}/${PREFIX}${GP_SEGMENT_ID} - */ + protected HashSet columnsLoaded; + protected String tmpTableName; + protected final Schema schema; - public GreenplumTarget(JdbcConnectionInformation conn, HillviewComputation c, String dir) { - super(conn, c, dir); + public GreenplumTarget(IDataSet table, HillviewComputation c, + @Nullable String metadataDir, + String tmpTableName, JdbcDatabase db, Schema schema) { + super(table, c, metadataDir); + this.database = db; + this.schema = schema; + this.columnsLoaded = new HashSet(); + this.tmpTableName = tmpTableName; + this.columnsLoaded.add(this.schema.getColumnNames().get(0)); + try { + this.database.connect(); + } catch (SQLException ex) { + throw new RuntimeException(ex); + } } - @HillviewRpc - public void dumpTable(RpcRequest request, RpcRequestContext context) throws SQLException { - String tmpTableName = request.parseArgs(String.class); - String dumpScriptName = RpcObjectManager.instance.properties.getProperty( - "greenplumDumpScript", "/home/gpadmin/hillview/dump-greenplum.sh"); - String dumpDirectory = RpcObjectManager.instance.properties.getProperty( - "greenplumDumpDirectory", "/tmp"); + /** + * Given a list of columns, find the ones which have not been loaded yet. + * @param columns Columns to check. + * @return A set of columns that need to be loaded. + */ + protected List columnsToLoad(Iterable columns) { + List result = new ArrayList<>(); + for (String c: columns) { + if (!this.columnsLoaded.contains(c)) + result.add(c); + } + return result; + } - // This creates a virtual table that will write its partitions - // in files named like ${dumpDirectory}/${tmpTableName}/${filePrefix}Number - String tableName = this.jdbc.table; + public static void writeColumns(Iterable columns, JdbcDatabase database, + Schema schema, + String tmpTableName) throws SQLException { + String tableName = database.connInfo.table; + StringBuilder cols = new StringBuilder(); + boolean first = true; + for (String c: columns) { + if (!first) + cols.append(", "); + first = false; + ColumnDescription desc = schema.getDescription(c); + String type = JdbcDatabase.sqlType(desc.kind); + cols.append(c).append(" ").append(type); + } + // Create an external table that will be written into String query = "CREATE WRITABLE EXTERNAL WEB TABLE " + - tmpTableName + " (LIKE " + tableName + ") EXECUTE '" + - dumpScriptName + " " + dumpDirectory + "/" + tmpTableName + "' FORMAT 'CSV'"; + tmpTableName + " (" + cols.toString() + ") EXECUTE '" + + Configuration.instance.getGreenplumDumpScript() + " " + + Configuration.instance.getGreenplumDumpDirectory() + "/" + tmpTableName + + "' FORMAT 'CSV'"; + database.executeUpdate(query); + // This triggers the dumping of the data on the workers + query = "INSERT INTO " + tmpTableName + " SELECT " + String.join(", ", columns) + " FROM " + tableName; + database.executeUpdate(query); + // Cleanup: remove temporary table and view + query = "DROP EXTERNAL TABLE " + tmpTableName; + database.executeUpdate(query); + } - this.database.executeUpdate(query); - query = "INSERT INTO " + tmpTableName + " SELECT * FROM " + tableName; - this.database.executeUpdate(query); + protected void writeColumns(Iterable columns) throws SQLException { + writeColumns(columns, this.database, this.schema, this.tmpTableName); + } + + protected void loadWrittenColumns(List columns) { + // Ask remote workers to parse their local files + HashSet set = new HashSet<>(columns); + Schema toLoad = this.schema.project(set::contains); + LoadCsvColumnsSketch sketch = new LoadCsvColumnsSketch(toLoad); + ControlMessage.StatusList sl = this.table.blockingSketch(sketch); + for (ControlMessage.Status s: Converters.checkNull(sl)) + if (s.isError()) + throw new RuntimeException(s.exception); + this.columnsLoaded.addAll(columns); + } - PrecomputedSketch sk = new PrecomputedSketch( - JsonInString.makeJsonString(dumpDirectory + "/" + tmpTableName + "/" + filePrefix + "*")); - this.runCompleteSketch(this.table, sk, request, context); + synchronized protected void ensureColumns(Iterable columns) { + try { + List cols = this.columnsToLoad(columns); + if (cols.isEmpty()) + return; + this.writeColumns(cols); + this.loadWrittenColumns(cols); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @HillviewRpc + public void getNextK(RpcRequest request, RpcRequestContext context) { + NextKArgs nextKArgs = request.parseArgs(NextKArgs.class); + Schema schema = nextKArgs.order.toSchema(); + this.ensureColumns(schema.getColumnNames()); + super.getNextK(request, context); } } diff --git a/web/src/main/java/org/hillview/targets/InitialObjectTarget.java b/web/src/main/java/org/hillview/targets/InitialObjectTarget.java index 3daf93597..bd0ba4d84 100644 --- a/web/src/main/java/org/hillview/targets/InitialObjectTarget.java +++ b/web/src/main/java/org/hillview/targets/InitialObjectTarget.java @@ -21,6 +21,7 @@ import org.hillview.*; import org.hillview.sketches.PrecomputedSketch; import org.hillview.storage.jdbc.JdbcConnectionInformation; +import org.hillview.storage.jdbc.JdbcDatabase; import org.hillview.table.PrivacySchema; import org.hillview.dataset.RemoteDataSet; import org.hillview.dataset.api.*; @@ -31,6 +32,7 @@ import org.hillview.maps.highorder.IdMap; import org.hillview.maps.LoadDatabaseTableMap; import org.hillview.storage.*; +import org.hillview.table.Schema; import org.hillview.utils.*; import javax.annotation.Nullable; @@ -92,31 +94,24 @@ private void initialize(final HostList description) { @HillviewRpc public void getUIConfig(RpcRequest request, RpcRequestContext context) { - JsonInString result; - try { - result = new JsonInString(Utilities.textFileContents("uiconfig.json")); - result.toJsonTree(); // force parsing of the JSON -- to catch syntax errors - } catch (Exception e) { - HillviewLogger.instance.warn("File uiconfig.json file could not be loaded", - "{0}", e.getMessage()); - result = new JsonInString("{}"); - } + UIConfig config = new UIConfig(); + config.enableSaveAs = Configuration.instance.getBooleanProperty("enableSaveAs"); + config.localDbMenu = Configuration.instance.getBooleanProperty("localDbMenu"); + config.showTestMenu = Configuration.instance.getBooleanProperty("showTestMenu"); + config.enableManagement = Configuration.instance.getBooleanProperty("enableManagement"); + config.privateIsCsv = Configuration.instance.getBooleanProperty("privateIsCsv"); + config.hideSuggestions = Configuration.instance.getBooleanProperty("hideSuggestions"); + config.hideDemoMenu = Configuration.instance.getBooleanProperty("hideDemoMenu"); Converters.checkNull(this.emptyDataset); - PrecomputedSketch sk = new PrecomputedSketch(result); + PrecomputedSketch sk = new PrecomputedSketch(config); this.runCompleteSketch(this.emptyDataset, sk, request, context); } @HillviewRpc - public void openingBookmark(RpcRequest request, RpcRequestContext context) { + public void openingBookmark(RpcRequest request, RpcRequestContext context) throws IOException { String bookmarkFile = request.parseArgs(String.class); - String content; - try { - File file = new File(InitialObjectTarget.bookmarkDirectory, bookmarkFile); - content = FileUtils.readFileToString(file, StandardCharsets.UTF_8); - } catch (Exception e) { - // Bookmark link is broken. Failed to find bookmarked content - throw new RuntimeException(e); - } + File file = new File(InitialObjectTarget.bookmarkDirectory, bookmarkFile); + String content = FileUtils.readFileToString(file, StandardCharsets.UTF_8); Converters.checkNull(this.emptyDataset); PrecomputedSketch sk = new PrecomputedSketch(new JsonInString(content)); this.runCompleteSketch(this.emptyDataset, sk, request, context); @@ -147,6 +142,28 @@ public void loadSimpleDBTable(RpcRequest request, RpcRequestContext context) { } } + @SuppressWarnings("NotNullFieldNotInitialized") + static class GreenplumInfo { + FileSetDescription files; + Schema schema; + JdbcConnectionInformation jdbc; + } + + @HillviewRpc + public void findGreenplumFiles(RpcRequest request, RpcRequestContext context) { + GreenplumInfo desc = request.parseArgs(GreenplumInfo.class); + HillviewLogger.instance.info("Finding files", "{0}", desc); + IMap> finder = new FindFilesMap<>(desc.files); + String folder = Utilities.getFolder(desc.files.fileNamePattern); + Converters.checkNull(desc.schema); + Converters.checkNull(this.emptyDataset); + JdbcDatabase database = new JdbcDatabase(desc.jdbc); + String tmpTableName = Utilities.getBasename(folder); + this.runFlatMap(this.emptyDataset, finder, + (d, c) -> new GreenplumFileDescriptionTarget( + d, c, folder, tmpTableName, database, desc.schema), request, context); + } + @HillviewRpc public void findCassandraFiles(RpcRequest request, RpcRequestContext context) { CassandraConnectionInfo desc = request.parseArgs(CassandraConnectionInfo.class); @@ -180,7 +197,7 @@ public void loadGreenplumTable(RpcRequest request, RpcRequestContext context) { String dir = Paths.get(Converters.checkNull(conn.databaseKind).toLowerCase(), Converters.checkNull(conn.database), conn.table).toString(); - this.runMap(this.emptyDataset, map, (e, c) -> new GreenplumTarget(conn, c, dir), request, context); + this.runMap(this.emptyDataset, map, (e, c) -> new GreenplumStubTarget(conn, c, dir), request, context); } @HillviewRpc @@ -189,7 +206,13 @@ public void findFiles(RpcRequest request, RpcRequestContext context) { HillviewLogger.instance.info("Finding files", "{0}", desc); IMap> finder = new FindFilesMap<>(desc); Converters.checkNull(this.emptyDataset); - String folder = Utilities.getFolder(desc.fileNamePattern); + String folder; + if (desc.fileKind.equals("lazycsv")) + // These files are generated from a database, use the database/table name, + // which is now in the 'name' field of the FileSetDescription + folder = desc.name; + else + folder = Utilities.getFolder(desc.fileNamePattern); String privacyMetadataFile = DPWrapper.privacyMetadataFile(folder); if (privacyMetadataFile != null) { diff --git a/web/src/main/java/org/hillview/targets/TableTarget.java b/web/src/main/java/org/hillview/targets/TableTarget.java index 3e9adff55..80188899a 100644 --- a/web/src/main/java/org/hillview/targets/TableTarget.java +++ b/web/src/main/java/org/hillview/targets/TableTarget.java @@ -48,7 +48,7 @@ * Almost all operations are triggered from this object. */ @SuppressWarnings("CanBeFinal") -public final class TableTarget extends TableRpcTarget { +public class TableTarget extends TableRpcTarget { static final long serialVersionUID = 1; TableTarget(IDataSet table, HillviewComputation computation, @Nullable String metadataDirectory) { @@ -222,6 +222,7 @@ public JsonInString createJSON(PolygonSet ps) { @Nullable GeoFileInformation getGeoFileInformation(String columnName) throws IOException { String fileName = "data/metadata/geo/" + this.metadataDirectory + "/geometa.json"; + HillviewLogger.instance.info("Looking for geo data", "file {0}", fileName); File file = new File(fileName); if (!file.exists()) return null; diff --git a/web/src/main/webapp/dataViews/geoView.ts b/web/src/main/webapp/dataViews/geoView.ts index 0b3a229cd..96ea5a69f 100644 --- a/web/src/main/webapp/dataViews/geoView.ts +++ b/web/src/main/webapp/dataViews/geoView.ts @@ -251,4 +251,9 @@ export class GeoDataReceiver extends Receiver { return; this.geoView.updateView(v.data); } -} \ No newline at end of file + + public onCompleted(): void { + super.onCompleted(); + this.geoView.updateCompleted(this.elapsedMilliseconds()); + } +} diff --git a/web/src/main/webapp/initialObject.ts b/web/src/main/webapp/initialObject.ts index 15f5d0589..96f9242e3 100644 --- a/web/src/main/webapp/initialObject.ts +++ b/web/src/main/webapp/initialObject.ts @@ -77,7 +77,7 @@ class FilesReceiver extends OnCompleteReceiver { constructor(sourcePage: FullPage, operation: ICancellable, protected data: DataLoaded, protected newDataset: boolean) { - super(sourcePage, operation, "Get file info"); + super(sourcePage, operation, "Discovering files on disk"); } public run(remoteObjId: RemoteObjectId): void { @@ -98,7 +98,7 @@ class FileSizeReceiver extends OnCompleteReceiver { protected data: DataLoaded, protected remoteObj: RemoteObject, protected newDataset: boolean) { - super(sourcePage, operation, "Load data"); + super(sourcePage, operation, "Enumerating files"); } public run(size: FileSizeSketchInfo): void { @@ -130,7 +130,7 @@ class FilePruneReceiver extends OnCompleteReceiver { constructor(sourcePage: FullPage, operation: ICancellable, protected data: DataLoaded, protected readonly size: FileSizeSketchInfo, protected newDataset: boolean) { - super(sourcePage, operation, "Load data"); + super(sourcePage, operation, "Disconnecting workers without data"); } public run(remoteObjId: RemoteObjectId): void { @@ -189,12 +189,11 @@ class GreenplumTableReceiver extends BaseReceiver { * @param initialObject Handle to the initial object; used later to load the files * obtained from dumping the table. * @param operation Operation that will bring the results. - * @param progressInfo Description of the files that are being loaded. */ constructor(loadMenuPage: FullPage, operation: ICancellable, protected initialObject: InitialObject, - protected data: DataLoaded, progressInfo: string) { - super(loadMenuPage, operation, progressInfo, null); + protected data: TablesLoaded) { + super(loadMenuPage, operation, "Connecting to Greenplum database", null); } public run(value: RemoteObjectId): void { @@ -204,15 +203,17 @@ class GreenplumTableReceiver extends BaseReceiver { const title = getDescription(this.data); const dataset = new DatasetView(this.remoteObject.remoteObjectId, title.format, this.data, this.page); const newPage = dataset.newPage(title, null); - rr.invoke(new GreenplumSchemaReceiver(newPage, rr, this.initialObject, this.remoteObject)); + rr.invoke(new GreenplumSchemaReceiver( + newPage, rr, this.initialObject, this.remoteObject, this.data.description)); } } class GreenplumSchemaReceiver extends OnCompleteReceiver { constructor(page: FullPage, operation: ICancellable, protected initialObject: InitialObject, - protected remoteObject: TableTargetAPI) { - super(page, operation, "Get schema"); + protected remoteObject: TableTargetAPI, + protected jdbc: JdbcConnectionInformation) { + super(page, operation, "Reading table metadata"); } public run(ts: TableSummary): void { @@ -224,33 +225,54 @@ class GreenplumSchemaReceiver extends OnCompleteReceiver { // where the tables are stored on the remote machines. // This is the name of the temporary table used. const tableName = "T" + getUUID().replace(/-/g, ''); - const rr = this.remoteObject.createStreamingRpcRequest("dumpTable", tableName); - rr.invoke(new GreenplumLoader(this.page, ts, this.initialObject, rr)); + const rr = this.remoteObject.createStreamingRpcRequest("initializeTable", tableName); + rr.chain(this.operation); + rr.invoke(new GreenplumLoader(this.page, ts, this.initialObject, this.jdbc, rr)); } } class GreenplumLoader extends OnCompleteReceiver { constructor(page: FullPage, protected summary: TableSummary, protected remoteObject: InitialObject, + protected jdbc: JdbcConnectionInformation, operation: ICancellable) { - super(page, operation, "Find table fragments"); + super(page, operation, "Dumping data from database"); } public run(value: string): void { + /* + This was an attempt to only load one column from greenplum, but it + does not seem to work. + const files: FileSetDescription = { - fileKind: "csv", + fileKind: "lazycsv", fileNamePattern: value, schemaFile: null, headerRow: false, schema: this.summary.schema, name: (this.page.dataset?.loaded as TablesLoaded).description.table, - repeat: 1, - logFormat: null, - startTime: null, - endTime: null, - deleteAfterLoading: true + deleteAfterLoading: true, }; - const rr = this.remoteObject.createStreamingRpcRequest("findFiles", files); + const rr = this.remoteObject.createStreamingRpcRequest( + "findGreenplumFiles", { + files: files, + schema: this.summary.schema, + jdbc: this.jdbc + }); + */ + const loaded = this.page.dataset?.loaded as TablesLoaded; + const files: FileSetDescription = { + fileKind: "lazycsv", + fileNamePattern: value, + schemaFile: null, + headerRow: false, + schema: this.summary.schema, + name: loaded.description.database + "/" + loaded.description.table, + deleteAfterLoading: true, + }; + const rr = this.remoteObject.createStreamingRpcRequest( + "findFiles", files); + rr.chain(this.operation); const observer = new FilesReceiver(this.page, rr, { kind: "Files", description: files }, false); rr.invoke(observer); @@ -307,7 +329,7 @@ export class InitialObject extends RemoteObject { protected loadGreenplumTable(conn: JdbcConnectionInformation, loadMenuPage: FullPage, method: string): void { const rr = this.createStreamingRpcRequest(method, conn); const observer = new GreenplumTableReceiver(loadMenuPage, rr, this, - { kind: "DB", description: conn }, "loading Greenplum table"); + { kind: "DB", description: conn }); rr.invoke(observer); } diff --git a/web/src/main/webapp/javaBridge.ts b/web/src/main/webapp/javaBridge.ts index 714fff301..f66a4a8af 100644 --- a/web/src/main/webapp/javaBridge.ts +++ b/web/src/main/webapp/javaBridge.ts @@ -92,6 +92,7 @@ export interface UIConfig { enableManagement?: boolean; // management menu enabled privateIsCsv?: boolean; // the private dataset is in csv form hideSuggestions?: boolean; // do not display the suggestions + hideDemoMenu?: boolean; // do not show the "demo" menu } export interface ColumnQuantization { @@ -158,7 +159,7 @@ export interface CassandraConnectionInfo extends JdbcConnectionInformation { cassandraRootDir: string; } -export type DataKinds = "csv" | "orc" | "parquet" | "json" | "hillviewlog" | "db" | "genericlog" | "sstable"; +export type DataKinds = "csv" | "orc" | "parquet" | "json" | "hillviewlog" | "db" | "genericlog" | "sstable" | "lazycsv"; export interface FileSetDescription { fileKind: DataKinds; @@ -167,12 +168,12 @@ export interface FileSetDescription { schema: Schema | null; headerRow?: boolean; cookie?: string; - repeat: number; - name: string | null; // not used on the Java side - logFormat: string | null; - startTime: number | null; - endTime: number | null; - deleteAfterLoading: boolean; + repeat?: number; + name?: string; + logFormat?: string; + startTime?: number | null; // in the same units used by the timestamp column + endTime?: number | null; + deleteAfterLoading?: boolean; } export interface CountWithConfidence { diff --git a/web/src/main/webapp/loadView.ts b/web/src/main/webapp/loadView.ts index a5ba4c35a..51afd03d5 100644 --- a/web/src/main/webapp/loadView.ts +++ b/web/src/main/webapp/loadView.ts @@ -89,13 +89,8 @@ export class LoadView extends RemoteObject implements IDataView { schemaFile: "short.schema", schema: null, headerRow: true, - repeat: 1, name: "Flights (15 columns)", fileKind: "csv", - logFormat: null, - startTime: null, - endTime: null, - deleteAfterLoading: false }; this.init.loadFiles(files, this.page); }, @@ -107,14 +102,8 @@ export class LoadView extends RemoteObject implements IDataView { fileNamePattern: "data/ontime_small_orc/*.orc", schemaFile: "schema", schema: null, - headerRow: true, - repeat: 1, name: "Flights (15 columns, ORC)", fileKind: "orc", - logFormat: null, - startTime: null, - endTime: null, - deleteAfterLoading: false }; this.init.loadFiles(files, this.page); }, @@ -126,14 +115,8 @@ export class LoadView extends RemoteObject implements IDataView { fileNamePattern: "data/ontime_big_orc/*.orc", schemaFile: "schema", schema: null, - headerRow: true, - repeat: 1, name: "Flights (ORC)", fileKind: "orc", - logFormat: null, - startTime: null, - endTime: null, - deleteAfterLoading: false }; this.init.loadFiles(files, this.page); }, @@ -147,13 +130,8 @@ export class LoadView extends RemoteObject implements IDataView { schemaFile: "short.schema", schema: null, headerRow: true, - repeat: 1, name: "Flights (private)", fileKind: "csv", - logFormat: null, - startTime: null, - endTime: null, - deleteAfterLoading: false }; this.init.loadFiles(files, this.page); }, @@ -167,14 +145,8 @@ export class LoadView extends RemoteObject implements IDataView { fileNamePattern: "data/ontime_private/*.orc", schemaFile: "schema", schema: null, - headerRow: true, - repeat: 1, name: "Flights (private)", fileKind: "orc", - logFormat: null, - startTime: null, - endTime: null, - deleteAfterLoading: false }; this.init.loadFiles(files, this.page); }, @@ -274,14 +246,14 @@ export class LoadView extends RemoteObject implements IDataView { help: "A database table in a single database." }); this.loadMenu = new SubMenu(loadMenuItems); - const items: TopMenuItem[] = [ - { text: "Test datasets", help: "Hardwired datasets for testing Hillview.", + const items: TopMenuItem[] = []; + if (!HillviewToplevel.instance.uiconfig.hideDemoMenu) + items.push({ text: "Demo datasets", help: "Hardwired datasets for testing Hillview.", subMenu: this.testDatasetsMenu, - }, { + }); + items.push({ text: "Load", help: "Load data from the worker machines.", - subMenu: this.loadMenu }, - ]; - + subMenu: this.loadMenu }); if (HillviewToplevel.instance.uiconfig.showTestMenu) { items.push({ text: "Test", help: "Run UI tests", subMenu: new SubMenu([ @@ -469,13 +441,7 @@ class CSVFileDialog extends Dialog { schema: null, fileNamePattern: this.getFieldValue("fileNamePattern"), headerRow: this.getBooleanValue("hasHeader"), - repeat: 1, - name: null, fileKind: "csv", - logFormat: null, - startTime: null, - endTime: null, - deleteAfterLoading: false }; } } @@ -508,14 +474,10 @@ class GenericLogDialog extends Dialog { schema: null, logFormat: this.getFieldValue("logFormat"), fileNamePattern: this.getFieldValue("fileNamePattern"), - headerRow: false, - repeat: 1, - name: null, cookie: getUUID(), fileKind: "genericlog", startTime: Converters.doubleFromDate(this.getDateTimeValue("startTime")), endTime: Converters.doubleFromDate(this.getDateTimeValue("endTime")), - deleteAfterLoading: false }; } } @@ -541,14 +503,7 @@ class JsonFileDialog extends Dialog { schemaFile: this.getFieldValue("schemaFile"), schema: null, fileNamePattern: this.getFieldValue("fileNamePattern"), - headerRow: false, - repeat: 1, - name: null, fileKind: "json", - logFormat: null, - startTime: null, - endTime: null, - deleteAfterLoading: false }; } } @@ -571,14 +526,7 @@ class ParquetFileDialog extends Dialog { schemaFile: null, // not used schema: null, fileNamePattern: this.getFieldValue("fileNamePattern"), - headerRow: false, // not used - repeat: 1, - name: null, fileKind: "parquet", - logFormat: null, - startTime: null, - endTime: null, - deleteAfterLoading: false }; } } @@ -604,14 +552,7 @@ class OrcFileDialog extends Dialog { schemaFile: this.getFieldValue("schemaFile"), schema: null, fileNamePattern: this.getFieldValue("fileNamePattern"), - headerRow: false, // not used - repeat: 1, - name: null, fileKind: "orc", - logFormat: null, - startTime: null, - endTime: null, - deleteAfterLoading: false }; } } @@ -622,9 +563,10 @@ class OrcFileDialog extends Dialog { class DBDialog extends Dialog { constructor(isFederated: boolean) { super("Load DB tables", "Loads data from a parallel or federated database."); - const arrDB: FederatedDatabase[] = ["mysql"]; - if (isFederated) arrDB.push("cassandra", "greenplum"); + const arrDB: FederatedDatabase[] = []; + if (isFederated) arrDB.push("greenplum", "cassandra"); else arrDB.push("impala"); + arrDB.push("mysql"); const sel = this.addSelectFieldAsObject( "databaseKind", "Database kind", arrDB, (l) => l.toString(), @@ -638,7 +580,7 @@ class DBDialog extends Dialog { "Absolute path of dCassandra's installation directory"); this.hideInputField("dbDir", dbDir); } - const port = this.addTextField("port", "Port", FieldKind.Integer, "3306", + const port = this.addTextField("port", "Port", FieldKind.Integer, null, "Network port to connect to database."); port.required = true; if (isFederated) { @@ -659,6 +601,7 @@ class DBDialog extends Dialog { // not caching the federated connection because the cache failed to show all Cassandra's fields if (!isFederated) this.setCacheTitle("DBDialog"); + this.dbChanged(); } public getDbKind(): FederatedDatabase | null { diff --git a/web/src/main/webapp/ui/progress.ts b/web/src/main/webapp/ui/progress.ts index c8086fadd..a23cc67fa 100644 --- a/web/src/main/webapp/ui/progress.ts +++ b/web/src/main/webapp/ui/progress.ts @@ -119,7 +119,7 @@ export class ProgressBar implements IHtmlElement { const progress = end - this.firstPosition; if (progress > 0 && elapsed > 2000) { const estimated = elapsed / progress - elapsed; - this.estimate.textContent = "Remaining time: " + readableTime(estimated); + this.estimate.textContent = "Est. remaining time: " + readableTime(estimated); } } this.end = end;