Skip to content

Commit

Permalink
add more supoort for one2multi
Browse files Browse the repository at this point in the history
  • Loading branch information
@ferventdesert
ferventdesert committed May 2, 2019
2 parents 0ae832c + ab408c8 commit 3139ffd
Show file tree
Hide file tree
Showing 17 changed files with 388 additions and 344 deletions.
25 changes: 24 additions & 1 deletion Docs/update.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -660,10 +660,33 @@ http://www.qnsb.com/fzepaper/site1/qnsb/page/1/2012-01/01/A03/20120101A03_pdf.pd
9. 百度AI

CREATE TABLE DFDSF (diggit_diggnum text,digg_diggit_onclick text,digg_digg_tip_id text,titlelnk text,属性4 text,pfs_src text,a_href text,属性7 text,post_item_foot_lightblue text,属性9 text,article_comment_gray text,article_view_gray text)


# 2019年04月02日
1. 数据清洗单元格，可以默认不换行 ok
2. xpath转换器的优化，不需刷新即可查找xpath ok



# 2019年4月28日
2. 加载市场的单个任务时，会提示找不到任务 ok
2. 当加载单个任务时，发现与当前名称重复时应当提示 pass
3. topic_supplier 不存在 ok
4. 弹出的数据对话框窗口太小 ok
5. 回补数据即使勾选执行器，也无法执行原有执行器
5. 回补数据即使勾选执行器，也无法执行原有执行器 pass
6. 提取测试时，无法将数据表写入 ok
7. 英文界面
8. 并发写入文件的问题
9. 新建工程后，并没有自动连接 ok


# 2019年5月2日
1. 当数据表中已经存在同名数据集时，禁止写入 ok
2. 数据表双击看不到 ok
3. mongodb 查看帮助提示，更换链接 ok
4. wiki更新
5. 修复图床
6. sqlite写入表失败 ok
7. 读取文件数据 应当提供图形化界面 ok


17 changes: 17 additions & 0 deletions Docs/教学视频录制.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,20 @@
5. 链家抓取（单页面/手气不错）

6. 链家多页面(请求)


### 一转多多转一的教程

单文档，多文档，不进行转换

多文档转换下，如何保存之前的列

带index:正则转换器，字符分割， 正则分割，提取数字。 如果不知道有多少个匹配，就只能用多文档模式
不带index: python转换器，json转换器，字符首尾抽取， XPath转换器, 从爬虫转换

特殊的： 矩阵转置，


### 人工智能相关

获取地图位置
2 changes: 1 addition & 1 deletion Hawk.Core/Connectors/FileConnectorExcel.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public override IEnumerable<FreeDocument> ReadFile(Action<int> alreadyGetSize =
string value = "";
if (cell != null)
value = cell.ToString();
dict.Set(title, value);
dict.SetValue(title, value);
}

if (data != null)
Expand Down
2 changes: 1 addition & 1 deletion Hawk.Core/Connectors/MongoDBConnector.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ public ReadOnlyCollection<ICommand> HelpCommands
new Command(GlobalHelper.Get("key_71"), obj =>
{
var url =
"https://github.com/ferventdesert/Hawk/wiki/5.-%E6%95%B0%E6%8D%AE%E5%BA%93%E7%B3%BB%E7%BB%9F";
"https://ferventdesert.github.io/Hawk/%E6%95%B0%E6%8D%AE%E8%A1%A8%E5%92%8C%E6%95%B0%E6%8D%AE%E5%BA%93%E8%BF%9E%E6%8E%A5/";
System.Diagnostics.Process.Start(url);

}),
Expand Down
83 changes: 28 additions & 55 deletions Hawk.Core/Connectors/SQLiteConnector.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
using System;
using Hawk.Core.Utils;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.ComponentModel;
Expand All @@ -9,6 +8,7 @@
using System.Windows.Controls.WpfPropertyGrid.Attributes;
using System.Windows.Forms;
using System.Windows.Input;
using Hawk.Core.Utils;
using Hawk.Core.Utils.Logs;
using Hawk.Core.Utils.MVVM;
using Hawk.Core.Utils.Plugins;
Expand All @@ -19,6 +19,7 @@ namespace Hawk.Core.Connectors
public class SQLiteDatabase : DBConnectorBase
{
private string _dbName;
private SQLiteConnection cnn;

#region Methods

Expand All @@ -33,10 +34,6 @@ protected override DataTable GetDataTable(string sql)

try
{
var cnn = new SQLiteConnection(ConnectionString);

cnn.Open();

var mycommand = new SQLiteCommand(cnn);

mycommand.CommandText = sql;
Expand All @@ -46,8 +43,6 @@ protected override DataTable GetDataTable(string sql)
dt.Load(reader);

reader.Close();

cnn.Close();
}

catch (Exception e)
Expand Down Expand Up @@ -120,7 +115,6 @@ private void CreateNewDB()
{
DBName = saveTifFileDialog.FileName;
SQLiteConnection.CreateFile(DBName);

}
SafeConnectDB();
}
Expand Down Expand Up @@ -169,7 +163,8 @@ public override ReadOnlyCollection<ICommand> Commands
this,
new[]
{
new Command(GlobalHelper.Get("connect_db"), obj => { SafeConnectDB(); }, obj => IsUseable == false, "connect"),
new Command(GlobalHelper.Get("connect_db"), obj => { SafeConnectDB(); },
obj => IsUseable == false, "connect"),
new Command(GlobalHelper.Get("key_36"), obj => CloseDB(), obj => IsUseable, "close")
});
}
Expand Down Expand Up @@ -198,32 +193,29 @@ protected override void ConnectStringToOtherInfos()

#region Public Methods

public override void BatchInsert(IEnumerable<IFreeDocument> source,List<string>keys, string dbTableName)
public override void BatchInsert(IEnumerable<IFreeDocument> source, List<string> keys, string dbTableName)
{
using (var cnn = new SQLiteConnection(ConnectionString))
if(this.IsUseable==false)
return;
using (var cmd = new SQLiteCommand(this.cnn))
{
cnn.Open();

using (var mytrans = cnn.BeginTransaction())
{
foreach (var data in source)
{
try
{
var sql = Insert(data,keys, dbTableName);
var mycommand = new SQLiteCommand(sql, cnn, mytrans);
mycommand.CommandTimeout = 180;
mycommand.ExecuteNonQuery();
var sql = Insert(data, keys, dbTableName);
cmd.CommandText = sql;
cmd.ExecuteNonQuery();
}
catch (Exception ex)
{
XLogSys.Print.Warn($"insert sqlite database error {ex.Message}");
}

mytrans.Commit();

cnn.Close();
}
mytrans.Commit();
}
}
}
Expand All @@ -232,20 +224,19 @@ public override void BatchInsert(IEnumerable<IFreeDocument> source,List<string>k
public override bool CloseDB()
{
IsUseable = false;

cnn.Close();
return true;
}

public override bool ConnectDB()
{
ConnectionString = GetConnectString();
var cnn = new SQLiteConnection(ConnectionString);
cnn = new SQLiteConnection(ConnectionString);

try
{
cnn.Open();
IsUseable = true;
cnn.Close();
return true;
}
catch (Exception ex)
Expand All @@ -260,11 +251,13 @@ public override bool CreateTable(IFreeDocument example, string name)
{
if (string.IsNullOrEmpty(name))
throw new Exception(GlobalHelper.Get("key_29"));
var txt = example.DictSerialize(Scenario.Database);
var txt = example.DictSerialize();
var sb = string.Join(",", txt.Select(d => $"{d.Key} {DataTypeConverter.ToType(d.Value)}"));
var sql = $"CREATE TABLE {GetTableName(name)} ({sb})";
var sql = $"CREATE TABLE {GetTableName(name)} ({sb});";
ExecuteNonQuery(sql);
RefreshTableNames();
if (TableNames.Collection.FirstOrDefault(d => d.Name == name) == null)
return false;
return true;
}

Expand All @@ -276,16 +269,11 @@ public override bool CreateTable(IFreeDocument example, string name)
/// <returns>An Integer containing the number of rows updated.</returns>
protected override int ExecuteNonQuery(string sql)
{
using (var cnn = new SQLiteConnection(ConnectionString))
{
cnn.Open();

using (var mytrans = cnn.BeginTransaction())
{
return ExecuteNonQuery(sql, cnn );
}
}

// using (cnn.BeginTransaction())
// {
var result = ExecuteNonQuery(sql, cnn);
return result;
// }
}


Expand All @@ -295,20 +283,15 @@ protected int ExecuteNonQuery(string sql, SQLiteConnection cnn)


var mycommand = new SQLiteCommand(sql, cnn);

try
{
mycommand.CommandTimeout = 180;

col = mycommand.ExecuteNonQuery();

}

catch (Exception e)
{
XLogSys.Print.Warn(GlobalHelper.Get("key_88")+ e.Message+ GlobalHelper.Get("key_89") + sql );
XLogSys.Print.Warn(GlobalHelper.Get("key_88") + e.Message + GlobalHelper.Get("key_89") + sql);
}

finally
{
mycommand.Dispose();
Expand All @@ -328,8 +311,6 @@ public override List<FreeDocument> QueryEntities(string querySQL, out int count,
public bool ExecuteNonQuery(string sql, IList<SQLiteParameter> cmdparams)
{
var successState = false;
var cnn = new SQLiteConnection(ConnectionString);
cnn.Open();

using (var mytrans = cnn.BeginTransaction())
{
Expand All @@ -346,8 +327,6 @@ public bool ExecuteNonQuery(string sql, IList<SQLiteParameter> cmdparams)
mytrans.Commit();

successState = true;

cnn.Close();
}

catch (Exception e)
Expand All @@ -360,8 +339,6 @@ public bool ExecuteNonQuery(string sql, IList<SQLiteParameter> cmdparams)
finally
{
mycommand.Dispose();

cnn.Close();
}
}

Expand Down Expand Up @@ -401,10 +378,6 @@ protected DataTable GetDataTable(string sql, IList<SQLiteParameter> cmdparams)

try
{
var cnn = new SQLiteConnection(ConnectionString);

cnn.Open();

var mycommand = new SQLiteCommand(cnn);

mycommand.CommandText = sql;
Expand All @@ -418,8 +391,6 @@ protected DataTable GetDataTable(string sql, IList<SQLiteParameter> cmdparams)
dt.Load(reader);

reader.Close();

cnn.Close();
}

catch (Exception e)
Expand Down Expand Up @@ -447,12 +418,14 @@ public override IEnumerable<FreeDocument> GetEntities(string tableName, int moun

public override List<TableInfo> RefreshTableNames()
{
var items = GetDataTable("select name from sqlite_master where type='table' order by name");
if (!this.IsUseable)
return new List<TableInfo>();
var items = GetDataTable("select name from sqlite_master where type='table' order by name;");
var res = new List<TableInfo>();
foreach (DataRow dr in items.Rows)
{
var name = dr.ItemArray[0].ToString();
var size = GetDataTable($"select count(*) from {name}").Rows[0].ItemArray[0];
var size = GetDataTable($"select count(*) from {name};").Rows[0].ItemArray[0];
res.Add(new TableInfo(name, this) {Size = int.Parse(size.ToString())});
}
TableNames.SetSource(res);
Expand Down
34 changes: 28 additions & 6 deletions Hawk.Core/Utils/ExtendMethod.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,12 +181,14 @@ public static IEnumerable<IFreeDocument> InserDataCollection(this IDataBaseConne
if (tableName == null)
tableName = collection.Name;
List<string> columns =new List<string>();
var sample = data.MergeToDocument();
columns = data.GetKeys().ToList();
if (connector.RefreshTableNames().FirstOrDefault(d => d.Name == tableName) == null)
{
var sample = data.MergeToDocument();
columns = data.GetKeys().ToList();
connector.CreateTable(sample, tableName);
connector.RefreshTableNames();
var result=connector.CreateTable(sample, tableName);
if(result==false)
throw new Exception(String.Format(GlobalHelper.Get("key_349"), tableName));
// connector.RefreshTableNames();
}
return columns;

Expand Down Expand Up @@ -248,7 +250,16 @@ public static IEnumerable<T> BatchDo<T>(this IEnumerable<T> documents,
{
if (list.Count == initMount)
{
result = initFunc(list);
try
{
result = initFunc(list);
}
catch (Exception ex)
{
XLogSys.Print.Error("batch execute failed "+ ex.Message);
yield break;
}

isInit = true;
}
else
Expand All @@ -274,7 +285,18 @@ public static IEnumerable<T> BatchDo<T>(this IEnumerable<T> documents,
i++;
}
if (isInit == false)
result = initFunc(list);
{
try
{
result = initFunc(list);
}
catch (Exception ex)
{
XLogSys.Print.Error("batch execute failed " + ex.Message);
yield break;
}

}
if (list.Count != 0)
{
batchFunc(list, result);
Expand Down
13 changes: 12 additions & 1 deletion Hawk.Core/Utils/ExtenedEnumerable.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -661,7 +661,18 @@ public static IEnumerable<T> MergeAll<T>(this IEnumerable<T> dict1, IEnumerable<
yield return f.Current;
}
}

public static IEnumerable<T> MergeAll<T>(this IEnumerable<T> dict1, Func<T, IEnumerable<T>> generator)
where T : IFreeDocument
{
var f = dict1.GetEnumerator();
while (f.MoveNext())
{
var dict2 = generator(f.Current).FirstOrDefault();
if(dict2!=null)
f.Current.AddRange(dict2);
yield return f.Current;
}
}
public static IEnumerable<T> Mix<T>(this IEnumerable<T> dict1, IEnumerable<T> dict2)
where T : IFreeDocument
{
Expand Down
Loading

0 comments on commit 3139ffd

Please sign in to comment.