Skip to content

Commit

Permalink
Cleanup for MySQL 5.7
Browse files Browse the repository at this point in the history 
Since librusec is now using innodb there is no need for skip-inno-db settings
Remove all memory related MySQL parameters (use mysql.ini if needed)
Set directories for innoddb files
Proper cleanup after innodb
  • Loading branch information
@rupor-github
rupor-github committed May 22, 2016
1 parent c7f91ec commit 006785a
Show file tree
Hide file tree
Showing 5 changed files with 437 additions and 520 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}/cmake)
project(InpxCreator)
# Project version number
set(PRJ_VERSION_MAJOR 6)
set(PRJ_VERSION_MINOR 4)
set(PRJ_VERSION_MINOR 41)
configure_file ("${PROJECT_SOURCE_DIR}/cmake/version.h.in" "${PROJECT_BINARY_DIR}/version.h")
include_directories("${PROJECT_BINARY_DIR}")

Expand Down
53 changes: 30 additions & 23 deletions lib2inpx/lib2inpx.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ static series_type g_series_type = eAuthorST;

static long g_last_fb2 = 0;
static string g_update;
static string g_db_name = "librusec";
static string g_db_name = "flibusta";

static bool g_clean_authors = false;
static bool g_clean_aliases = false;
Expand Down Expand Up @@ -131,8 +131,9 @@ static const char* options_pattern[] = {"%s",
"--datadir=%s/data",
"--language=%s/language",
"--skip-grant-tables",
"--skip-innodb",
"--key-buffer-size=64M",
"--innodb_data_home_dir=%s/data/dbtmp_%s",
"--innodb_log_group_home_dir=%s/data/dbtmp_%s",
"--innodb_tmpdir=%s/data/dbtmp_%s",
"--log_syslog=0",
NULL};

Expand All @@ -142,7 +143,7 @@ class mysql_connection : boost::noncopyable {
char* m_options[num_options];

public:
mysql_connection(const char* module_path, const char* name) : m_mysql(NULL)
mysql_connection(const char* module_path, const char* name, const char* dbname) : m_mysql(NULL)
{
if (0 == m_initialized) {
for (int ni = 0; ni < num_options; ++ni) {
Expand All @@ -156,7 +157,7 @@ class mysql_connection : boost::noncopyable {
if (0 == ni) {
sprintf(mem, pattern, name);
} else {
sprintf(mem, pattern, module_path);
sprintf(mem, pattern, module_path, dbname);
}
m_options[ni] = mem;
}
Expand Down Expand Up @@ -286,8 +287,12 @@ void clean_directory(const char* path)
string name(path);
name += fd.name;

if (0 != _unlink(name.c_str())) {
throw runtime_error(tmp_str("Unable to delete file \"%s\"", name.c_str()));
if (0 != fd.attrib & _A_SUBDIR) {
clean_directory((name + "/").c_str());
} else {
if (0 != _unlink(name.c_str())) {
throw runtime_error(tmp_str("Unable to delete file \"%s\"", name.c_str()));
}
}
}
} while (0 == _findnext(dir, &fd));
Expand All @@ -297,7 +302,7 @@ void clean_directory(const char* path)
}
}

void prepare_mysql(const char* path)
void prepare_mysql(const char* path, const string& dbname)
{
bool rc = true;
string config;
Expand All @@ -323,6 +328,14 @@ void prepare_mysql(const char* path)
throw runtime_error(tmp_str("Unable to create directory \"%s\"", config.c_str()));
}
}

config += "/dbtmp_" + dbname;

if (0 != _access(config.c_str(), 6)) {
if (0 != _mkdir(config.c_str())) {
throw runtime_error(tmp_str("Unable to create directory \"%s\"", config.c_str()));
}
}
}

string get_dump_date(const string& file)
Expand Down Expand Up @@ -1188,7 +1201,7 @@ int main(int argc, char* argv[])
("process", po::value< string >(), "What to process - \"fb2\", \"usr\", \"all\" (default: fb2)")
("strict", po::value< string >(), "What to put in INPX as file type - \"ext\", \"db\", \"ignore\" (default: ext). ext - use real file extension. db - use file type from database. ignore - ignore files with file extension not equal to file type")
("no-import", "Do not import dumps, just check dump time and use existing database")
("db-name", po::value< string >(), "Name of MYSQL database (default: librusec)")
("db-name", po::value< string >(), "Name of MYSQL database (default: flibusta)")
("archives", po::value< string >(), "Path(s) to off-line archives. Multiple entries should be separated by ';'. Each path must be valid and must point to some archives, or processing would be aborted. (If not present - entire database in converted for online usage)")
("read-fb2", po::value< string >(), "When archived book is not present in the database - try to parse fb2 in archive to get information. \"all\" - do it for all absent books, \"last\" - only process books with ids larger than last database id (If not present - no fb2 parsing)")
("prefer-fb2", po::value< string >(), "Try to parse fb2 in archive to get information (default: ignore). \"ignore\" - ignore fb2 information, \"merge\" - always prefer book sequence info from fb2, \"replace\" - always use book sequence info from fb2")
Expand Down Expand Up @@ -1498,7 +1511,7 @@ int main(int argc, char* argv[])
}

normalize_path(module_path);
prepare_mysql(module_path);
prepare_mysql(module_path, db_name);

if (vm.count("inpx")) {
inpx = vm["inpx"].as<string>();
Expand All @@ -1512,7 +1525,7 @@ int main(int argc, char* argv[])
{
string table_name = (e20111106 == g_format) ? "libavtors" : "libavtorname";

mysql_connection mysql(module_path, g_db_name.c_str());
mysql_connection mysql(module_path, g_db_name.c_str(), db_name.c_str());

if (!g_no_import) {
cout << endl << "Creating MYSQL database \"" << db_name << "\"" << endl << endl;
Expand Down Expand Up @@ -1741,24 +1754,18 @@ int main(int argc, char* argv[])
}

if (g_clean_when_done) {
string db_dir(module_path);
db_dir += "/data/";
db_dir += db_name;
db_dir += "/";
string data_dir(module_path);
data_dir += "/data/";

string db_tmp_dir = data_dir + "dbtmp_" + db_name + "/";
clean_directory(db_tmp_dir.c_str());
string db_dir = data_dir + db_name + "/";
clean_directory(db_dir.c_str());

string file_to_del = string(module_path) + "/data/auto.cnf";
if (0 != _unlink(file_to_del.c_str())) {
throw runtime_error(tmp_str("Unable to delete file \"%s\"", file_to_del.c_str()));
}
file_to_del = string(module_path) + "/data/ib_buffer_pool";
_unlink(file_to_del.c_str());
file_to_del = string(module_path) + "/data/ib_logfile0";
_unlink(file_to_del.c_str());
file_to_del = string(module_path) + "/data/ib_logfile1";
_unlink(file_to_del.c_str());
file_to_del = string(module_path) + "/data/ibdata1";
_unlink(file_to_del.c_str());
}

rc = 0;
Expand Down
93 changes: 47 additions & 46 deletions lib2inpx/libsplit.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,10 @@ namespace po = boost::program_options;

__int64 get_file_size(const string& name)
{
__int64 res = 0;
__int64 res = 0;
WIN32_FILE_ATTRIBUTE_DATA data;

if(GetFileAttributesEx(name.c_str(), GetFileExInfoStandard, &data)) {
if (GetFileAttributesEx(name.c_str(), GetFileExInfoStandard, &data)) {
res = ((__int64)(data.nFileSizeHigh)) * (0x100000000LL) + (__int64)(data.nFileSizeLow);
} else {
throw runtime_error(tmp_str("Unable to obtain file length \"%s\"", name.c_str()));
Expand All @@ -69,30 +69,31 @@ __int64 get_file_size(const string& name)

int main(int argc, char* argv[])
{
int rc = 1;
int rc = 1;

string from, to, spec;
string from, to, spec;

long rsize;
long rsize;

_finddata_t fd;
vector< long > files;
_finddata_t fd;
vector<long> files;

const char* file_name;
char module_path[ MAX_PATH + 1 ];
const char* file_name;
char module_path[MAX_PATH + 1];

__int64 archive_size;
__int64 archive_size;

long total_count = 0;
timer td;
long total_count = 0;
timer td;

bool f_binary = true;
bool f_binary = true;

try {
::GetModuleFileName(NULL, module_path, sizeof module_path);

file_name = separate_file_name(module_path);

// clang-format off
po::options_description options("options");
options.add_options()
("help", "Print help message")
Expand All @@ -101,13 +102,14 @@ int main(int argc, char* argv[])
("size", po::value<long>(&rsize)->default_value(2000), "Individual archive size in MB, if greater than 2GB - Zip64 archive will be created")
("text", "Open books in text mode")
;
// clang-format on

po::variables_map vm;

po::store(po::parse_command_line(argc, argv, options), vm);
po::notify(vm);

if(vm.count("help") || ! vm.count("from") || ! vm.count("to")) {
if (vm.count("help") || !vm.count("from") || !vm.count("to")) {
cout << endl;
cout << "Tool to prepare library archives" << endl;
cout << "Version " << PRJ_VERSION_MAJOR << "." << PRJ_VERSION_MINOR << endl;
Expand All @@ -126,42 +128,42 @@ int main(int argc, char* argv[])

archive_size = (__int64)1024 * 1024 * rsize;

if(vm.count("from")) {
from = vm[ "from" ].as< string >();
if (vm.count("from")) {
from = vm["from"].as<string>();
}

if(vm.count("to")) {
to = vm[ "to" ].as< string >();
if (vm.count("to")) {
to = vm["to"].as<string>();
}

if(! from.empty()) {
if(0 != _access(from.c_str(), 4)) {
if (!from.empty()) {
if (0 != _access(from.c_str(), 4)) {
throw runtime_error(tmp_str("Wrong path to books \"%s\"", from.c_str()));
}

normalize_path(from);
}

if(! to.empty()) {
if(0 != _access(to.c_str(), 6)) {
if (!to.empty()) {
if (0 != _access(to.c_str(), 6)) {
throw runtime_error(tmp_str("Wrong path to archives \"%s\"", to.c_str()));
}

normalize_path(to);
}

if(vm.count("text")) {
if (vm.count("text")) {
f_binary = false;
}

cout << endl << "Processing books...";

spec = from;
spec = from;
spec += "*.fb2";

auto_ffn books_dir(_findfirst(spec.c_str(), &fd));

if(! books_dir) {
if (!books_dir) {
throw runtime_error(tmp_str("Unable to process books path (%d) \"%s\"", errno, spec.c_str()));
}

Expand All @@ -170,12 +172,12 @@ int main(int argc, char* argv[])

name.erase(name.end() - 4, name.end());

if(is_numeric(name)) {
if (is_numeric(name)) {
files.push_back(atol(name.c_str()));
}
} while(0 == _findnext(books_dir, &fd));
} while (0 == _findnext(books_dir, &fd));

if(files.empty()) {
if (files.empty()) {
throw runtime_error("No books found!");
}

Expand All @@ -185,37 +187,37 @@ int main(int argc, char* argv[])

string temp_name = to + "temp.zip";

vector< long >::const_iterator it = files.begin();
string now = to_iso_extended_string(second_clock::universal_time());
vector<long>::const_iterator it = files.begin();
string now = to_iso_extended_string(second_clock::universal_time());

while(it != files.end()) {
while (it != files.end()) {
__int64 current_size = archive_size;
long count = 0;
timer ftd;
bool new_zip = true;
long count = 0;
timer ftd;
bool new_zip = true;

long book_start = *it;
long book_end = 0;

while((current_size > 0) && (it != files.end())) {
while ((current_size > 0) && (it != files.end())) {
zip zz(temp_name, now, new_zip);

new_zip = false;

for(; it != files.end(); ++it) {
for (; it != files.end(); ++it) {
tmp_str book_name("%d.fb2", *it);
ifstream in(from + book_name, f_binary ? ios_base::in | ios_base::binary : ios_base::in);
stringstream ss;

book_end = *it;

if(!in) {
if (!in) {
throw runtime_error(tmp_str("Cannot open book file \"%s\"", book_name.c_str()));
}

ss << in.rdbuf();

if(!in && !in.eof()) {
if (!in && !in.eof()) {
throw runtime_error(tmp_str("Problem reading book file \"%s\"", book_name.c_str()));
}

Expand All @@ -224,9 +226,9 @@ int main(int argc, char* argv[])

count++;

current_size -= ss.str().size() ;
current_size -= ss.str().size();

if(current_size <= 0) {
if (current_size <= 0) {
++it;
break;
}
Expand All @@ -237,14 +239,13 @@ int main(int argc, char* argv[])
current_size = archive_size - get_file_size(temp_name);
}

if(0 != rename(temp_name.c_str(), tmp_str("%sfb2-%06d-%06d.zip", to.c_str(), book_start, book_end))) {
if (0 != rename(temp_name.c_str(), tmp_str("%sfb2-%06d-%06d.zip", to.c_str(), book_start, book_end))) {
throw runtime_error(tmp_str("Unable to rename temp.zip to \"fb2-%06d-%06d.zip\"", book_start, book_end));
}

cout << endl << "Processed: " << setw(8) << count << " books. Range: "
<< setw(8) << book_start << " - "
<< setw(8) << book_end << ". Done in - "
<< ftd.passed();
cout << endl
<< "Processed: " << setw(8) << count << " books. Range: " << setw(8) << book_start << " - " << setw(8) << book_end
<< ". Done in - " << ftd.passed();

new_zip = true;

Expand All @@ -254,7 +255,7 @@ int main(int argc, char* argv[])
cout << endl << endl << "Processing of " << total_count << " books completed in " << td.passed() << endl;

rc = 0;
} catch(exception& e) {
} catch (exception& e) {
cerr << endl << endl << "***ERROR: " << e.what() << endl;
}

Expand Down
Loading

0 comments on commit 006785a

Please sign in to comment.