void FileChecker::load(Map & map) const
{
map.clear();
if (!Poco::File(files_info_path).exists())
return;
std::string content;
{
ReadBufferFromFile in(files_info_path);
WriteBufferFromString out(content);
/// Библиотека JSON не поддерживает пробельные символы. Удаляем их. Неэффективно.
while (!in.eof())
{
char c;
readChar(c, in);
if (!isspace(c))
writeChar(c, out);
}
}
JSON json(content);
JSON files = json["yandex"];
for (const auto & name_value : files)
map[unescapeForFileName(name_value.getName())] = name_value.getValue()["size"].toUInt();
}
ConnectionPoolPtrs createPoolsForAddresses(const std::string & name, PoolFactory && factory)
{
ConnectionPoolPtrs pools;
for (auto it = boost::make_split_iterator(name, boost::first_finder(",")); it != decltype(it){}; ++it)
{
const auto address = boost::copy_range<std::string>(*it);
const char * address_begin = static_cast<const char*>(address.data());
const char * address_end = address_begin + address.size();
const char * user_pw_end = strchr(address.data(), '@');
const char * colon = strchr(address.data(), ':');
if (!user_pw_end || !colon)
throw Exception{
"Shard address '" + address + "' does not match to 'user[:password]@host:port#default_database' pattern",
ErrorCodes::INCORRECT_FILE_NAME};
const bool has_pw = colon < user_pw_end;
const char * host_end = has_pw ? strchr(user_pw_end + 1, ':') : colon;
if (!host_end)
throw Exception{
"Shard address '" + address + "' does not contain port",
ErrorCodes::INCORRECT_FILE_NAME};
const char * has_db = strchr(address.data(), '#');
const char * port_end = has_db ? has_db : address_end;
const auto user = unescapeForFileName(std::string(address_begin, has_pw ? colon : user_pw_end));
const auto password = has_pw ? unescapeForFileName(std::string(colon + 1, user_pw_end)) : std::string();
const auto host = unescapeForFileName(std::string(user_pw_end + 1, host_end));
const auto port = parse<UInt16>(host_end + 1, port_end - (host_end + 1));
const auto database = has_db ? unescapeForFileName(std::string(has_db + 1, address_end))
: std::string();
pools.emplace_back(factory(host, port, user, password, database));
}
return pools;
}
void loadMetadata(Context & context)
{
String path = context.getPath() + "metadata";
/** There may exist 'force_restore_data' file, that means,
* skip safety threshold on difference of data parts while initializing tables.
* This file is deleted after successful loading of tables.
* (flag is "one-shot")
*/
Poco::File force_restore_data_flag_file(context.getFlagsPath() + "force_restore_data");
bool has_force_restore_data_flag = force_restore_data_flag_file.exists();
/// For parallel tables loading.
ThreadPool thread_pool(SettingMaxThreads().getAutoValue());
/// Loop over databases.
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator it(path); it != dir_end; ++it)
{
if (!it->isDirectory())
continue;
/// For '.svn', '.gitignore' directory and similar.
if (it.name().at(0) == '.')
continue;
String database = unescapeForFileName(it.name());
/// There may exist .sql file with database creation statement.
/// Or, if it is absent, then database with default engine is created.
String database_attach_query;
String database_metadata_file = it->path() + ".sql";
if (Poco::File(database_metadata_file).exists())
{
ReadBufferFromFile in(database_metadata_file, 1024);
readStringUntilEOF(database_attach_query, in);
}
else
database_attach_query = "ATTACH DATABASE " + backQuoteIfNeed(database);
executeCreateQuery(database_attach_query, context, database, it->path(), thread_pool, has_force_restore_data_flag);
}
thread_pool.wait();
if (has_force_restore_data_flag)
force_restore_data_flag_file.remove();
}
void loadMetadata(Context & context)
{
String path = context.getPath() + "metadata";
/// Используется для параллельной загрузки таблиц.
ThreadPool thread_pool(SettingMaxThreads().getAutoValue());
/// Цикл по базам данных
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator it(path); it != dir_end; ++it)
{
if (!it->isDirectory())
continue;
/// Для директории .svn
if (it.name().at(0) == '.')
continue;
String database = unescapeForFileName(it.name());
/// Для базы данных может быть расположен .sql файл, где описан запрос на её создание.
/// А если такого файла нет, то создаётся база данных с движком по-умолчанию.
String database_attach_query;
String database_metadata_file = it->path() + ".sql";
if (Poco::File(database_metadata_file).exists())
{
ReadBufferFromFile in(database_metadata_file, 1024);
readStringUntilEOF(database_attach_query, in);
}
else
database_attach_query = "ATTACH DATABASE " + backQuoteIfNeed(database);
executeCreateQuery(database_attach_query, context, database, it->path(), thread_pool);
}
thread_pool.wait();
}