void DatabaseOrdinary::alterTable(
const Context & context,
const String & name,
const ColumnsDescription & columns,
const ASTModifier & storage_modifier)
{
/// Read the definition of the table and replace the necessary parts with new ones.
String table_name_escaped = escapeForFileName(name);
String table_metadata_tmp_path = metadata_path + "/" + table_name_escaped + ".sql.tmp";
String table_metadata_path = metadata_path + "/" + table_name_escaped + ".sql";
String statement;
{
char in_buf[METADATA_FILE_BUFFER_SIZE];
ReadBufferFromFile in(table_metadata_path, METADATA_FILE_BUFFER_SIZE, -1, in_buf);
readStringUntilEOF(statement, in);
}
ParserCreateQuery parser;
ASTPtr ast = parseQuery(parser, statement.data(), statement.data() + statement.size(), "in file " + table_metadata_path, 0);
ASTCreateQuery & ast_create_query = typeid_cast<ASTCreateQuery &>(*ast);
ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns);
ast_create_query.replace(ast_create_query.columns, new_columns);
if (storage_modifier)
storage_modifier(*ast_create_query.storage);
statement = getTableDefinitionFromCreateQuery(ast);
{
WriteBufferFromFile out(table_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL);
writeString(statement, out);
out.next();
if (context.getSettingsRef().fsync_metadata)
out.sync();
out.close();
}
try
{
/// rename atomically replaces the old file with the new one.
Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path);
}
catch (...)
{
Poco::File(table_metadata_tmp_path).remove();
throw;
}
}
void loadMetadata(Context & context)
{
String path = context.getPath() + "metadata";
/** There may exist 'force_restore_data' file, that means,
* skip safety threshold on difference of data parts while initializing tables.
* This file is deleted after successful loading of tables.
* (flag is "one-shot")
*/
Poco::File force_restore_data_flag_file(context.getFlagsPath() + "force_restore_data");
bool has_force_restore_data_flag = force_restore_data_flag_file.exists();
/// For parallel tables loading.
ThreadPool thread_pool(SettingMaxThreads().getAutoValue());
/// Loop over databases.
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator it(path); it != dir_end; ++it)
{
if (!it->isDirectory())
continue;
/// For '.svn', '.gitignore' directory and similar.
if (it.name().at(0) == '.')
continue;
String database = unescapeForFileName(it.name());
/// There may exist .sql file with database creation statement.
/// Or, if it is absent, then database with default engine is created.
String database_attach_query;
String database_metadata_file = it->path() + ".sql";
if (Poco::File(database_metadata_file).exists())
{
ReadBufferFromFile in(database_metadata_file, 1024);
readStringUntilEOF(database_attach_query, in);
}
else
database_attach_query = "ATTACH DATABASE " + backQuoteIfNeed(database);
executeCreateQuery(database_attach_query, context, database, it->path(), thread_pool, has_force_restore_data_flag);
}
thread_pool.wait();
if (has_force_restore_data_flag)
force_restore_data_flag_file.remove();
}
static void loadTable(
Context & context,
const String & database_metadata_path,
DatabaseOrdinary & database,
const String & database_name,
const String & database_data_path,
const String & file_name,
bool has_force_restore_data_flag)
{
Logger * log = &Logger::get("loadTable");
const String table_metadata_path = database_metadata_path + "/" + file_name;
String s;
{
char in_buf[METADATA_FILE_BUFFER_SIZE];
ReadBufferFromFile in(table_metadata_path, METADATA_FILE_BUFFER_SIZE, -1, in_buf);
readStringUntilEOF(s, in);
}
/** Empty files with metadata are generated after a rough restart of the server.
* Remove these files to slightly reduce the work of the admins on startup.
*/
if (s.empty())
{
LOG_ERROR(log, "File " << table_metadata_path << " is empty. Removing.");
Poco::File(table_metadata_path).remove();
return;
}
try
{
String table_name;
StoragePtr table;
std::tie(table_name, table) = createTableFromDefinition(
s, database_name, database_data_path, context, has_force_restore_data_flag, "in file " + table_metadata_path);
database.attachTable(table_name, table);
}
catch (const Exception & e)
{
throw Exception("Cannot create table from metadata file " + table_metadata_path + ", error: " + e.displayText() +
", stack trace:\n" + e.getStackTrace().toString(),
ErrorCodes::CANNOT_CREATE_TABLE_FROM_METADATA);
}
}
void loadMetadata(Context & context)
{
String path = context.getPath() + "metadata";
/// Используется для параллельной загрузки таблиц.
ThreadPool thread_pool(SettingMaxThreads().getAutoValue());
/// Цикл по базам данных
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator it(path); it != dir_end; ++it)
{
if (!it->isDirectory())
continue;
/// Для директории .svn
if (it.name().at(0) == '.')
continue;
String database = unescapeForFileName(it.name());
/// Для базы данных может быть расположен .sql файл, где описан запрос на её создание.
/// А если такого файла нет, то создаётся база данных с движком по-умолчанию.
String database_attach_query;
String database_metadata_file = it->path() + ".sql";
if (Poco::File(database_metadata_file).exists())
{
ReadBufferFromFile in(database_metadata_file, 1024);
readStringUntilEOF(database_attach_query, in);
}
else
database_attach_query = "ATTACH DATABASE " + backQuoteIfNeed(database);
executeCreateQuery(database_attach_query, context, database, it->path(), thread_pool);
}
thread_pool.wait();
}
static ASTPtr getQueryFromMetadata(const String & metadata_path, bool throw_on_error = true)
{
if (!Poco::File(metadata_path).exists())
return nullptr;
String query;
{
ReadBufferFromFile in(metadata_path, 4096);
readStringUntilEOF(query, in);
}
ParserCreateQuery parser;
const char * pos = query.data();
std::string error_message;
auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false,
"in file " + metadata_path, /* allow_multi_statements = */ false, 0);
if (!ast && throw_on_error)
throw Exception(error_message, ErrorCodes::SYNTAX_ERROR);
return ast;
}
void Compiler::compile(
HashedKey hashed_key,
std::string file_name,
const std::string & additional_compiler_flags,
CodeGenerator get_code,
ReadyCallback on_ready)
{
ProfileEvents::increment(ProfileEvents::CompileAttempt);
std::string prefix = path + "/" + file_name;
std::string cpp_file_path = prefix + ".cpp";
std::string so_file_path = prefix + ".so";
std::string so_tmp_file_path = prefix + ".so.tmp";
{
WriteBufferFromFile out(cpp_file_path);
out << get_code();
}
std::stringstream command;
/// Slightly uncomfortable.
command <<
"LD_LIBRARY_PATH=" PATH_SHARE "/clickhouse/bin/"
" " INTERNAL_COMPILER_EXECUTABLE
" -B " PATH_SHARE "/clickhouse/bin/"
" " INTERNAL_COMPILER_FLAGS
#if INTERNAL_COMPILER_CUSTOM_ROOT
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/include/"
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/"
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/c++/*/"
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/x86_64-linux-gnu/"
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/x86_64-linux-gnu/c++/*/"
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/lib/clang/*/include/"
#endif
" -I " INTERNAL_COMPILER_HEADERS "/dbms/src/"
" -I " INTERNAL_COMPILER_HEADERS "/contrib/libcityhash/include/"
" -I " INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR
" -I " INTERNAL_Poco_Foundation_INCLUDE_DIR
" -I " INTERNAL_Boost_INCLUDE_DIRS
" -I " INTERNAL_COMPILER_HEADERS "/libs/libcommon/include/"
" " << additional_compiler_flags <<
" -o " << so_tmp_file_path << " " << cpp_file_path
<< " 2>&1 || echo Exit code: $?";
std::string compile_result;
{
auto process = ShellCommand::execute(command.str());
readStringUntilEOF(compile_result, process->out);
process->wait();
}
if (!compile_result.empty())
throw Exception("Cannot compile code:\n\n" + command.str() + "\n\n" + compile_result);
/// If there was an error before, the file with the code remains for viewing.
Poco::File(cpp_file_path).remove();
Poco::File(so_tmp_file_path).renameTo(so_file_path);
SharedLibraryPtr lib(new SharedLibrary(so_file_path));
{
std::lock_guard<std::mutex> lock(mutex);
libraries[hashed_key] = lib;
}
LOG_INFO(log, "Compiled code " << file_name);
ProfileEvents::increment(ProfileEvents::CompileSuccess);
on_ready(lib);
}
void Compiler::compile(
HashedKey hashed_key,
std::string file_name,
const std::string & additional_compiler_flags,
CodeGenerator get_code,
ReadyCallback on_ready)
{
ProfileEvents::increment(ProfileEvents::CompileAttempt);
std::string prefix = path + "/" + file_name;
std::string cpp_file_path = prefix + ".cpp";
std::string so_file_path = prefix + ".so";
std::string so_tmp_file_path = prefix + ".so.tmp";
{
WriteBufferFromFile out(cpp_file_path);
out << get_code();
}
std::stringstream command;
/// Слегка неудобно.
command <<
"LD_LIBRARY_PATH=/usr/share/clickhouse/bin/"
" /usr/share/clickhouse/bin/clang"
" -B /usr/share/clickhouse/bin/"
" -x c++ -std=gnu++1y -O3 -g -Wall -Werror -Wnon-virtual-dtor -march=native -msse4 -mpopcnt -D NDEBUG"
#if _GLIBCXX_USE_CXX11_ABI == 0
" -D_GLIBCXX_USE_CXX11_ABI=0"
#elif _GLIBCXX_USE_CXX11_ABI == 1
" -D_GLIBCXX_USE_CXX11_ABI=1"
#endif
" -shared -fPIC -fvisibility=hidden -fno-implement-inlines"
" -isystem /usr/share/clickhouse/headers/usr/local/include/"
" -isystem /usr/share/clickhouse/headers/usr/include/"
" -isystem /usr/share/clickhouse/headers/usr/include/mysql/"
" -isystem /usr/share/clickhouse/headers/usr/include/c++/*/"
" -isystem /usr/share/clickhouse/headers/usr/include/x86_64-linux-gnu/"
" -isystem /usr/share/clickhouse/headers/usr/include/x86_64-linux-gnu/c++/*/"
" -isystem /usr/share/clickhouse/headers/usr/local/lib/clang/*/include/"
" -I /usr/share/clickhouse/headers/dbms/include/"
" -I /usr/share/clickhouse/headers/contrib/libcityhash/include/"
" -I /usr/share/clickhouse/headers/contrib/libdouble-conversion/"
" -I /usr/share/clickhouse/headers/contrib/libpoco/Foundation/include/"
" -I /usr/share/clickhouse/headers/contrib/libpoco/Util/include/"
" -I /usr/share/clickhouse/headers/contrib/libboost/boost_1_62_0/"
" -I /usr/share/clickhouse/headers/libs/libcommon/include/"
" -I /usr/share/clickhouse/headers/libs/libmysqlxx/include/"
" " << additional_compiler_flags <<
" -o " << so_tmp_file_path << " " << cpp_file_path
<< " 2>&1 || echo Exit code: $?";
std::string compile_result;
{
auto process = ShellCommand::execute(command.str());
readStringUntilEOF(compile_result, process->out);
process->wait();
}
if (!compile_result.empty())
throw Exception("Cannot compile code:\n\n" + command.str() + "\n\n" + compile_result);
/// Если до этого была ошибка, то файл с кодом остаётся для возможности просмотра.
Poco::File(cpp_file_path).remove();
Poco::File(so_tmp_file_path).renameTo(so_file_path);
SharedLibraryPtr lib(new SharedLibrary(so_file_path));
{
std::lock_guard<std::mutex> lock(mutex);
libraries[hashed_key] = lib;
}
LOG_INFO(log, "Compiled code " << file_name);
ProfileEvents::increment(ProfileEvents::CompileSuccess);
on_ready(lib);
}
