void DatabaseOrdinary::alterTable( const Context & context, const String & name, const ColumnsDescription & columns, const ASTModifier & storage_modifier) { /// Read the definition of the table and replace the necessary parts with new ones. String table_name_escaped = escapeForFileName(name); String table_metadata_tmp_path = metadata_path + "/" + table_name_escaped + ".sql.tmp"; String table_metadata_path = metadata_path + "/" + table_name_escaped + ".sql"; String statement; { char in_buf[METADATA_FILE_BUFFER_SIZE]; ReadBufferFromFile in(table_metadata_path, METADATA_FILE_BUFFER_SIZE, -1, in_buf); readStringUntilEOF(statement, in); } ParserCreateQuery parser; ASTPtr ast = parseQuery(parser, statement.data(), statement.data() + statement.size(), "in file " + table_metadata_path, 0); ASTCreateQuery & ast_create_query = typeid_cast<ASTCreateQuery &>(*ast); ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns); ast_create_query.replace(ast_create_query.columns, new_columns); if (storage_modifier) storage_modifier(*ast_create_query.storage); statement = getTableDefinitionFromCreateQuery(ast); { WriteBufferFromFile out(table_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL); writeString(statement, out); out.next(); if (context.getSettingsRef().fsync_metadata) out.sync(); out.close(); } try { /// rename atomically replaces the old file with the new one. Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path); } catch (...) { Poco::File(table_metadata_tmp_path).remove(); throw; } }
void loadMetadata(Context & context) { String path = context.getPath() + "metadata"; /** There may exist 'force_restore_data' file, that means, * skip safety threshold on difference of data parts while initializing tables. * This file is deleted after successful loading of tables. * (flag is "one-shot") */ Poco::File force_restore_data_flag_file(context.getFlagsPath() + "force_restore_data"); bool has_force_restore_data_flag = force_restore_data_flag_file.exists(); /// For parallel tables loading. ThreadPool thread_pool(SettingMaxThreads().getAutoValue()); /// Loop over databases. Poco::DirectoryIterator dir_end; for (Poco::DirectoryIterator it(path); it != dir_end; ++it) { if (!it->isDirectory()) continue; /// For '.svn', '.gitignore' directory and similar. if (it.name().at(0) == '.') continue; String database = unescapeForFileName(it.name()); /// There may exist .sql file with database creation statement. /// Or, if it is absent, then database with default engine is created. String database_attach_query; String database_metadata_file = it->path() + ".sql"; if (Poco::File(database_metadata_file).exists()) { ReadBufferFromFile in(database_metadata_file, 1024); readStringUntilEOF(database_attach_query, in); } else database_attach_query = "ATTACH DATABASE " + backQuoteIfNeed(database); executeCreateQuery(database_attach_query, context, database, it->path(), thread_pool, has_force_restore_data_flag); } thread_pool.wait(); if (has_force_restore_data_flag) force_restore_data_flag_file.remove(); }
static void loadTable( Context & context, const String & database_metadata_path, DatabaseOrdinary & database, const String & database_name, const String & database_data_path, const String & file_name, bool has_force_restore_data_flag) { Logger * log = &Logger::get("loadTable"); const String table_metadata_path = database_metadata_path + "/" + file_name; String s; { char in_buf[METADATA_FILE_BUFFER_SIZE]; ReadBufferFromFile in(table_metadata_path, METADATA_FILE_BUFFER_SIZE, -1, in_buf); readStringUntilEOF(s, in); } /** Empty files with metadata are generated after a rough restart of the server. * Remove these files to slightly reduce the work of the admins on startup. */ if (s.empty()) { LOG_ERROR(log, "File " << table_metadata_path << " is empty. Removing."); Poco::File(table_metadata_path).remove(); return; } try { String table_name; StoragePtr table; std::tie(table_name, table) = createTableFromDefinition( s, database_name, database_data_path, context, has_force_restore_data_flag, "in file " + table_metadata_path); database.attachTable(table_name, table); } catch (const Exception & e) { throw Exception("Cannot create table from metadata file " + table_metadata_path + ", error: " + e.displayText() + ", stack trace:\n" + e.getStackTrace().toString(), ErrorCodes::CANNOT_CREATE_TABLE_FROM_METADATA); } }
void loadMetadata(Context & context) { String path = context.getPath() + "metadata"; /// Используется для параллельной загрузки таблиц. ThreadPool thread_pool(SettingMaxThreads().getAutoValue()); /// Цикл по базам данных Poco::DirectoryIterator dir_end; for (Poco::DirectoryIterator it(path); it != dir_end; ++it) { if (!it->isDirectory()) continue; /// Для директории .svn if (it.name().at(0) == '.') continue; String database = unescapeForFileName(it.name()); /// Для базы данных может быть расположен .sql файл, где описан запрос на её создание. /// А если такого файла нет, то создаётся база данных с движком по-умолчанию. String database_attach_query; String database_metadata_file = it->path() + ".sql"; if (Poco::File(database_metadata_file).exists()) { ReadBufferFromFile in(database_metadata_file, 1024); readStringUntilEOF(database_attach_query, in); } else database_attach_query = "ATTACH DATABASE " + backQuoteIfNeed(database); executeCreateQuery(database_attach_query, context, database, it->path(), thread_pool); } thread_pool.wait(); }
static ASTPtr getQueryFromMetadata(const String & metadata_path, bool throw_on_error = true) { if (!Poco::File(metadata_path).exists()) return nullptr; String query; { ReadBufferFromFile in(metadata_path, 4096); readStringUntilEOF(query, in); } ParserCreateQuery parser; const char * pos = query.data(); std::string error_message; auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false, "in file " + metadata_path, /* allow_multi_statements = */ false, 0); if (!ast && throw_on_error) throw Exception(error_message, ErrorCodes::SYNTAX_ERROR); return ast; }
void Compiler::compile( HashedKey hashed_key, std::string file_name, const std::string & additional_compiler_flags, CodeGenerator get_code, ReadyCallback on_ready) { ProfileEvents::increment(ProfileEvents::CompileAttempt); std::string prefix = path + "/" + file_name; std::string cpp_file_path = prefix + ".cpp"; std::string so_file_path = prefix + ".so"; std::string so_tmp_file_path = prefix + ".so.tmp"; { WriteBufferFromFile out(cpp_file_path); out << get_code(); } std::stringstream command; /// Slightly uncomfortable. command << "LD_LIBRARY_PATH=" PATH_SHARE "/clickhouse/bin/" " " INTERNAL_COMPILER_EXECUTABLE " -B " PATH_SHARE "/clickhouse/bin/" " " INTERNAL_COMPILER_FLAGS #if INTERNAL_COMPILER_CUSTOM_ROOT " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/include/" " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/" " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/c++/*/" " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/x86_64-linux-gnu/" " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/x86_64-linux-gnu/c++/*/" " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/lib/clang/*/include/" #endif " -I " INTERNAL_COMPILER_HEADERS "/dbms/src/" " -I " INTERNAL_COMPILER_HEADERS "/contrib/libcityhash/include/" " -I " INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR " -I " INTERNAL_Poco_Foundation_INCLUDE_DIR " -I " INTERNAL_Boost_INCLUDE_DIRS " -I " INTERNAL_COMPILER_HEADERS "/libs/libcommon/include/" " " << additional_compiler_flags << " -o " << so_tmp_file_path << " " << cpp_file_path << " 2>&1 || echo Exit code: $?"; std::string compile_result; { auto process = ShellCommand::execute(command.str()); readStringUntilEOF(compile_result, process->out); process->wait(); } if (!compile_result.empty()) throw Exception("Cannot compile code:\n\n" + command.str() + "\n\n" + compile_result); /// If there was an error before, the file with the code remains for viewing. Poco::File(cpp_file_path).remove(); Poco::File(so_tmp_file_path).renameTo(so_file_path); SharedLibraryPtr lib(new SharedLibrary(so_file_path)); { std::lock_guard<std::mutex> lock(mutex); libraries[hashed_key] = lib; } LOG_INFO(log, "Compiled code " << file_name); ProfileEvents::increment(ProfileEvents::CompileSuccess); on_ready(lib); }
void Compiler::compile( HashedKey hashed_key, std::string file_name, const std::string & additional_compiler_flags, CodeGenerator get_code, ReadyCallback on_ready) { ProfileEvents::increment(ProfileEvents::CompileAttempt); std::string prefix = path + "/" + file_name; std::string cpp_file_path = prefix + ".cpp"; std::string so_file_path = prefix + ".so"; std::string so_tmp_file_path = prefix + ".so.tmp"; { WriteBufferFromFile out(cpp_file_path); out << get_code(); } std::stringstream command; /// Слегка неудобно. command << "LD_LIBRARY_PATH=/usr/share/clickhouse/bin/" " /usr/share/clickhouse/bin/clang" " -B /usr/share/clickhouse/bin/" " -x c++ -std=gnu++1y -O3 -g -Wall -Werror -Wnon-virtual-dtor -march=native -msse4 -mpopcnt -D NDEBUG" #if _GLIBCXX_USE_CXX11_ABI == 0 " -D_GLIBCXX_USE_CXX11_ABI=0" #elif _GLIBCXX_USE_CXX11_ABI == 1 " -D_GLIBCXX_USE_CXX11_ABI=1" #endif " -shared -fPIC -fvisibility=hidden -fno-implement-inlines" " -isystem /usr/share/clickhouse/headers/usr/local/include/" " -isystem /usr/share/clickhouse/headers/usr/include/" " -isystem /usr/share/clickhouse/headers/usr/include/mysql/" " -isystem /usr/share/clickhouse/headers/usr/include/c++/*/" " -isystem /usr/share/clickhouse/headers/usr/include/x86_64-linux-gnu/" " -isystem /usr/share/clickhouse/headers/usr/include/x86_64-linux-gnu/c++/*/" " -isystem /usr/share/clickhouse/headers/usr/local/lib/clang/*/include/" " -I /usr/share/clickhouse/headers/dbms/include/" " -I /usr/share/clickhouse/headers/contrib/libcityhash/include/" " -I /usr/share/clickhouse/headers/contrib/libdouble-conversion/" " -I /usr/share/clickhouse/headers/contrib/libpoco/Foundation/include/" " -I /usr/share/clickhouse/headers/contrib/libpoco/Util/include/" " -I /usr/share/clickhouse/headers/contrib/libboost/boost_1_62_0/" " -I /usr/share/clickhouse/headers/libs/libcommon/include/" " -I /usr/share/clickhouse/headers/libs/libmysqlxx/include/" " " << additional_compiler_flags << " -o " << so_tmp_file_path << " " << cpp_file_path << " 2>&1 || echo Exit code: $?"; std::string compile_result; { auto process = ShellCommand::execute(command.str()); readStringUntilEOF(compile_result, process->out); process->wait(); } if (!compile_result.empty()) throw Exception("Cannot compile code:\n\n" + command.str() + "\n\n" + compile_result); /// Если до этого была ошибка, то файл с кодом остаётся для возможности просмотра. Poco::File(cpp_file_path).remove(); Poco::File(so_tmp_file_path).renameTo(so_file_path); SharedLibraryPtr lib(new SharedLibrary(so_file_path)); { std::lock_guard<std::mutex> lock(mutex); libraries[hashed_key] = lib; } LOG_INFO(log, "Compiled code " << file_name); ProfileEvents::increment(ProfileEvents::CompileSuccess); on_ready(lib); }