void DataTypeArray::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
/// There is no good way to serialize an array in CSV. Therefore, we serialize it into a string, and then write the resulting string in CSV.
WriteBufferFromOwnString wb;
serializeText(column, row_num, wb, settings);
writeCSV(wb.str(), ostr);
}
String ASTFunction::getColumnNameImpl() const
{
WriteBufferFromOwnString wb;
writeString(name, wb);
if (parameters)
{
writeChar('(', wb);
for (ASTs::const_iterator it = parameters->children.begin(); it != parameters->children.end(); ++it)
{
if (it != parameters->children.begin())
writeCString(", ", wb);
writeString((*it)->getColumnName(), wb);
}
writeChar(')', wb);
}
writeChar('(', wb);
for (ASTs::const_iterator it = arguments->children.begin(); it != arguments->children.end(); ++it)
{
if (it != arguments->children.begin())
writeCString(", ", wb);
writeString((*it)->getColumnName(), wb);
}
writeChar(')', wb);
return wb.str();
}
std::string DataTypeDateTime::getName() const
{
if (!has_explicit_time_zone)
return "DateTime";
WriteBufferFromOwnString out;
out << "DateTime(" << quote << time_zone.getTimeZone() << ")";
return out.str();
}
String CSVRowInputStream::getDiagnosticInfo()
{
if (istr.eof()) /// Buffer has gone, cannot extract information about what has been parsed.
return {};
WriteBufferFromOwnString out;
MutableColumns columns = header.cloneEmptyColumns();
/// It is possible to display detailed diagnostics only if the last and next to last rows are still in the read buffer.
size_t bytes_read_at_start_of_buffer = istr.count() - istr.offset();
if (bytes_read_at_start_of_buffer != bytes_read_at_start_of_buffer_on_prev_row)
{
out << "Could not print diagnostic info because two last rows aren't in buffer (rare case)\n";
return out.str();
}
size_t max_length_of_column_name = 0;
for (size_t i = 0; i < header.columns(); ++i)
if (header.safeGetByPosition(i).name.size() > max_length_of_column_name)
max_length_of_column_name = header.safeGetByPosition(i).name.size();
size_t max_length_of_data_type_name = 0;
for (size_t i = 0; i < header.columns(); ++i)
if (header.safeGetByPosition(i).type->getName().size() > max_length_of_data_type_name)
max_length_of_data_type_name = header.safeGetByPosition(i).type->getName().size();
/// Roll back the cursor to the beginning of the previous or current row and parse all over again. But now we derive detailed information.
if (pos_of_prev_row)
{
istr.position() = pos_of_prev_row;
out << "\nRow " << (row_num - 1) << ":\n";
if (!parseRowAndPrintDiagnosticInfo(columns, out, max_length_of_column_name, max_length_of_data_type_name))
return out.str();
}
else
{
if (!pos_of_current_row)
{
out << "Could not print diagnostic info because parsing of data hasn't started.\n";
return out.str();
}
istr.position() = pos_of_current_row;
}
out << "\nRow " << row_num << ":\n";
parseRowAndPrintDiagnosticInfo(columns, out, max_length_of_column_name, max_length_of_data_type_name);
out << "\n";
return out.str();
}
void StorageSystemParts::processNextStorage(MutableColumns & columns, const StoragesInfo & info, bool has_state_column)
{
using State = MergeTreeDataPart::State;
for (size_t part_number = 0; part_number < info.all_parts.size(); ++part_number)
{
const auto & part = info.all_parts[part_number];
auto part_state = info.all_parts_state[part_number];
MergeTreeDataPart::ColumnSize columns_size = part->getTotalColumnsSize();
size_t i = 0;
{
WriteBufferFromOwnString out;
part->partition.serializeText(*info.data, out, format_settings);
columns[i++]->insert(out.str());
}
columns[i++]->insert(part->name);
columns[i++]->insert(part_state == State::Committed);
columns[i++]->insert(part->marks_count);
columns[i++]->insert(part->rows_count);
columns[i++]->insert(part->bytes_on_disk.load(std::memory_order_relaxed));
columns[i++]->insert(columns_size.data_compressed);
columns[i++]->insert(columns_size.data_uncompressed);
columns[i++]->insert(columns_size.marks);
columns[i++]->insert(static_cast<UInt64>(part->modification_time));
time_t remove_time = part->remove_time.load(std::memory_order_relaxed);
columns[i++]->insert(static_cast<UInt64>(remove_time == std::numeric_limits<time_t>::max() ? 0 : remove_time));
/// For convenience, in returned refcount, don't add references that was due to local variables in this method: all_parts, active_parts.
columns[i++]->insert(static_cast<UInt64>(part.use_count() - 1));
columns[i++]->insert(part->getMinDate());
columns[i++]->insert(part->getMaxDate());
columns[i++]->insert(part->getMinTime());
columns[i++]->insert(part->getMaxTime());
columns[i++]->insert(part->info.partition_id);
columns[i++]->insert(part->info.min_block);
columns[i++]->insert(part->info.max_block);
columns[i++]->insert(part->info.level);
columns[i++]->insert(static_cast<UInt64>(part->info.getDataVersion()));
columns[i++]->insert(part->getIndexSizeInBytes());
columns[i++]->insert(part->getIndexSizeInAllocatedBytes());
columns[i++]->insert(info.database);
columns[i++]->insert(info.table);
columns[i++]->insert(info.engine);
columns[i++]->insert(part->getFullPath());
if (has_state_column)
columns[i++]->insert(part->stateString());
}
}
TSKVRowOutputStream::TSKVRowOutputStream(WriteBuffer & ostr_, const Block & sample_, const FormatSettings & format_settings_)
: TabSeparatedRowOutputStream(ostr_, sample_, false, false, format_settings_)
{
NamesAndTypesList columns(sample_.getNamesAndTypesList());
fields.assign(columns.begin(), columns.end());
for (auto & field : fields)
{
WriteBufferFromOwnString wb;
writeAnyEscapedString<'='>(field.name.data(), field.name.data() + field.name.size(), wb);
writeCString("=", wb);
field.name = wb.str();
}
}
String IColumn::dumpStructure() const
{
WriteBufferFromOwnString res;
res << getFamilyName() << "(size = " << size();
ColumnCallback callback = [&](ColumnPtr & subcolumn)
{
res << ", " << subcolumn->dumpStructure();
};
const_cast<IColumn*>(this)->forEachSubcolumn(callback);
res << ")";
return res.str();
}
String IAST::getTreeID() const
{
WriteBufferFromOwnString out;
getTreeIDImpl(out);
return out.str();
}
std::string ExternalQueryBuilder::composeLoadAllQuery() const
{
WriteBufferFromOwnString out;
writeString("SELECT ", out);
if (dict_struct.id)
{
if (!dict_struct.id->expression.empty())
{
writeParenthesisedString(dict_struct.id->expression, out);
writeString(" AS ", out);
}
writeQuoted(dict_struct.id->name, out);
if (dict_struct.range_min && dict_struct.range_max)
{
writeString(", ", out);
if (!dict_struct.range_min->expression.empty())
{
writeParenthesisedString(dict_struct.range_min->expression, out);
writeString(" AS ", out);
}
writeQuoted(dict_struct.range_min->name, out);
writeString(", ", out);
if (!dict_struct.range_max->expression.empty())
{
writeParenthesisedString(dict_struct.range_max->expression, out);
writeString(" AS ", out);
}
writeQuoted(dict_struct.range_max->name, out);
}
}
else if (dict_struct.key)
{
auto first = true;
for (const auto & key : *dict_struct.key)
{
if (!first)
writeString(", ", out);
first = false;
if (!key.expression.empty())
{
writeParenthesisedString(key.expression, out);
writeString(" AS ", out);
}
writeQuoted(key.name, out);
}
}
for (const auto & attr : dict_struct.attributes)
{
writeString(", ", out);
if (!attr.expression.empty())
{
writeParenthesisedString(attr.expression, out);
writeString(" AS ", out);
}
writeQuoted(attr.name, out);
}
writeString(" FROM ", out);
if (!db.empty())
{
writeQuoted(db, out);
writeChar('.', out);
}
if (!schema.empty())
{
writeQuoted(schema, out);
writeChar('.', out);
}
writeQuoted(table, out);
if (!where.empty())
{
writeString(" WHERE ", out);
writeString(where, out);
}
writeChar(';', out);
return out.str();
}
std::string
ExternalQueryBuilder::composeLoadKeysQuery(const Columns & key_columns, const std::vector<size_t> & requested_rows, LoadKeysMethod method)
{
if (!dict_struct.key)
throw Exception{"Composite key required for method", ErrorCodes::UNSUPPORTED_METHOD};
WriteBufferFromOwnString out;
writeString("SELECT ", out);
auto first = true;
for (const auto & key_or_attribute : boost::join(*dict_struct.key, dict_struct.attributes))
{
if (!first)
writeString(", ", out);
first = false;
if (!key_or_attribute.expression.empty())
{
writeParenthesisedString(key_or_attribute.expression, out);
writeString(" AS ", out);
}
writeQuoted(key_or_attribute.name, out);
}
writeString(" FROM ", out);
if (!db.empty())
{
writeQuoted(db, out);
writeChar('.', out);
}
if (!schema.empty())
{
writeQuoted(schema, out);
writeChar('.', out);
}
writeQuoted(table, out);
writeString(" WHERE ", out);
if (!where.empty())
{
writeString("(", out);
writeString(where, out);
writeString(") AND (", out);
}
if (method == AND_OR_CHAIN)
{
first = true;
for (const auto row : requested_rows)
{
if (!first)
writeString(" OR ", out);
first = false;
composeKeyCondition(key_columns, row, out);
}
}
else /* if (method == IN_WITH_TUPLES) */
{
writeString(composeKeyTupleDefinition(), out);
writeString(" IN (", out);
first = true;
for (const auto row : requested_rows)
{
if (!first)
writeString(", ", out);
first = false;
composeKeyTuple(key_columns, row, out);
}
writeString(")", out);
}
if (!where.empty())
{
writeString(")", out);
}
writeString(";", out);
return out.str();
}
std::string ExternalQueryBuilder::composeLoadIdsQuery(const std::vector<UInt64> & ids)
{
if (!dict_struct.id)
throw Exception{"Simple key required for method", ErrorCodes::UNSUPPORTED_METHOD};
WriteBufferFromOwnString out;
writeString("SELECT ", out);
if (!dict_struct.id->expression.empty())
{
writeParenthesisedString(dict_struct.id->expression, out);
writeString(" AS ", out);
}
writeQuoted(dict_struct.id->name, out);
for (const auto & attr : dict_struct.attributes)
{
writeString(", ", out);
if (!attr.expression.empty())
{
writeParenthesisedString(attr.expression, out);
writeString(" AS ", out);
}
writeQuoted(attr.name, out);
}
writeString(" FROM ", out);
if (!db.empty())
{
writeQuoted(db, out);
writeChar('.', out);
}
if (!schema.empty())
{
writeQuoted(schema, out);
writeChar('.', out);
}
writeQuoted(table, out);
writeString(" WHERE ", out);
if (!where.empty())
{
writeString(where, out);
writeString(" AND ", out);
}
writeQuoted(dict_struct.id->name, out);
writeString(" IN (", out);
auto first = true;
for (const auto id : ids)
{
if (!first)
writeString(", ", out);
first = false;
writeString(DB::toString(id), out);
}
writeString(");", out);
return out.str();
}
StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & ast_function, const Context & context) const
{
const ASTFunction & args_func = typeid_cast<const ASTFunction &>(*ast_function);
if (!args_func.arguments)
throw Exception("Table function 'mysql' must have arguments.", ErrorCodes::LOGICAL_ERROR);
ASTs & args = typeid_cast<ASTExpressionList &>(*args_func.arguments).children;
if (args.size() < 5 || args.size() > 7)
throw Exception("Table function 'mysql' requires 5-7 parameters: MySQL('host:port', database, table, 'user', 'password'[, replace_query, 'on_duplicate_clause']).",
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
for (size_t i = 0; i < args.size(); ++i)
args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(args[i], context);
std::string host_port = static_cast<const ASTLiteral &>(*args[0]).value.safeGet<String>();
std::string database_name = static_cast<const ASTLiteral &>(*args[1]).value.safeGet<String>();
std::string table_name = static_cast<const ASTLiteral &>(*args[2]).value.safeGet<String>();
std::string user_name = static_cast<const ASTLiteral &>(*args[3]).value.safeGet<String>();
std::string password = static_cast<const ASTLiteral &>(*args[4]).value.safeGet<String>();
bool replace_query = false;
std::string on_duplicate_clause;
if (args.size() >= 6)
replace_query = static_cast<const ASTLiteral &>(*args[5]).value.safeGet<UInt64>() > 0;
if (args.size() == 7)
on_duplicate_clause = static_cast<const ASTLiteral &>(*args[6]).value.safeGet<String>();
if (replace_query && !on_duplicate_clause.empty())
throw Exception(
"Only one of 'replace_query' and 'on_duplicate_clause' can be specified, or none of them",
ErrorCodes::BAD_ARGUMENTS);
/// 3306 is the default MySQL port number
auto parsed_host_port = parseAddress(host_port, 3306);
mysqlxx::Pool pool(database_name, parsed_host_port.first, user_name, password, parsed_host_port.second);
/// Determine table definition by running a query to INFORMATION_SCHEMA.
Block sample_block
{
{ std::make_shared<DataTypeString>(), "name" },
{ std::make_shared<DataTypeString>(), "type" },
{ std::make_shared<DataTypeUInt8>(), "is_nullable" },
{ std::make_shared<DataTypeUInt8>(), "is_unsigned" },
{ std::make_shared<DataTypeUInt64>(), "length" },
};
WriteBufferFromOwnString query;
query << "SELECT"
" COLUMN_NAME AS name,"
" DATA_TYPE AS type,"
" IS_NULLABLE = 'YES' AS is_nullable,"
" COLUMN_TYPE LIKE '%unsigned' AS is_unsigned,"
" CHARACTER_MAXIMUM_LENGTH AS length"
" FROM INFORMATION_SCHEMA.COLUMNS"
" WHERE TABLE_SCHEMA = " << quote << database_name
<< " AND TABLE_NAME = " << quote << table_name
<< " ORDER BY ORDINAL_POSITION";
MySQLBlockInputStream result(pool.Get(), query.str(), sample_block, DEFAULT_BLOCK_SIZE);
NamesAndTypesList columns;
while (Block block = result.read())
{
size_t rows = block.rows();
for (size_t i = 0; i < rows; ++i)
columns.emplace_back(
(*block.getByPosition(0).column)[i].safeGet<String>(),
getDataType(
(*block.getByPosition(1).column)[i].safeGet<String>(),
(*block.getByPosition(2).column)[i].safeGet<UInt64>() && context.getSettings().external_table_functions_use_nulls,
(*block.getByPosition(3).column)[i].safeGet<UInt64>(),
(*block.getByPosition(4).column)[i].safeGet<UInt64>()));
}
auto res = StorageMySQL::create(
table_name,
std::move(pool),
database_name,
table_name,
replace_query,
on_duplicate_clause,
ColumnsDescription{columns},
context);
res->startup();
return res;
}
String NamesAndTypesList::toString() const
{
WriteBufferFromOwnString out;
writeText(out);
return out.str();
}
std::string ExecutionStatus::serializeText() const
{
WriteBufferFromOwnString wb;
wb << code << "\n" << escape << message;
return wb.str();
}
