MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream(
const MergeTreeData & storage_,
const MergeTreeData::DataPartPtr & data_part_,
Names columns_to_read_,
bool read_with_direct_io_,
bool take_column_types_from_storage,
bool quiet)
: storage(storage_)
, data_part(data_part_)
, part_columns_lock(data_part->columns_lock)
, columns_to_read(columns_to_read_)
, read_with_direct_io(read_with_direct_io_)
, mark_cache(storage.global_context.getMarkCache())
{
if (!quiet)
{
std::stringstream message;
message << "Reading " << data_part->marks_count << " marks from part " << data_part->name
<< ", total " << data_part->rows_count
<< " rows starting from the beginning of the part, columns: ";
for (size_t i = 0, size = columns_to_read.size(); i < size; ++i)
message << (i == 0 ? "" : ", ") << columns_to_read[i];
LOG_TRACE(log, message.rdbuf());
}
addTotalRowsApprox(data_part->rows_count);
header = storage.getSampleBlockForColumns(columns_to_read);
fixHeader(header);
/// Add columns because we don't want to read empty blocks
injectRequiredColumns(storage, data_part, columns_to_read);
NamesAndTypesList columns_for_reader;
if (take_column_types_from_storage)
{
const NamesAndTypesList & physical_columns = storage.getColumns().getAllPhysical();
columns_for_reader = physical_columns.addTypes(columns_to_read);
}
else
{
/// take columns from data_part
columns_for_reader = data_part->columns.addTypes(columns_to_read);
}
reader = std::make_unique<MergeTreeReader>(
data_part->getFullPath(), data_part, columns_for_reader, /* uncompressed_cache = */ nullptr,
mark_cache.get(), /* save_marks_in_cache = */ false, storage,
MarkRanges{MarkRange(0, data_part->marks_count)},
/* bytes to use AIO (this is hack) */
read_with_direct_io ? 1UL : std::numeric_limits<size_t>::max(),
DBMS_DEFAULT_BUFFER_SIZE);
}
static void RangeSection (void)
/* Parse a range section */
{
static const IdentTok RangeDefs[] = {
{ "COMMENT", INFOTOK_COMMENT },
{ "END", INFOTOK_END },
{ "NAME", INFOTOK_NAME },
{ "START", INFOTOK_START },
{ "TYPE", INFOTOK_TYPE },
};
static const IdentTok TypeDefs[] = {
{ "ADDRTABLE", INFOTOK_ADDRTAB },
{ "BYTETABLE", INFOTOK_BYTETAB },
{ "CODE", INFOTOK_CODE },
{ "DBYTETABLE", INFOTOK_DBYTETAB },
{ "DWORDTABLE", INFOTOK_DWORDTAB },
{ "RTSTABLE", INFOTOK_RTSTAB },
{ "SKIP", INFOTOK_SKIP },
{ "TEXTTABLE", INFOTOK_TEXTTAB },
{ "WORDTABLE", INFOTOK_WORDTAB },
};
/* Which values did we get? */
enum {
tNone = 0x00,
tStart = 0x01,
tEnd = 0x02,
tType = 0x04,
tName = 0x08,
tComment= 0x10,
tNeeded = (tStart | tEnd | tType)
};
unsigned Attributes = tNone;
/* Locals - initialize to avoid gcc warnings */
unsigned Start = 0;
unsigned End = 0;
unsigned char Type = 0;
char* Name = 0;
char* Comment = 0;
unsigned MemberSize = 0;
/* Skip the token */
InfoNextTok ();
/* Expect the opening curly brace */
InfoConsumeLCurly ();
/* Look for section tokens */
while (InfoTok != INFOTOK_RCURLY) {
/* Convert to special token */
InfoSpecialToken (RangeDefs, ENTRY_COUNT (RangeDefs), "Range attribute");
/* Look at the token */
switch (InfoTok) {
case INFOTOK_COMMENT:
AddAttr ("COMMENT", &Attributes, tComment);
InfoNextTok ();
InfoAssureStr ();
if (InfoSVal[0] == '\0') {
InfoError ("Comment may not be empty");
}
Comment = xstrdup (InfoSVal);
Attributes |= tComment;
InfoNextTok ();
break;
case INFOTOK_END:
AddAttr ("END", &Attributes, tEnd);
InfoNextTok ();
InfoAssureInt ();
InfoRangeCheck (0x0000, 0xFFFF);
End = InfoIVal;
InfoNextTok ();
break;
case INFOTOK_NAME:
AddAttr ("NAME", &Attributes, tName);
InfoNextTok ();
InfoAssureStr ();
if (InfoSVal[0] == '\0') {
InfoError ("Name may not be empty");
}
Name = xstrdup (InfoSVal);
Attributes |= tName;
InfoNextTok ();
break;
case INFOTOK_START:
AddAttr ("START", &Attributes, tStart);
InfoNextTok ();
InfoAssureInt ();
InfoRangeCheck (0x0000, 0xFFFF);
Start = InfoIVal;
InfoNextTok ();
break;
case INFOTOK_TYPE:
AddAttr ("TYPE", &Attributes, tType);
InfoNextTok ();
InfoSpecialToken (TypeDefs, ENTRY_COUNT (TypeDefs), "TYPE");
switch (InfoTok) {
case INFOTOK_ADDRTAB: Type = atAddrTab; MemberSize = 2; break;
case INFOTOK_BYTETAB: Type = atByteTab; MemberSize = 1; break;
case INFOTOK_CODE: Type = atCode; MemberSize = 1; break;
case INFOTOK_DBYTETAB: Type = atDByteTab; MemberSize = 2; break;
case INFOTOK_DWORDTAB: Type = atDWordTab; MemberSize = 4; break;
case INFOTOK_RTSTAB: Type = atRtsTab; MemberSize = 2; break;
case INFOTOK_SKIP: Type = atSkip; MemberSize = 1; break;
case INFOTOK_TEXTTAB: Type = atTextTab; MemberSize = 1; break;
case INFOTOK_WORDTAB: Type = atWordTab; MemberSize = 2; break;
}
InfoNextTok ();
break;
default:
Internal ("Unexpected token: %u", InfoTok);
}
/* Directive is followed by a semicolon */
InfoConsumeSemi ();
}
/* Did we get all required values? */
if ((Attributes & tNeeded) != tNeeded) {
InfoError ("Required values missing from this section");
}
/* Start must be less than end */
if (Start > End) {
InfoError ("Start value must not be greater than end value");
}
/* Check the granularity */
if (((End - Start + 1) % MemberSize) != 0) {
InfoError ("Type of range needs a granularity of %u", MemberSize);
}
/* Set the range */
MarkRange (Start, End, Type);
/* Do we have a label? */
if (Attributes & tName) {
/* Define a label for the table */
AddExtLabel (Start, Name);
/* Set the comment if we have one */
if (Comment) {
SetComment (Start, Comment);
}
/* Delete name and comment */
xfree (Name);
xfree (Comment);
}
/* Consume the closing brace */
InfoConsumeRCurly ();
}
