void Sorter::Init(CodeGen &codegen, llvm::Value *sorter_ptr,
llvm::Value *executor_ctx,
llvm::Value *comparison_func) const {
auto *tuple_size = codegen.Const32(storage_format_.GetStorageSize());
codegen.Call(SorterProxy::Init,
{sorter_ptr, executor_ctx, comparison_func, tuple_size});
}
void Sorter::SortTopKParallel(CodeGen &codegen, llvm::Value *sorter_ptr,
llvm::Value *thread_states,
uint32_t sorter_offset, uint64_t top_k) const {
auto *offset = codegen.Const32(sorter_offset);
codegen.Call(SorterProxy::SortTopKParallel,
{sorter_ptr, thread_states, offset, codegen.Const64(top_k)});
}
// Iterate over the tuples in the sorter in batches/vectors of the given size
void Sorter::VectorizedIterate(
CodeGen &codegen, llvm::Value *sorter_ptr, uint32_t vector_size,
uint64_t offset, Sorter::VectorizedIterateCallback &callback) const {
llvm::Value *start_pos = codegen.Load(SorterProxy::tuples_start, sorter_ptr);
llvm::Value *num_tuples = NumTuples(codegen, sorter_ptr);
num_tuples = codegen->CreateTrunc(num_tuples, codegen.Int32Type());
if (offset != 0) {
start_pos = codegen->CreateConstInBoundsGEP1_32(codegen.CharPtrType(),
start_pos, offset);
num_tuples = codegen->CreateSub(num_tuples, codegen.Const32(offset));
}
lang::VectorizedLoop loop(codegen, num_tuples, vector_size, {});
{
// Current loop range
auto curr_range = loop.GetCurrentRange();
// Provide an accessor into the sorted space
SorterAccess sorter_access(*this, start_pos);
// Issue the callback
callback.ProcessEntries(codegen, curr_range.start, curr_range.end,
sorter_access);
// That's it
loop.LoopEnd(codegen, {});
}
}
// Iterate over the tuples in the sorter in batches/vectors of the given size
void Sorter::VectorizedIterate(
CodeGen &codegen, llvm::Value *sorter_ptr, uint32_t vector_size,
Sorter::VectorizedIterateCallback &callback) const {
llvm::Value *start_pos = GetStartPosition(codegen, sorter_ptr);
llvm::Value *num_tuples = GetNumberOfStoredTuples(codegen, sorter_ptr);
// Determine the number of bytes to skip per vector
llvm::Value *vec_sz = codegen.Const32(vector_size);
llvm::Value *tuple_size = GetTupleSize(codegen);
llvm::Value *skip = codegen->CreateMul(vec_sz, tuple_size);
lang::VectorizedLoop loop{
codegen, num_tuples, vector_size, {{"pos", start_pos}}};
{
llvm::Value *curr_pos = loop.GetLoopVar(0);
auto curr_range = loop.GetCurrentRange();
// Provide an accessor into the sorted space
SorterAccess sorter_access{*this, start_pos};
// Issue the callback
callback.ProcessEntries(codegen, curr_range.start, curr_range.end,
sorter_access);
// Bump the pointer by the size of a tuple
llvm::Value *next_pos = codegen->CreateInBoundsGEP(curr_pos, skip);
loop.LoopEnd(codegen, {next_pos});
}
}
// Iterate over all valid rows in this batch
void RowBatch::Iterate(CodeGen &codegen, RowBatch::IterateCallback &cb) {
// The starting position in the batch
llvm::Value *start = codegen.Const32(0);
// The ending position in the batch
llvm::Value *end = GetNumValidRows(codegen);
// Generating the loop
std::vector<lang::Loop::LoopVariable> loop_vars = {
{"readIdx", start}, {"writeIdx", codegen.Const32(0)}};
llvm::Value *loop_cond = codegen->CreateICmpULT(start, end);
lang::Loop batch_loop{codegen, loop_cond, loop_vars};
{
// Pull out loop vars for convenience
auto *batch_pos = batch_loop.GetLoopVar(0);
auto *write_pos = batch_loop.GetLoopVar(1);
// Create an output tracker to track the final position of the row
OutputTracker tracker{GetSelectionVector(), write_pos};
// Get the current row
RowBatch::Row row = GetRowAt(batch_pos, &tracker);
// Invoke callback
cb.ProcessRow(row);
// The next read position is one next
auto *next_read_pos = codegen->CreateAdd(batch_pos, codegen.Const32(1));
// The write position from the output track
auto *next_write_pos = tracker.GetFinalOutputPos();
// Close up loop
llvm::Value *loop_cond = codegen->CreateICmpULT(next_read_pos, end);
batch_loop.LoopEnd(loop_cond, {next_read_pos, next_write_pos});
}
// After the batch loop, we need to reset the size of the selection vector
std::vector<llvm::Value *> final_vals;
batch_loop.CollectFinalLoopVariables(final_vals);
// Mark the last position in the batch
UpdateWritePosition(final_vals[1]);
}
void BufferAccessor::Append(CodeGen &codegen, llvm::Value *buffer_ptr,
const std::vector<codegen::Value> &tuple) const {
auto *size = codegen.Const32(storage_format_.GetStorageSize());
auto *space = codegen.Call(BufferProxy::Append, {buffer_ptr, size});
// Now, individually store the attributes of the tuple into the free space
UpdateableStorage::NullBitmap null_bitmap(codegen, storage_format_, space);
for (uint32_t col_id = 0; col_id < tuple.size(); col_id++) {
storage_format_.SetValue(codegen, space, col_id, tuple[col_id],
null_bitmap);
}
null_bitmap.WriteBack(codegen);
}
// Generate a scan over all tile groups.
//
// @code
// column_layouts := alloca<peloton::ColumnLayoutInfo>(
// table.GetSchema().GetColumnCount())
//
// oid_t tile_group_idx := 0
// num_tile_groups = GetTileGroupCount(table_ptr)
//
// for (; tile_group_idx < num_tile_groups; ++tile_group_idx) {
// tile_group_ptr := GetTileGroup(table_ptr, tile_group_idx)
// consumer.TileGroupStart(tile_group_ptr);
// tile_group.TidScan(tile_group_ptr, column_layouts, vector_size, consumer);
// consumer.TileGroupEnd(tile_group_ptr);
// }
//
// @endcode
void Table::GenerateScan(CodeGen &codegen, llvm::Value *table_ptr,
uint32_t batch_size, ScanCallback &consumer) const {
// First get the columns from the table the consumer needs. For every column,
// we'll need to have a ColumnInfoLayout struct
const uint32_t num_columns =
static_cast<uint32_t>(table_.GetSchema()->GetColumnCount());
llvm::Value *column_layouts = codegen->CreateAlloca(
RuntimeFunctionsProxy::_ColumnLayoutInfo::GetType(codegen),
codegen.Const32(num_columns));
// Get the number of tile groups in the given table
llvm::Value *tile_group_idx = codegen.Const64(0);
llvm::Value *num_tile_groups = GetTileGroupCount(codegen, table_ptr);
// Iterate over all tile groups in the table
lang::Loop loop{codegen,
codegen->CreateICmpULT(tile_group_idx, num_tile_groups),
{{"tileGroupIdx", tile_group_idx}}};
{
// Get the tile group with the given tile group ID
tile_group_idx = loop.GetLoopVar(0);
llvm::Value *tile_group_ptr =
GetTileGroup(codegen, table_ptr, tile_group_idx);
llvm::Value *tile_group_id =
tile_group_.GetTileGroupId(codegen, tile_group_ptr);
// Invoke the consumer to let her know that we're starting to iterate over
// the tile group now
consumer.TileGroupStart(codegen, tile_group_id, tile_group_ptr);
// Generate the scan cover over the given tile group
tile_group_.GenerateTidScan(codegen, tile_group_ptr, column_layouts,
batch_size, consumer);
// Invoke the consumer to let her know that we're done with this tile group
consumer.TileGroupFinish(codegen, tile_group_ptr);
// Move to next tile group in the table
tile_group_idx = codegen->CreateAdd(tile_group_idx, codegen.Const64(1));
loop.LoopEnd(codegen->CreateICmpULT(tile_group_idx, num_tile_groups),
{tile_group_idx});
}
}
// Iterate over all valid rows in this batch in vectors of a given size
void RowBatch::VectorizedIterate(CodeGen &codegen,
RowBatch::VectorizedIterateCallback &cb) {
// The size of the vectors we use for iteration
auto vector_size = cb.GetVectorSize();
// The number of valid rows in the batch
auto *num_rows = GetNumValidRows(codegen);
// The current write/output position
llvm::Value *write_pos = codegen.Const32(0);
// The vectorized loop
lang::VectorizedLoop vector_loop{
codegen, num_rows, vector_size, {{"writePos", write_pos}}};
{
auto curr_range = vector_loop.GetCurrentRange();
write_pos = vector_loop.GetLoopVar(0);
// The current instance of the vectorized loop
VectorizedIterateCallback::IterationInstance iter_instance;
iter_instance.start = curr_range.start;
iter_instance.end = curr_range.end;
iter_instance.write_pos = write_pos;
// Invoke the callback
write_pos = cb.ProcessRows(iter_instance);
// End it
vector_loop.LoopEnd(codegen, {write_pos});
}
// After the loop, we need to reset the size of the selection vector
std::vector<llvm::Value *> final_vals;
vector_loop.CollectFinalLoopVariables(final_vals);
// Mark the last position in the batch
UpdateWritePosition(final_vals[1]);
}
//===----------------------------------------------------------------------===//
// Here, we discover the layout of every column that will be accessed. A
// column's layout includes three pieces of information:
//
// 1. The starting memory address (where the first value of the column is)
// 2. The stride length
// 3. Whether the column is in columnar layout
//===----------------------------------------------------------------------===//
std::vector<TileGroup::ColumnLayout> TileGroup::GetColumnLayouts(
CodeGen &codegen, llvm::Value *tile_group_ptr,
llvm::Value *column_layout_infos) const {
// Call RuntimeFunctions::GetTileGroupLayout()
uint32_t num_cols = schema_.GetColumnCount();
codegen.Call(
RuntimeFunctionsProxy::GetTileGroupLayout,
{tile_group_ptr, column_layout_infos, codegen.Const32(num_cols)});
// Collect <start, stride, is_columnar> triplets of all columns
std::vector<TileGroup::ColumnLayout> layouts;
auto *layout_type = ColumnLayoutInfoProxy::GetType(codegen);
for (uint32_t col_id = 0; col_id < num_cols; col_id++) {
auto *start = codegen->CreateLoad(codegen->CreateConstInBoundsGEP2_32(
layout_type, column_layout_infos, col_id, 0));
auto *stride = codegen->CreateLoad(codegen->CreateConstInBoundsGEP2_32(
layout_type, column_layout_infos, col_id, 1));
auto *columnar = codegen->CreateLoad(codegen->CreateConstInBoundsGEP2_32(
layout_type, column_layout_infos, col_id, 2));
layouts.push_back(ColumnLayout{col_id, start, stride, columnar});
}
return layouts;
}
void ProcessEntries(CodeGen &codegen, llvm::Value *start_index,
llvm::Value *end_index, SorterAccess &access) const {
lang::Loop loop(codegen,
codegen->CreateICmpULT(start_index, end_index),
{{"start", start_index}});
{
llvm::Value *curr_index = loop.GetLoopVar(0);
// Parse the row
std::vector<codegen::Value> vals;
auto &row = access.GetRow(curr_index);
for (uint32_t i = 0; i < storage.GetNumElements(); i++) {
vals.emplace_back(row.LoadColumn(codegen, i));
}
// Call the actual callback
callback.ProcessEntry(codegen, vals);
curr_index = codegen->CreateAdd(curr_index, codegen.Const32(1));
loop.LoopEnd(codegen->CreateICmpULT(curr_index, end_index),
{curr_index});
}
}
Value Varchar::GetNullValue(CodeGen &codegen) const {
return Value{Type{TypeId(), true}, codegen.NullPtr(codegen.CharPtrType()),
codegen.Const32(0), codegen.ConstBool(true)};
}
Value Date::GetNullValue(CodeGen &codegen) const {
auto *raw_val = codegen.Const32(peloton::type::PELOTON_DATE_NULL);
return Value{Type{TypeId(), true}, raw_val, nullptr, codegen.ConstBool(true)};
}
Value Date::GetMaxValue(CodeGen &codegen) const {
auto *raw_val = codegen.Const32(peloton::type::PELOTON_DATE_MAX);
return Value{*this, raw_val, nullptr, nullptr};
}
llvm::Value *Sorter::GetTupleSize(CodeGen &codegen) const {
return codegen.Const32(storage_format_.GetStorageSize());
}
// Just make a call to util::Sorter::Init(...)
void Sorter::Init(CodeGen &codegen, llvm::Value *sorter_ptr,
llvm::Value *comparison_func) const {
auto *tuple_size = codegen.Const32(storage_format_.GetStorageSize());
codegen.CallFunc(SorterProxy::_Init::GetFunction(codegen),
{sorter_ptr, comparison_func, tuple_size});
}