void Sorter::Init(CodeGen &codegen, llvm::Value *sorter_ptr, llvm::Value *executor_ctx, llvm::Value *comparison_func) const { auto *tuple_size = codegen.Const32(storage_format_.GetStorageSize()); codegen.Call(SorterProxy::Init, {sorter_ptr, executor_ctx, comparison_func, tuple_size}); }
void Sorter::SortTopKParallel(CodeGen &codegen, llvm::Value *sorter_ptr, llvm::Value *thread_states, uint32_t sorter_offset, uint64_t top_k) const { auto *offset = codegen.Const32(sorter_offset); codegen.Call(SorterProxy::SortTopKParallel, {sorter_ptr, thread_states, offset, codegen.Const64(top_k)}); }
// Iterate over the tuples in the sorter in batches/vectors of the given size void Sorter::VectorizedIterate( CodeGen &codegen, llvm::Value *sorter_ptr, uint32_t vector_size, uint64_t offset, Sorter::VectorizedIterateCallback &callback) const { llvm::Value *start_pos = codegen.Load(SorterProxy::tuples_start, sorter_ptr); llvm::Value *num_tuples = NumTuples(codegen, sorter_ptr); num_tuples = codegen->CreateTrunc(num_tuples, codegen.Int32Type()); if (offset != 0) { start_pos = codegen->CreateConstInBoundsGEP1_32(codegen.CharPtrType(), start_pos, offset); num_tuples = codegen->CreateSub(num_tuples, codegen.Const32(offset)); } lang::VectorizedLoop loop(codegen, num_tuples, vector_size, {}); { // Current loop range auto curr_range = loop.GetCurrentRange(); // Provide an accessor into the sorted space SorterAccess sorter_access(*this, start_pos); // Issue the callback callback.ProcessEntries(codegen, curr_range.start, curr_range.end, sorter_access); // That's it loop.LoopEnd(codegen, {}); } }
// Iterate over the tuples in the sorter in batches/vectors of the given size void Sorter::VectorizedIterate( CodeGen &codegen, llvm::Value *sorter_ptr, uint32_t vector_size, Sorter::VectorizedIterateCallback &callback) const { llvm::Value *start_pos = GetStartPosition(codegen, sorter_ptr); llvm::Value *num_tuples = GetNumberOfStoredTuples(codegen, sorter_ptr); // Determine the number of bytes to skip per vector llvm::Value *vec_sz = codegen.Const32(vector_size); llvm::Value *tuple_size = GetTupleSize(codegen); llvm::Value *skip = codegen->CreateMul(vec_sz, tuple_size); lang::VectorizedLoop loop{ codegen, num_tuples, vector_size, {{"pos", start_pos}}}; { llvm::Value *curr_pos = loop.GetLoopVar(0); auto curr_range = loop.GetCurrentRange(); // Provide an accessor into the sorted space SorterAccess sorter_access{*this, start_pos}; // Issue the callback callback.ProcessEntries(codegen, curr_range.start, curr_range.end, sorter_access); // Bump the pointer by the size of a tuple llvm::Value *next_pos = codegen->CreateInBoundsGEP(curr_pos, skip); loop.LoopEnd(codegen, {next_pos}); } }
// Iterate over all valid rows in this batch void RowBatch::Iterate(CodeGen &codegen, RowBatch::IterateCallback &cb) { // The starting position in the batch llvm::Value *start = codegen.Const32(0); // The ending position in the batch llvm::Value *end = GetNumValidRows(codegen); // Generating the loop std::vector<lang::Loop::LoopVariable> loop_vars = { {"readIdx", start}, {"writeIdx", codegen.Const32(0)}}; llvm::Value *loop_cond = codegen->CreateICmpULT(start, end); lang::Loop batch_loop{codegen, loop_cond, loop_vars}; { // Pull out loop vars for convenience auto *batch_pos = batch_loop.GetLoopVar(0); auto *write_pos = batch_loop.GetLoopVar(1); // Create an output tracker to track the final position of the row OutputTracker tracker{GetSelectionVector(), write_pos}; // Get the current row RowBatch::Row row = GetRowAt(batch_pos, &tracker); // Invoke callback cb.ProcessRow(row); // The next read position is one next auto *next_read_pos = codegen->CreateAdd(batch_pos, codegen.Const32(1)); // The write position from the output track auto *next_write_pos = tracker.GetFinalOutputPos(); // Close up loop llvm::Value *loop_cond = codegen->CreateICmpULT(next_read_pos, end); batch_loop.LoopEnd(loop_cond, {next_read_pos, next_write_pos}); } // After the batch loop, we need to reset the size of the selection vector std::vector<llvm::Value *> final_vals; batch_loop.CollectFinalLoopVariables(final_vals); // Mark the last position in the batch UpdateWritePosition(final_vals[1]); }
void BufferAccessor::Append(CodeGen &codegen, llvm::Value *buffer_ptr, const std::vector<codegen::Value> &tuple) const { auto *size = codegen.Const32(storage_format_.GetStorageSize()); auto *space = codegen.Call(BufferProxy::Append, {buffer_ptr, size}); // Now, individually store the attributes of the tuple into the free space UpdateableStorage::NullBitmap null_bitmap(codegen, storage_format_, space); for (uint32_t col_id = 0; col_id < tuple.size(); col_id++) { storage_format_.SetValue(codegen, space, col_id, tuple[col_id], null_bitmap); } null_bitmap.WriteBack(codegen); }
// Generate a scan over all tile groups. // // @code // column_layouts := alloca<peloton::ColumnLayoutInfo>( // table.GetSchema().GetColumnCount()) // // oid_t tile_group_idx := 0 // num_tile_groups = GetTileGroupCount(table_ptr) // // for (; tile_group_idx < num_tile_groups; ++tile_group_idx) { // tile_group_ptr := GetTileGroup(table_ptr, tile_group_idx) // consumer.TileGroupStart(tile_group_ptr); // tile_group.TidScan(tile_group_ptr, column_layouts, vector_size, consumer); // consumer.TileGroupEnd(tile_group_ptr); // } // // @endcode void Table::GenerateScan(CodeGen &codegen, llvm::Value *table_ptr, uint32_t batch_size, ScanCallback &consumer) const { // First get the columns from the table the consumer needs. For every column, // we'll need to have a ColumnInfoLayout struct const uint32_t num_columns = static_cast<uint32_t>(table_.GetSchema()->GetColumnCount()); llvm::Value *column_layouts = codegen->CreateAlloca( RuntimeFunctionsProxy::_ColumnLayoutInfo::GetType(codegen), codegen.Const32(num_columns)); // Get the number of tile groups in the given table llvm::Value *tile_group_idx = codegen.Const64(0); llvm::Value *num_tile_groups = GetTileGroupCount(codegen, table_ptr); // Iterate over all tile groups in the table lang::Loop loop{codegen, codegen->CreateICmpULT(tile_group_idx, num_tile_groups), {{"tileGroupIdx", tile_group_idx}}}; { // Get the tile group with the given tile group ID tile_group_idx = loop.GetLoopVar(0); llvm::Value *tile_group_ptr = GetTileGroup(codegen, table_ptr, tile_group_idx); llvm::Value *tile_group_id = tile_group_.GetTileGroupId(codegen, tile_group_ptr); // Invoke the consumer to let her know that we're starting to iterate over // the tile group now consumer.TileGroupStart(codegen, tile_group_id, tile_group_ptr); // Generate the scan cover over the given tile group tile_group_.GenerateTidScan(codegen, tile_group_ptr, column_layouts, batch_size, consumer); // Invoke the consumer to let her know that we're done with this tile group consumer.TileGroupFinish(codegen, tile_group_ptr); // Move to next tile group in the table tile_group_idx = codegen->CreateAdd(tile_group_idx, codegen.Const64(1)); loop.LoopEnd(codegen->CreateICmpULT(tile_group_idx, num_tile_groups), {tile_group_idx}); } }
// Iterate over all valid rows in this batch in vectors of a given size void RowBatch::VectorizedIterate(CodeGen &codegen, RowBatch::VectorizedIterateCallback &cb) { // The size of the vectors we use for iteration auto vector_size = cb.GetVectorSize(); // The number of valid rows in the batch auto *num_rows = GetNumValidRows(codegen); // The current write/output position llvm::Value *write_pos = codegen.Const32(0); // The vectorized loop lang::VectorizedLoop vector_loop{ codegen, num_rows, vector_size, {{"writePos", write_pos}}}; { auto curr_range = vector_loop.GetCurrentRange(); write_pos = vector_loop.GetLoopVar(0); // The current instance of the vectorized loop VectorizedIterateCallback::IterationInstance iter_instance; iter_instance.start = curr_range.start; iter_instance.end = curr_range.end; iter_instance.write_pos = write_pos; // Invoke the callback write_pos = cb.ProcessRows(iter_instance); // End it vector_loop.LoopEnd(codegen, {write_pos}); } // After the loop, we need to reset the size of the selection vector std::vector<llvm::Value *> final_vals; vector_loop.CollectFinalLoopVariables(final_vals); // Mark the last position in the batch UpdateWritePosition(final_vals[1]); }
//===----------------------------------------------------------------------===// // Here, we discover the layout of every column that will be accessed. A // column's layout includes three pieces of information: // // 1. The starting memory address (where the first value of the column is) // 2. The stride length // 3. Whether the column is in columnar layout //===----------------------------------------------------------------------===// std::vector<TileGroup::ColumnLayout> TileGroup::GetColumnLayouts( CodeGen &codegen, llvm::Value *tile_group_ptr, llvm::Value *column_layout_infos) const { // Call RuntimeFunctions::GetTileGroupLayout() uint32_t num_cols = schema_.GetColumnCount(); codegen.Call( RuntimeFunctionsProxy::GetTileGroupLayout, {tile_group_ptr, column_layout_infos, codegen.Const32(num_cols)}); // Collect <start, stride, is_columnar> triplets of all columns std::vector<TileGroup::ColumnLayout> layouts; auto *layout_type = ColumnLayoutInfoProxy::GetType(codegen); for (uint32_t col_id = 0; col_id < num_cols; col_id++) { auto *start = codegen->CreateLoad(codegen->CreateConstInBoundsGEP2_32( layout_type, column_layout_infos, col_id, 0)); auto *stride = codegen->CreateLoad(codegen->CreateConstInBoundsGEP2_32( layout_type, column_layout_infos, col_id, 1)); auto *columnar = codegen->CreateLoad(codegen->CreateConstInBoundsGEP2_32( layout_type, column_layout_infos, col_id, 2)); layouts.push_back(ColumnLayout{col_id, start, stride, columnar}); } return layouts; }
void ProcessEntries(CodeGen &codegen, llvm::Value *start_index, llvm::Value *end_index, SorterAccess &access) const { lang::Loop loop(codegen, codegen->CreateICmpULT(start_index, end_index), {{"start", start_index}}); { llvm::Value *curr_index = loop.GetLoopVar(0); // Parse the row std::vector<codegen::Value> vals; auto &row = access.GetRow(curr_index); for (uint32_t i = 0; i < storage.GetNumElements(); i++) { vals.emplace_back(row.LoadColumn(codegen, i)); } // Call the actual callback callback.ProcessEntry(codegen, vals); curr_index = codegen->CreateAdd(curr_index, codegen.Const32(1)); loop.LoopEnd(codegen->CreateICmpULT(curr_index, end_index), {curr_index}); } }
Value Varchar::GetNullValue(CodeGen &codegen) const { return Value{Type{TypeId(), true}, codegen.NullPtr(codegen.CharPtrType()), codegen.Const32(0), codegen.ConstBool(true)}; }
Value Date::GetNullValue(CodeGen &codegen) const { auto *raw_val = codegen.Const32(peloton::type::PELOTON_DATE_NULL); return Value{Type{TypeId(), true}, raw_val, nullptr, codegen.ConstBool(true)}; }
Value Date::GetMaxValue(CodeGen &codegen) const { auto *raw_val = codegen.Const32(peloton::type::PELOTON_DATE_MAX); return Value{*this, raw_val, nullptr, nullptr}; }
llvm::Value *Sorter::GetTupleSize(CodeGen &codegen) const { return codegen.Const32(storage_format_.GetStorageSize()); }
// Just make a call to util::Sorter::Init(...) void Sorter::Init(CodeGen &codegen, llvm::Value *sorter_ptr, llvm::Value *comparison_func) const { auto *tuple_size = codegen.Const32(storage_format_.GetStorageSize()); codegen.CallFunc(SorterProxy::_Init::GetFunction(codegen), {sorter_ptr, comparison_func, tuple_size}); }