/**
* Note the process from getting block of child to sending to mergers in
* different buffer.
* if the state_.partition_schema_ is hash partitioned, every tuple of the block
* which get from child will be hash repartition and copied into
* partitioned_block_stream_, if it is full, then
* serialize it and insert into corresponding partition buffer.
* else the state_.partition_schema_ is broadcast, straightly insert the block
* from child into each partition buffer.
*/
bool ExchangeSenderPipeline::Next(SegmentExecStatus* const exec_status,
BlockStreamBase* no_block) {
void* tuple_from_child;
void* tuple_in_cur_block_stream;
while (true) {
RETURN_IF_CANCELLED(exec_status);
block_for_asking_->setEmpty();
if (state_.child_->Next(exec_status, block_for_asking_)) {
RETURN_IF_CANCELLED(exec_status);
/**
* if a blocks is obtained from child, we repartition the tuples in the
* block to corresponding partition_block_stream_.
*/
if (state_.partition_schema_.isHashPartition()) {
BlockStreamBase::BlockStreamTraverseIterator* traverse_iterator =
block_for_asking_->createIterator();
while ((tuple_from_child = traverse_iterator->nextTuple()) > 0) {
/**
* for each tuple in the newly obtained block, insert the tuple to
* one partitioned block according to the partition hash value
*/
const unsigned partition_id = GetHashPartitionId(
tuple_from_child, state_.schema_,
state_.partition_schema_.partition_key_index, upper_num_);
// calculate the tuple size for the current tuple
const unsigned bytes =
state_.schema_->getTupleActualSize(tuple_from_child);
// insert the tuple into the corresponding partitioned block
while (!(tuple_in_cur_block_stream =
partitioned_block_stream_[partition_id]->allocateTuple(
bytes))) {
/**
* if the destination block is full, it should be serialized and
* inserted into the partitioned_data_buffer.
*/
partitioned_block_stream_[partition_id]->serialize(
*block_for_serialization_);
partitioned_data_buffer_->insertBlockToPartitionedList(
block_for_serialization_, partition_id);
partitioned_block_stream_[partition_id]->setEmpty();
}
/**
* thread arriving here means that the space for the tuple is
* successfully allocated, so we copy the tuple
*/
state_.schema_->copyTuple(tuple_from_child,
tuple_in_cur_block_stream);
}
DELETE_PTR(traverse_iterator); // by hAN MEMORY LEAK
} else if (state_.partition_schema_.isBroadcastPartition()) {
/**
* for boardcast case, all block from child should inserted into all
* partitioned_data_buffer
*/
block_for_asking_->serialize(*block_for_serialization_);
for (unsigned i = 0; i < upper_num_; ++i) {
partitioned_data_buffer_->insertBlockToPartitionedList(
block_for_serialization_, i);
}
}
} else {
RETURN_IF_CANCELLED(exec_status);
if (state_.partition_schema_.isHashPartition()) {
/* the child iterator is exhausted. We add the last block stream block
* which would be not full into the buffer for hash partitioned case.
*/
for (unsigned i = 0; i < upper_num_; ++i) {
partitioned_block_stream_[i]->serialize(*block_for_serialization_);
partitioned_data_buffer_->insertBlockToPartitionedList(
block_for_serialization_, i);
}
/* The following lines send an empty block to the upper, indicating that
* all the data from current sent has been transmit to the uppers.
*/
for (unsigned i = 0; i < upper_num_; ++i) {
if (!partitioned_block_stream_[i]->Empty()) {
partitioned_block_stream_[i]->setEmpty();
partitioned_block_stream_[i]->serialize(*block_for_serialization_);
partitioned_data_buffer_->insertBlockToPartitionedList(
block_for_serialization_, i);
}
}
} else if (state_.partition_schema_.isBroadcastPartition()) {
/* The following lines send an empty block to the upper, indicating that
* all the data from current sent has been transmit to the uppers.
*/
block_for_asking_->setEmpty();
block_for_asking_->serialize(*block_for_serialization_);
for (unsigned i = 0; i < upper_num_; ++i) {
partitioned_data_buffer_->insertBlockToPartitionedList(
block_for_serialization_, i);
}
}
/*
* waiting until all the block in the buffer has been
* transformed to the uppers.
*/
LOG(INFO) << "(exchane_id= " << state_.exchange_id_
<< " partition_offset= " << state_.partition_offset_
<< " ) Waiting until all the blocks in the buffer is sent!"
<< std::endl;
RETURN_IF_CANCELLED(exec_status);
while (!partitioned_data_buffer_->isEmpty()) {
RETURN_IF_CANCELLED(exec_status);
usleep(1);
}
/*
* waiting until all the uppers send the close notification which means
* that
* blocks in the uppers' socket buffer have all been
* consumed.
*/
LOG(INFO) << "(exchane_id= " << state_.exchange_id_
<< " partition_offset= " << state_.partition_offset_
<< " ) Waiting for close notification from all merger!"
<< std::endl;
RETURN_IF_CANCELLED(exec_status);
for (unsigned i = 0; i < upper_num_; i++) {
RETURN_IF_CANCELLED(exec_status);
WaitingForCloseNotification(socket_fd_upper_list_[i]);
}
LOG(INFO) << " received all close notification, closing.. " << endl;
return false;
}
}
}
bool ExpandableBlockStreamExchangeLowerEfficient::next(BlockStreamBase*) {
void* tuple_from_child;
void* tuple_in_cur_block_stream;
while (true) {
block_stream_for_asking_->setEmpty();
if (state_.child_->next(block_stream_for_asking_)) {
/** if a blocks is obtained from child, we partition the tuples in the block. **/
if (state_.partition_schema_.isHashPartition()) {
BlockStreamBase::BlockStreamTraverseIterator* traverse_iterator =
block_stream_for_asking_->createIterator();
while ((tuple_from_child = traverse_iterator->nextTuple()) > 0) {
/** for each tuple in the newly obtained block, insert the tuple to one partitioned block according to the
* partition hash value**/
const unsigned partition_id = hash(
tuple_from_child, state_.schema_,
state_.partition_schema_.partition_key_index, nuppers_);
/** calculate the tuple size for the current tuple **/
const unsigned bytes = state_.schema_->getTupleActualSize(
tuple_from_child);
/** insert the tuple into the corresponding partitioned block **/
while (!(tuple_in_cur_block_stream =
partitioned_block_stream_[partition_id]->allocateTuple(bytes))) {
/** if the destination block is full, we insert the block into the buffer **/
partitioned_block_stream_[partition_id]->serialize(
*block_for_serialization_);
partitioned_data_buffer_->insertBlockToPartitionedList(
block_for_serialization_, partition_id);
partitioned_block_stream_[partition_id]->setEmpty();
}
/** thread arriving here means that the space for the tuple is successfully allocated, so we copy the tuple **/
state_.schema_->copyTuple(tuple_from_child,
tuple_in_cur_block_stream);
}
}
else if (state_.partition_schema_.isBoardcastPartition()) {
block_stream_for_asking_->serialize(*block_for_serialization_);
for (unsigned i = 0; i < nuppers_; i++) {
partitioned_data_buffer_->insertBlockToPartitionedList(
block_for_serialization_, i);
}
}
}
else {
/* the child iterator is exhausted. We add the cur block steram block into the buffer*/
for (unsigned i = 0; i < nuppers_; i++) {
partitioned_block_stream_[i]->serialize(*block_for_serialization_);
partitioned_data_buffer_->insertBlockToPartitionedList(
block_for_serialization_, i);
/* The following lines send an empty block to the upper, indicating that all
* the data from current sent has been transmit to the uppers.
*/
if (!partitioned_block_stream_[i]->Empty()) {
partitioned_block_stream_[i]->setEmpty();
partitioned_block_stream_[i]->serialize(*block_for_serialization_);
partitioned_data_buffer_->insertBlockToPartitionedList(
block_for_serialization_, i);
}
}
/*
* waiting until all the block in the buffer has been transformed to the uppers.
*/
logging_->log("Waiting until all the blocks in the buffer is sent!");
while (!partitioned_data_buffer_->isEmpty()) {
usleep(1);
}
/*
* waiting until all the uppers send the close notification which means that
* blocks in the uppers' socket buffer have all been consumed.
*/
logging_->log("Waiting for close notification!");
for (unsigned i = 0; i < nuppers_; i++) {
WaitingForCloseNotification(socket_fd_upper_list[i]);
}
return false;
}
}
}
