void DeleteWorkOrder::execute() {
MutableBlockReference block(
storage_manager_->getBlockMutable(input_block_id_, input_relation_));
block->deleteTuples(predicate_);
// TODO(harshad): Stream the block ID only if the predicate returned at least
// one match in the StorageBlock.
serialization::DataPipelineMessage proto;
proto.set_operator_index(delete_operator_index_);
proto.set_block_id(input_block_id_);
proto.set_relation_id(input_relation_.getID());
// NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
const std::size_t proto_length = proto.ByteSize();
char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
CHECK(proto.SerializeToArray(proto_bytes, proto_length));
tmb::TaggedMessage tagged_message(static_cast<const void *>(proto_bytes),
proto_length,
kDataPipelineMessage);
std::free(proto_bytes);
const tmb::client_id worker_thread_client_id = ClientIDMap::Instance()->getValue();
const tmb::MessageBus::SendStatus send_status =
QueryExecutionUtil::SendTMBMessage(
bus_,
worker_thread_client_id,
scheduler_client_id_,
std::move(tagged_message));
CHECK(send_status == tmb::MessageBus::SendStatus::kOK) << "Message could not"
" be sent from thread with TMB client ID " <<
worker_thread_client_id << " to Foreman with TMB client ID "
<< scheduler_client_id_;
}
void execute() {
if (!block_ref_->rebuild()) {
LOG_WARNING("Rebuilding of StorageBlock with ID: "
<< block_ref_->getID() << " invalidated one or more "
"IndexSubBlocks.");
}
serialization::DataPipelineMessage proto;
proto.set_operator_index(input_operator_index_);
proto.set_block_id(block_ref_->getID());
proto.set_relation_id(input_relation_id_);
// NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
const std::size_t proto_length = proto.ByteSize();
char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
CHECK(proto.SerializeToArray(proto_bytes, proto_length));
tmb::TaggedMessage tagged_message(static_cast<const void *>(proto_bytes),
proto_length,
kDataPipelineMessage);
std::free(proto_bytes);
// Refer to InsertDestination::sendBlockFilledMessage for the rationale
// behind using the ClientIDMap map.
const tmb::MessageBus::SendStatus send_status =
QueryExecutionUtil::SendTMBMessage(bus_,
ClientIDMap::Instance()->getValue(),
foreman_client_id_,
std::move(tagged_message));
CHECK(send_status == tmb::MessageBus::SendStatus::kOK) << "Message could "
" not be sent from thread with TMB client ID " <<
ClientIDMap::Instance()->getValue() << " to Foreman with TMB client ID "
<< foreman_client_id_;
}
void UpdateWorkOrder::execute(QueryContext *query_context,
CatalogDatabase *database,
StorageManager *storage_manager) {
DCHECK(query_context != nullptr);
DCHECK(database != nullptr);
DCHECK(storage_manager != nullptr);
MutableBlockReference block(
storage_manager->getBlockMutable(input_block_id_,
*database->getRelationById(rel_id_)));
InsertDestination *relocation_destination =
query_context->getInsertDestination(relocation_destination_index_);
DCHECK(relocation_destination != nullptr);
StorageBlock::UpdateResult result =
block->update(query_context->getUpdateGroup(update_group_index_),
query_context->getPredicate(predicate_index_),
relocation_destination);
if (!result.indices_consistent) {
LOG_WARNING("An UPDATE caused one or more IndexSubBlocks in StorageBlock "
<< BlockIdUtil::ToString(input_block_id_) << " to become inconsistent "
<< "(not enough space to index all tuples in block).");
}
// TODO(harshad): Stream the block ID only if the predicate returned at least
// one match in the StorageBlock.
serialization::DataPipelineMessage proto;
proto.set_operator_index(update_operator_index_);
proto.set_block_id(input_block_id_);
proto.set_relation_id(rel_id_);
// NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
const std::size_t proto_length = proto.ByteSize();
char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
CHECK(proto.SerializeToArray(proto_bytes, proto_length));
tmb::TaggedMessage tagged_message(static_cast<const void *>(proto_bytes),
proto_length,
kDataPipelineMessage);
std::free(proto_bytes);
const tmb::MessageBus::SendStatus send_status =
QueryExecutionUtil::SendTMBMessage(
bus_,
ClientIDMap::Instance()->getValue(),
foreman_client_id_,
std::move(tagged_message));
CHECK(send_status == tmb::MessageBus::SendStatus::kOK) << "Message could not"
" be sent from thread with TMB client ID " <<
ClientIDMap::Instance()->getValue() << " to Foreman with TMB client ID "
<< foreman_client_id_;
}
/**
* @brief When a StorageBlock becomes full, pipeline the block id to the
* scheduler.
*
* @param id The id of the StorageBlock to be pipelined.
**/
void sendBlockFilledMessage(const block_id id) const {
serialization::DataPipelineMessage proto;
proto.set_operator_index(relational_op_index_);
proto.set_block_id(id);
proto.set_relation_id(relation_.getID());
// NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
const std::size_t proto_length = proto.ByteSize();
char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
CHECK(proto.SerializeToArray(proto_bytes, proto_length));
tmb::TaggedMessage tagged_message(static_cast<const void *>(proto_bytes),
proto_length,
kDataPipelineMessage);
std::free(proto_bytes);
// The reason we use the ClientIDMap is as follows:
// InsertDestination needs to send data pipeline messages to scheduler. To
// send a TMB message, we need to know the sender and receiver's TMB client
// ID. In this case, the sender thread is the worker thread that executes
// this function. To figure out the TMB client ID of the executing thread,
// there are multiple ways :
// 1. Trickle down the worker's client ID all the way from Worker::run()
// method until here.
// 2. Use thread-local storage - Each worker saves its TMB client ID in the
// local storage.
// 3. Use a globally accessible map whose key is the caller thread's
// process level ID and value is the TMB client ID.
//
// Option 1 involves modifying the signature of several functions across
// different modules. Option 2 was difficult to implement given that Apple's
// Clang doesn't allow C++11's thread_local keyword. Therefore we chose
// option 3.
DCHECK(bus_ != nullptr);
const tmb::client_id worker_thread_client_id = thread_id_map_.getValue();
const tmb::MessageBus::SendStatus send_status =
QueryExecutionUtil::SendTMBMessage(bus_,
worker_thread_client_id,
scheduler_client_id_,
std::move(tagged_message));
CHECK(send_status == tmb::MessageBus::SendStatus::kOK) <<
"Message could not be sent from thread with TMB client ID "
<< worker_thread_client_id << " to Scheduler with TMB client"
" ID " << scheduler_client_id_;
}
void Worker::sendWorkOrderCompleteMessage(const tmb::client_id receiver,
const CompletionMessageProtoT &proto,
const message_type_id message_type) {
// NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
const size_t proto_length = proto.ByteSize();
char *proto_bytes = static_cast<char *>(std::malloc(proto_length));
CHECK(proto.SerializeToArray(proto_bytes, proto_length));
TaggedMessage tagged_message(
static_cast<const void *>(proto_bytes), proto_length, message_type);
std::free(proto_bytes);
const tmb::MessageBus::SendStatus send_status =
QueryExecutionUtil::SendTMBMessage(
bus_, worker_client_id_, receiver, std::move(tagged_message));
CHECK(send_status == tmb::MessageBus::SendStatus::kOK)
<< "Message could not be sent from worker with TMB client ID "
<< worker_client_id_ << " to Foreman with TMB client ID " << receiver;
}
void TextSplitWorkOrder::sendBlobInfoToOperator(const bool write_row_aligned) {
std::size_t text_len = written_;
std::string residue;
if (write_row_aligned) {
// Find last row terminator in current blob.
text_len = findLastRowTerminator();
// Copy the residual bytes after the last row terminator.
residue = std::string(
static_cast<char *>(text_blob_->getMemoryMutable()) + text_len,
written_ - text_len);
}
// Notify the operator for the split-up blob.
serialization::TextBlob proto;
proto.set_blob_id(text_blob_id_);
proto.set_size(text_len);
const std::size_t payload_size = proto.ByteSize();
// NOTE(zuyu): 'payload' gets released by FeedbackMessage's destructor.
char *payload = static_cast<char *>(std::malloc(payload_size));
CHECK(proto.SerializeToArray(payload, payload_size));
const tmb::client_id worker_thread_client_id = ClientIDMap::Instance()->getValue();
FeedbackMessage feedback_msg(TextScanOperator::kNewTextBlobMessage,
operator_index_,
payload,
payload_size);
SendFeedbackMessage(bus_, worker_thread_client_id, scheduler_client_id_, feedback_msg);
// Notify Foreman for the avaiable work order on the blob.
serialization::WorkOrdersAvailableMessage message_proto;
message_proto.set_operator_index(operator_index_);
// NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
const size_t message_proto_length = message_proto.ByteSize();
char *message_proto_bytes = static_cast<char*>(std::malloc(message_proto_length));
CHECK(message_proto.SerializeToArray(message_proto_bytes, message_proto_length));
tmb::TaggedMessage tagged_message(static_cast<const void *>(message_proto_bytes),
message_proto_length,
kWorkOrdersAvailableMessage);
std::free(message_proto_bytes);
// Send new work order available message to Foreman.
const tmb::MessageBus::SendStatus send_status =
QueryExecutionUtil::SendTMBMessage(
bus_,
worker_thread_client_id,
scheduler_client_id_,
std::move(tagged_message));
CHECK(send_status == tmb::MessageBus::SendStatus::kOK) << "Message could not "
"be sent from thread with TMB client ID "
<< worker_thread_client_id << " to Foreman with TMB client "
"ID " << scheduler_client_id_;
if (residue.size()) {
// Allocate new blob, and copy residual bytes from last blob.
allocateBlob();
std::memcpy(writeableBlobAddress(), residue.data(), residue.size());
written_ += residue.size();
}
}