void DeleteWorkOrder::execute() { MutableBlockReference block( storage_manager_->getBlockMutable(input_block_id_, input_relation_)); block->deleteTuples(predicate_); // TODO(harshad): Stream the block ID only if the predicate returned at least // one match in the StorageBlock. serialization::DataPipelineMessage proto; proto.set_operator_index(delete_operator_index_); proto.set_block_id(input_block_id_); proto.set_relation_id(input_relation_.getID()); // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string. const std::size_t proto_length = proto.ByteSize(); char *proto_bytes = static_cast<char*>(std::malloc(proto_length)); CHECK(proto.SerializeToArray(proto_bytes, proto_length)); tmb::TaggedMessage tagged_message(static_cast<const void *>(proto_bytes), proto_length, kDataPipelineMessage); std::free(proto_bytes); const tmb::client_id worker_thread_client_id = ClientIDMap::Instance()->getValue(); const tmb::MessageBus::SendStatus send_status = QueryExecutionUtil::SendTMBMessage( bus_, worker_thread_client_id, scheduler_client_id_, std::move(tagged_message)); CHECK(send_status == tmb::MessageBus::SendStatus::kOK) << "Message could not" " be sent from thread with TMB client ID " << worker_thread_client_id << " to Foreman with TMB client ID " << scheduler_client_id_; }
void execute() { if (!block_ref_->rebuild()) { LOG_WARNING("Rebuilding of StorageBlock with ID: " << block_ref_->getID() << " invalidated one or more " "IndexSubBlocks."); } serialization::DataPipelineMessage proto; proto.set_operator_index(input_operator_index_); proto.set_block_id(block_ref_->getID()); proto.set_relation_id(input_relation_id_); // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string. const std::size_t proto_length = proto.ByteSize(); char *proto_bytes = static_cast<char*>(std::malloc(proto_length)); CHECK(proto.SerializeToArray(proto_bytes, proto_length)); tmb::TaggedMessage tagged_message(static_cast<const void *>(proto_bytes), proto_length, kDataPipelineMessage); std::free(proto_bytes); // Refer to InsertDestination::sendBlockFilledMessage for the rationale // behind using the ClientIDMap map. const tmb::MessageBus::SendStatus send_status = QueryExecutionUtil::SendTMBMessage(bus_, ClientIDMap::Instance()->getValue(), foreman_client_id_, std::move(tagged_message)); CHECK(send_status == tmb::MessageBus::SendStatus::kOK) << "Message could " " not be sent from thread with TMB client ID " << ClientIDMap::Instance()->getValue() << " to Foreman with TMB client ID " << foreman_client_id_; }
void UpdateWorkOrder::execute(QueryContext *query_context, CatalogDatabase *database, StorageManager *storage_manager) { DCHECK(query_context != nullptr); DCHECK(database != nullptr); DCHECK(storage_manager != nullptr); MutableBlockReference block( storage_manager->getBlockMutable(input_block_id_, *database->getRelationById(rel_id_))); InsertDestination *relocation_destination = query_context->getInsertDestination(relocation_destination_index_); DCHECK(relocation_destination != nullptr); StorageBlock::UpdateResult result = block->update(query_context->getUpdateGroup(update_group_index_), query_context->getPredicate(predicate_index_), relocation_destination); if (!result.indices_consistent) { LOG_WARNING("An UPDATE caused one or more IndexSubBlocks in StorageBlock " << BlockIdUtil::ToString(input_block_id_) << " to become inconsistent " << "(not enough space to index all tuples in block)."); } // TODO(harshad): Stream the block ID only if the predicate returned at least // one match in the StorageBlock. serialization::DataPipelineMessage proto; proto.set_operator_index(update_operator_index_); proto.set_block_id(input_block_id_); proto.set_relation_id(rel_id_); // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string. const std::size_t proto_length = proto.ByteSize(); char *proto_bytes = static_cast<char*>(std::malloc(proto_length)); CHECK(proto.SerializeToArray(proto_bytes, proto_length)); tmb::TaggedMessage tagged_message(static_cast<const void *>(proto_bytes), proto_length, kDataPipelineMessage); std::free(proto_bytes); const tmb::MessageBus::SendStatus send_status = QueryExecutionUtil::SendTMBMessage( bus_, ClientIDMap::Instance()->getValue(), foreman_client_id_, std::move(tagged_message)); CHECK(send_status == tmb::MessageBus::SendStatus::kOK) << "Message could not" " be sent from thread with TMB client ID " << ClientIDMap::Instance()->getValue() << " to Foreman with TMB client ID " << foreman_client_id_; }
/** * @brief When a StorageBlock becomes full, pipeline the block id to the * scheduler. * * @param id The id of the StorageBlock to be pipelined. **/ void sendBlockFilledMessage(const block_id id) const { serialization::DataPipelineMessage proto; proto.set_operator_index(relational_op_index_); proto.set_block_id(id); proto.set_relation_id(relation_.getID()); // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string. const std::size_t proto_length = proto.ByteSize(); char *proto_bytes = static_cast<char*>(std::malloc(proto_length)); CHECK(proto.SerializeToArray(proto_bytes, proto_length)); tmb::TaggedMessage tagged_message(static_cast<const void *>(proto_bytes), proto_length, kDataPipelineMessage); std::free(proto_bytes); // The reason we use the ClientIDMap is as follows: // InsertDestination needs to send data pipeline messages to scheduler. To // send a TMB message, we need to know the sender and receiver's TMB client // ID. In this case, the sender thread is the worker thread that executes // this function. To figure out the TMB client ID of the executing thread, // there are multiple ways : // 1. Trickle down the worker's client ID all the way from Worker::run() // method until here. // 2. Use thread-local storage - Each worker saves its TMB client ID in the // local storage. // 3. Use a globally accessible map whose key is the caller thread's // process level ID and value is the TMB client ID. // // Option 1 involves modifying the signature of several functions across // different modules. Option 2 was difficult to implement given that Apple's // Clang doesn't allow C++11's thread_local keyword. Therefore we chose // option 3. DCHECK(bus_ != nullptr); const tmb::client_id worker_thread_client_id = thread_id_map_.getValue(); const tmb::MessageBus::SendStatus send_status = QueryExecutionUtil::SendTMBMessage(bus_, worker_thread_client_id, scheduler_client_id_, std::move(tagged_message)); CHECK(send_status == tmb::MessageBus::SendStatus::kOK) << "Message could not be sent from thread with TMB client ID " << worker_thread_client_id << " to Scheduler with TMB client" " ID " << scheduler_client_id_; }
void Worker::sendWorkOrderCompleteMessage(const tmb::client_id receiver, const CompletionMessageProtoT &proto, const message_type_id message_type) { // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string. const size_t proto_length = proto.ByteSize(); char *proto_bytes = static_cast<char *>(std::malloc(proto_length)); CHECK(proto.SerializeToArray(proto_bytes, proto_length)); TaggedMessage tagged_message( static_cast<const void *>(proto_bytes), proto_length, message_type); std::free(proto_bytes); const tmb::MessageBus::SendStatus send_status = QueryExecutionUtil::SendTMBMessage( bus_, worker_client_id_, receiver, std::move(tagged_message)); CHECK(send_status == tmb::MessageBus::SendStatus::kOK) << "Message could not be sent from worker with TMB client ID " << worker_client_id_ << " to Foreman with TMB client ID " << receiver; }
void TextSplitWorkOrder::sendBlobInfoToOperator(const bool write_row_aligned) { std::size_t text_len = written_; std::string residue; if (write_row_aligned) { // Find last row terminator in current blob. text_len = findLastRowTerminator(); // Copy the residual bytes after the last row terminator. residue = std::string( static_cast<char *>(text_blob_->getMemoryMutable()) + text_len, written_ - text_len); } // Notify the operator for the split-up blob. serialization::TextBlob proto; proto.set_blob_id(text_blob_id_); proto.set_size(text_len); const std::size_t payload_size = proto.ByteSize(); // NOTE(zuyu): 'payload' gets released by FeedbackMessage's destructor. char *payload = static_cast<char *>(std::malloc(payload_size)); CHECK(proto.SerializeToArray(payload, payload_size)); const tmb::client_id worker_thread_client_id = ClientIDMap::Instance()->getValue(); FeedbackMessage feedback_msg(TextScanOperator::kNewTextBlobMessage, operator_index_, payload, payload_size); SendFeedbackMessage(bus_, worker_thread_client_id, scheduler_client_id_, feedback_msg); // Notify Foreman for the avaiable work order on the blob. serialization::WorkOrdersAvailableMessage message_proto; message_proto.set_operator_index(operator_index_); // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string. const size_t message_proto_length = message_proto.ByteSize(); char *message_proto_bytes = static_cast<char*>(std::malloc(message_proto_length)); CHECK(message_proto.SerializeToArray(message_proto_bytes, message_proto_length)); tmb::TaggedMessage tagged_message(static_cast<const void *>(message_proto_bytes), message_proto_length, kWorkOrdersAvailableMessage); std::free(message_proto_bytes); // Send new work order available message to Foreman. const tmb::MessageBus::SendStatus send_status = QueryExecutionUtil::SendTMBMessage( bus_, worker_thread_client_id, scheduler_client_id_, std::move(tagged_message)); CHECK(send_status == tmb::MessageBus::SendStatus::kOK) << "Message could not " "be sent from thread with TMB client ID " << worker_thread_client_id << " to Foreman with TMB client " "ID " << scheduler_client_id_; if (residue.size()) { // Allocate new blob, and copy residual bytes from last blob. allocateBlob(); std::memcpy(writeableBlobAddress(), residue.data(), residue.size()); written_ += residue.size(); } }