void QueryManagerBase::markOperatorFinished(const dag_node_index index) {
  query_exec_state_->setExecutionFinished(index);

  for (const dag_node_index dependent_op_index : blocking_dependents_[index]) {
    blocking_dependencies_[dependent_op_index].erase(index);
  }

  for (const dag_node_index dependent_op_index : output_consumers_[index]) {
    non_blocking_dependencies_[dependent_op_index].erase(index);
  }

  RelationalOperator *op = query_dag_->getNodePayloadMutable(index);
  op->updateCatalogOnCompletion();

  const relation_id output_rel = op->getOutputRelationID();

  for (const pair<dag_node_index, bool> &dependent_link : query_dag_->getDependents(index)) {
    const dag_node_index dependent_op_index = dependent_link.first;
    if (output_rel >= 0) {
      // Signal dependent operator that current operator is done feeding input blocks.
      query_dag_->getNodePayloadMutable(dependent_op_index)->doneFeedingInputBlocks(output_rel);
    }

    if (checkAllBlockingDependenciesMet(dependent_op_index)) {
      // Process the dependent operator (of the operator whose WorkOrder
      // was just executed) for which all the dependencies have been met.
      if (!fetchNormalWorkOrders(dependent_op_index) &&
          non_blocking_dependencies_[dependent_op_index].empty() &&
          checkNormalExecutionOver(dependent_op_index) &&
          (!checkRebuildRequired(dependent_op_index) || initiateRebuild(dependent_op_index))) {
        markOperatorFinished(dependent_op_index);
      }
    }
  }
}
Example #2
0
void Foreman::processOperator(RelationalOperator *op,
                              dag_node_index index,
                              bool recursively_check_dependents) {
  if (fetchNormalWorkOrders(op, index)) {
    // Fetched work orders. Return to wait for the generated work orders to
    // execute, and skip the execution-finished checks.
    return;
  }

  if (checkNormalExecutionOver(index)) {
    if (checkRebuildRequired(index)) {
      if (!checkRebuildInitiated(index)) {
        // Rebuild hasn't started, initiate it.
        if (initiateRebuild(index)) {
          // Rebuild initiated and completed right away.
          markOperatorFinished(index);
        } else {
          // Rebuild WorkOrders have been generated.
          return;
        }
      } else if (checkRebuildOver(index)) {
        // Rebuild had been initiated and it is over.
        markOperatorFinished(index);
      }
    } else {
      // Rebuild is not required and normal execution over, mark finished.
      markOperatorFinished(index);
    }
    // If we reach here, that means the operator has been marked as finished.
    if (recursively_check_dependents) {
      for (pair<dag_node_index, bool> dependent_link :
           query_dag_->getDependents(index)) {
        if (checkAllBlockingDependenciesMet(dependent_link.first)) {
          processOperator(
              query_dag_->getNodePayloadMutable(dependent_link.first),
              dependent_link.first, true);
        }
      }
    }
  }
}
void QueryManagerBase::processWorkOrderCompleteMessage(
    const dag_node_index op_index,
    const partition_id part_id) {
  query_exec_state_->decrementNumQueuedWorkOrders(op_index);

  if (!checkNormalExecutionOver(op_index)) {
    // Normal execution under progress for this operator.
    return;
  }

  if (checkRebuildRequired(op_index)) {
    DCHECK(!checkRebuildInitiated(op_index));
    if (!initiateRebuild(op_index)) {
      // Rebuild under progress.
      return;
    }
    // Rebuild initiated and completed right away.
  }

  markOperatorFinished(op_index);
}
Example #4
0
void QueryManager::processWorkOrderCompleteMessage(
    const dag_node_index op_index) {
  query_exec_state_->decrementNumQueuedWorkOrders(op_index);

  // Check if new work orders are available and fetch them if so.
  fetchNormalWorkOrders(op_index);

  if (checkRebuildRequired(op_index)) {
    if (checkNormalExecutionOver(op_index)) {
      if (!checkRebuildInitiated(op_index)) {
        if (initiateRebuild(op_index)) {
          // Rebuild initiated and completed right away.
          markOperatorFinished(op_index);
        } else {
          // Rebuild under progress.
        }
      } else if (checkRebuildOver(op_index)) {
        // Rebuild was under progress and now it is over.
        markOperatorFinished(op_index);
      }
    } else {
      // Normal execution under progress for this operator.
    }
  } else if (checkOperatorExecutionOver(op_index)) {
    // Rebuild not required for this operator and its normal execution is
    // complete.
    markOperatorFinished(op_index);
  }

  for (const pair<dag_node_index, bool> &dependent_link :
       query_dag_->getDependents(op_index)) {
    const dag_node_index dependent_op_index = dependent_link.first;
    if (checkAllBlockingDependenciesMet(dependent_op_index)) {
      // Process the dependent operator (of the operator whose WorkOrder
      // was just executed) for which all the dependencies have been met.
      processOperator(dependent_op_index, true);
    }
  }
}
Example #5
0
// TODO(harshad) - There is duplication in terms of functionality provided by
// TMB and ForemanMessage class with respect to determining the message types.
// Try to use TMB message types for infering the messsage types consistently.
bool Foreman::processMessage(const ForemanMessage &message) {
  const dag_node_index dag_size = query_dag_->size();
  // Get the relational operator that caused this message to be sent.
  dag_node_index response_op_index = message.getRelationalOpIndex();
  const int worker_id = message.getWorkerID();

  switch (message.getType()) {
    case ForemanMessage::kWorkOrderCompletion:
      {
        // Completion of a regular WorkOrder.
        DEBUG_ASSERT(worker_id >= 0);
        --queued_workorders_per_op_[response_op_index];
        // As the given worker finished executing a WorkOrder, decrement its
        // number of queued WorkOrders.
        workers_->decrementNumQueuedWorkOrders(worker_id);

        // Check if new work orders are available and fetch them if so.
        fetchNormalWorkOrders(
            query_dag_->getNodePayloadMutable(response_op_index),
            response_op_index);

        if (checkRebuildRequired(response_op_index)) {
          if (checkNormalExecutionOver(response_op_index)) {
            if (!checkRebuildInitiated(response_op_index)) {
              if (initiateRebuild(response_op_index)) {
                // Rebuild initiated and completed right away.
                markOperatorFinished(response_op_index);
              } else {
                // Rebuild under progress.
              }
            } else if (checkRebuildOver(response_op_index)) {
              // Rebuild was under progress and now it is over.
              markOperatorFinished(response_op_index);
            }
          } else {
            // Normal execution under progress for this operator.
          }
        } else if (checkOperatorExecutionOver(response_op_index)) {
          // Rebuild not required for this operator and its normal execution is
          // complete.
          markOperatorFinished(response_op_index);
        }

        for (pair<dag_node_index, bool> dependent_link :
             query_dag_->getDependents(response_op_index)) {
          RelationalOperator *dependent_op =
              query_dag_->getNodePayloadMutable(dependent_link.first);
          if (checkAllBlockingDependenciesMet(dependent_link.first)) {
            // Process the dependent operator (of the operator whose WorkOrder
            // was just executed) for which all the dependencies have been met.
            processOperator(dependent_op, dependent_link.first, true);
          }
        }
      }
      break;
    case ForemanMessage::kRebuildCompletion:
      {
        DEBUG_ASSERT(worker_id >= 0);
        // Completion of a rebuild WorkOrder.
        --rebuild_status_[response_op_index].second;
        workers_->decrementNumQueuedWorkOrders(worker_id);

        if (checkRebuildOver(response_op_index)) {
          markOperatorFinished(response_op_index);
          for (pair<dag_node_index, bool> dependent_link :
               query_dag_->getDependents(response_op_index)) {
            RelationalOperator *dependent_op =
                query_dag_->getNodePayloadMutable(dependent_link.first);
            if (checkAllBlockingDependenciesMet(dependent_link.first)) {
              processOperator(dependent_op, dependent_link.first, true);
            }
          }
        }
      }
      break;
    case ForemanMessage::kDataPipeline:
      {
        // Data streaming message. Possible senders of this message include
        // InsertDestination and some operators which modify existing blocks.
        for (dag_node_index consumer_index :
             output_consumers_[response_op_index]) {
          RelationalOperator *consumer_op =
              query_dag_->getNodePayloadMutable(consumer_index);
          // Feed the streamed block to the consumer. Note that
          // output_consumers_ only contain those dependents of operator with
          // index = response_op_index which are eligible to receive streamed
          // input.
          consumer_op->feedInputBlock(message.getOutputBlockID(),
                                      message.getRelationID());
          // Because of the streamed input just fed, check if there are any new
          // WorkOrders available and if so, fetch them.
          fetchNormalWorkOrders(consumer_op, consumer_index);
        }  // end for (feeding input to dependents)
      }
      break;
    case ForemanMessage::kWorkOrdersAvailable: {
      // Check if new work orders are available.
      fetchNormalWorkOrders(
          query_dag_->getNodePayloadMutable(response_op_index),
          response_op_index);
      break;
    }
    default:
      FATAL_ERROR("Unknown ForemanMessage type");
  }

  // Dispatch the WorkerMessages to the workers. We prefer to start the search
  // for the schedulable WorkOrders beginning from response_op_index. The first
  // candidate worker to receive the next WorkOrder is the one that sent the
  // response message to Foreman.
  dispatchWorkerMessages(((worker_id >= 0) ? worker_id : 0), response_op_index);
  return num_operators_finished_ == dag_size;
}