static void compute_req(Requirement& r, Coord first, Coord last) {
Coord natural = last - first;
r.natural(natural);
r.stretch(0.0);
r.shrink(0.0);
if (Math::equal(natural, float(0), float(1e-3))) {
r.alignment(0.0);
} else {
r.alignment(-first / natural);
}
}
bool operator<(Requirement rhs, Requirement lhs) {
// Normalize output for comparison
// Grossly inefficient, considering that Clauses themselves sort on
// EACH compare.
rhs.sort();
rhs.unique();
lhs.sort();
lhs.unique();
list<Clause>& rhs_ref = rhs;
list<Clause>& lhs_ref = lhs;
return rhs_ref < lhs_ref;
}
QString RequirementReference::getTitle() const
{
Requirement *source = getSource();
QString title = "[LINK] %1";
if(source){
title = title.arg(source->getTitle());
}
else{
title = title.arg("UNRESOLVED");
}
return title;
}
QTextDocument *RequirementReference::getDescription()
{
QString s = QString("UNRESOLVED LINK to [%1]").arg(targetID, 3, 10, QChar('0'));
Requirement *source = getSource();
if(source){
QString html = source->getDescription()->toHtml();
QString ref = QString("LINK to %1").arg(source->getNumberedTitle());
s = QString("%1<br>\n%2").arg(ref).arg(html);
}
refDescription->setHtml(s);
return refDescription.get();
}
void
Req_Handler::handle_requirement (const Requirement& desc,
Deployment::Requirement& toconfig)
{
DANCE_TRACE("Req_Handler::get_Requirement");
//Map the basic string types to their Deployment::Req
//counterparts.
toconfig.name = ACE_TEXT_ALWAYS_CHAR ( desc.name ().c_str ());
toconfig.resourceType = ACE_TEXT_ALWAYS_CHAR ( desc.resourceType ().c_str ());
toconfig.property.length (desc.count_property ());
std::for_each (desc.begin_property (),
desc.end_property (),
Property_Functor (toconfig.property));
}
std::pair<CombSet::iterator,bool> ReqS_LACP::operator() (
std::list<CombSet::iterator> &local_state_cache,
std::set<int> &dealed_factors,
const Requirement &req)
{
float min_num=m_max_number;
std::list<CombSet::iterator>::iterator selected_it1=local_state_cache.begin();
std::list<CombSet::iterator>::iterator selected_it2=local_state_cache.end();
for(std::list<CombSet::iterator>::iterator it=local_state_cache.begin();
it!=local_state_cache.end();++it)
{
const std::set<int> &inter=(*it)->getParaSet();
std::vector<int> temp(dealed_factors.size()+inter.size(),-1);
std::vector<int>::iterator the_end=set_intersection(
dealed_factors.begin(),dealed_factors.end(),
inter.begin(),inter.end(),temp.begin());
int num=1;
for(std::vector<int>::const_iterator itp=temp.begin();itp!=the_end;++itp)
num*=req.getParaValue(*itp);
float number=((float)(num))/((float)((*it)->Size()));
if(number<min_num && num>1)
{
min_num=number;
selected_it2=it;
}
}
CombSet::iterator selected_one;
bool flag=ModifyLocalStateCache(selected_it1,selected_it2,
local_state_cache,dealed_factors,selected_one);
return std::pair<CombSet::iterator,bool>(selected_one,flag);
}
QString RequirementReference::getNumberedTitle() const
{
QString idString;
if(settings->idIsVisible()){
idString = QString("[%1] ").arg(id, 3, 10, QChar('0'));
}
QString title = "UNRESOLVED";
Requirement *source = getSource();
if(source){
title = source->getTitle();
}
return QString("%1 %2[LINK] %3")
.arg(number())
.arg(idString)
.arg(title);
}
static bool
IsHintInteresting(const Requirement &requirement, const Requirement &hint)
{
if (hint.kind() == Requirement::NONE)
return false;
if (hint.kind() != Requirement::FIXED && hint.kind() != Requirement::REGISTER)
return true;
Requirement merge = requirement;
if (!merge.mergeRequirement(hint))
return true;
return merge.kind() != requirement.kind();
}
bool LogicalScan::GetOptimalPhysicalPlan(Requirement requirement,PhysicalPlanDescriptor& physical_plan_descriptor, const unsigned & block_size){
Dataflow dataflow=getDataflow();
NetworkTransfer transfer=requirement.requireNetworkTransfer(dataflow);
ExpandableBlockStreamProjectionScan::State state;
state.block_size_=block_size;
state.projection_id_=target_projection_->getProjectionID();
state.schema_=getSchema(dataflow_->attribute_list_);
state.sample_rate_=sample_rate_;
PhysicalPlan scan=new ExpandableBlockStreamProjectionScan(state);
if(transfer==NONE){
physical_plan_descriptor.plan=scan;
physical_plan_descriptor.dataflow=dataflow;
physical_plan_descriptor.cost+=0;
}
else{
physical_plan_descriptor.cost+=dataflow.getAggregatedDatasize();
ExpandableBlockStreamExchangeEpoll::State state;
state.block_size_=block_size;
state.child_=scan;//child_iterator;
state.exchange_id_=IDsGenerator::getInstance()->generateUniqueExchangeID();
state.schema_=getSchema(dataflow.attribute_list_);
std::vector<NodeID> lower_id_list=getInvolvedNodeID(dataflow.property_.partitioner);
state.lower_id_list_=lower_id_list;
std::vector<NodeID> upper_id_list;
if(requirement.hasRequiredLocations()){
upper_id_list=requirement.getRequiredLocations();
}
else{
if(requirement.hasRequiredPartitionFunction()){
/* partition function contains the number of partitions*/
PartitionFunction* partitoin_function=requirement.getPartitionFunction();
upper_id_list=std::vector<NodeID>(NodeTracker::getInstance()->getNodeIDList().begin(),NodeTracker::getInstance()->getNodeIDList().begin()+partitoin_function->getNumberOfPartitions()-1);
}
else{
//TODO: decide the degree of parallelism
upper_id_list=NodeTracker::getInstance()->getNodeIDList();
}
}
state.upper_id_list_=upper_id_list;
state.partition_schema_=partition_schema::set_hash_partition(getIndexInAttributeList(dataflow.attribute_list_,requirement.getPartitionKey()));
assert(state.partition_schema_.partition_key_index>=0);
BlockStreamIteratorBase* exchange=new ExpandableBlockStreamExchangeEpoll(state);
Dataflow new_dataflow;
new_dataflow.attribute_list_=dataflow.attribute_list_;
new_dataflow.property_.partitioner.setPartitionKey(requirement.getPartitionKey());
new_dataflow.property_.partitioner.setPartitionFunction(PartitionFunctionFactory::createBoostHashFunction(state.upper_id_list_.size()));
const unsigned total_size=dataflow.getAggregatedDatasize();
const unsigned degree_of_parallelism=state.upper_id_list_.size();
std::vector<DataflowPartition> dataflow_partition_list;
for(unsigned i=0;i<degree_of_parallelism;i++){
const NodeID location=upper_id_list[i];
/* Currently, the join output size cannot be predicted due to the absence of data statistics.
* We just use the magic number as following */
const unsigned datasize=total_size/degree_of_parallelism;
DataflowPartition dfp(i,datasize,location);
dataflow_partition_list.push_back(dfp);
}
new_dataflow.property_.partitioner.setPartitionList(dataflow_partition_list);
physical_plan_descriptor.plan=exchange;
physical_plan_descriptor.dataflow=new_dataflow;
physical_plan_descriptor.cost+=new_dataflow.getAggregatedDatasize();
}
if(requirement.passLimits(physical_plan_descriptor.cost))
return true;
else
return false;
}
bool Filter::GetOptimalPhysicalPlan(Requirement requirement,PhysicalPlanDescriptor& physical_plan_descriptor, const unsigned & block_size){
PhysicalPlanDescriptor physical_plan;
std::vector<PhysicalPlanDescriptor> candidate_physical_plans;
/* no requirement to the child*/
if(child_->GetOptimalPhysicalPlan(Requirement(),physical_plan)){
NetworkTransfer transfer=requirement.requireNetworkTransfer(physical_plan.dataflow);
if(transfer==NONE){
ExpandableBlockStreamFilter::State state;
state.block_size_=block_size;
state.child_=physical_plan.plan;
state.qual_=qual_;
state.colindex_=colindex_;
state.comparator_list_=comparator_list_;
state.v_ei_=exprArray_;
Dataflow dataflow=getDataflow();
state.schema_=getSchema(dataflow.attribute_list_);
BlockStreamIteratorBase* filter=new ExpandableBlockStreamFilter(state);
physical_plan.plan=filter;
candidate_physical_plans.push_back(physical_plan);
}
else if((transfer==OneToOne)||(transfer==Shuffle)){
/* the input data flow should be transfered in the network to meet the requirement
* TODO: implement OneToOne Exchange
* */
ExpandableBlockStreamFilter::State state_f;
state_f.block_size_=block_size;
state_f.child_=physical_plan.plan;
state_f.v_ei_=exprArray_;
state_f.qual_=qual_;
state_f.colindex_=colindex_;
state_f.comparator_list_=comparator_list_;
Dataflow dataflow=getDataflow();
state_f.schema_=getSchema(dataflow.attribute_list_);
BlockStreamIteratorBase* filter=new ExpandableBlockStreamFilter(state_f);
physical_plan.plan=filter;
physical_plan.cost+=physical_plan.dataflow.getAggregatedDatasize();
ExpandableBlockStreamExchangeEpoll::State state;
state.block_size_=block_size;
state.child_=physical_plan.plan;//child_iterator;
state.exchange_id_=IDsGenerator::getInstance()->generateUniqueExchangeID();
state.schema_=getSchema(physical_plan.dataflow.attribute_list_);
std::vector<NodeID> upper_id_list;
if(requirement.hasRequiredLocations()){
upper_id_list=requirement.getRequiredLocations();
}
else{
if(requirement.hasRequiredPartitionFunction()){
/* partition function contains the number of partitions*/
PartitionFunction* partitoin_function=requirement.getPartitionFunction();
upper_id_list=std::vector<NodeID>(NodeTracker::getInstance()->getNodeIDList().begin(),NodeTracker::getInstance()->getNodeIDList().begin()+partitoin_function->getNumberOfPartitions()-1);
}
else{
//TODO: decide the degree of parallelism
upper_id_list=NodeTracker::getInstance()->getNodeIDList();
}
}
state.upper_ip_list_=convertNodeIDListToNodeIPList(upper_id_list);
assert(requirement.hasReuiredPartitionKey());
state.partition_schema_=partition_schema::set_hash_partition(this->getIndexInAttributeList(physical_plan.dataflow.attribute_list_,requirement.getPartitionKey()));
assert(state.partition_schema_.partition_key_index>=0);
std::vector<NodeID> lower_id_list=getInvolvedNodeID(physical_plan.dataflow.property_.partitioner);
state.lower_ip_list_=convertNodeIDListToNodeIPList(lower_id_list);
BlockStreamIteratorBase* exchange=new ExpandableBlockStreamExchangeEpoll(state);
physical_plan.plan=exchange;
}
candidate_physical_plans.push_back(physical_plan);
}
if(child_->GetOptimalPhysicalPlan(requirement,physical_plan)){
ExpandableBlockStreamFilter::State state;
state.block_size_=block_size;
state.child_=physical_plan.plan;
state.v_ei_=exprArray_;
state.qual_=qual_;
state.colindex_=colindex_;
state.comparator_list_=comparator_list_;
Dataflow dataflow=getDataflow();
state.schema_=getSchema(dataflow.attribute_list_);
BlockStreamIteratorBase* filter=new ExpandableBlockStreamFilter(state);
physical_plan.plan=filter;
candidate_physical_plans.push_back(physical_plan);
}
physical_plan_descriptor=getBestPhysicalPlanDescriptor(candidate_physical_plans);
if(requirement.passLimits(physical_plan_descriptor.cost))
return true;
else
return false;
}
bool LogicalQueryPlanRoot::GetOptimalPhysicalPlan(
Requirement requirement, PhysicalPlanDescriptor& final_physical_plan_desc,
const unsigned& block_size) {
std::vector<PhysicalPlanDescriptor> candidate_physical_plan;
Requirement current_req;
current_req.setRequiredLocations(std::vector<NodeID>(1, collecter_node));
Requirement merged_req;
bool requirement_merged = current_req.tryMerge(requirement, merged_req);
if (requirement_merged) {
current_req = merged_req;
}
PhysicalPlanDescriptor physical_plan;
/** no requirement**/
if (child_->GetOptimalPhysicalPlan(Requirement(), physical_plan,
block_size)) {
NetworkTransfer transfer =
current_req.requireNetworkTransfer(physical_plan.plan_context_);
if (transfer == NONE) {
candidate_physical_plan.push_back(physical_plan);
} else if ((transfer == OneToOne) || (transfer == Shuffle)) {
// why transfer is compared with OneToOne, whose type is binding_mode?
// ---Yu
/* the input PlanContext should be transfered in the network to meet the
* requirement
* TODO: implement OneToOne Exchange
* */
physical_plan.cost += physical_plan.plan_context_.GetAggregatedDatasize();
ExchangeMerger::State state;
state.block_size_ = block_size;
state.child_ = physical_plan.plan; // child_iterator;
state.exchange_id_ =
IDsGenerator::getInstance()->generateUniqueExchangeID();
state.schema_ = GetSchema(physical_plan.plan_context_.attribute_list_);
state.upper_id_list_.push_back(collecter_node);
state.partition_schema_ = partition_schema::set_hash_partition(0);
state.lower_id_list_ =
GetInvolvedNodeID(physical_plan.plan_context_.plan_partitioner_);
PhysicalOperatorBase* exchange = new ExchangeMerger(state);
physical_plan.plan = exchange;
}
}
/** with requirement**/
if (child_->GetOptimalPhysicalPlan(current_req, physical_plan, block_size)) {
candidate_physical_plan.push_back(physical_plan);
}
PhysicalPlanDescriptor best_plan =
GetBestPhysicalPlanDescriptor(candidate_physical_plan);
PhysicalPlan final_plan;
switch (style_) {
case kPrint: {
ResultPrinter::State print_state(
GetSchema(best_plan.plan_context_.attribute_list_), best_plan.plan,
block_size, GetAttributeName(physical_plan.plan_context_));
final_plan = new ResultPrinter(print_state);
break;
}
case kPerformance: {
PerformanceMonitor::State performance_state(
GetSchema(best_plan.plan_context_.attribute_list_), best_plan.plan,
block_size);
final_plan = new PerformanceMonitor(performance_state);
}
}
if (requirement_merged) {
final_physical_plan_desc.cost = best_plan.cost;
final_physical_plan_desc.plan_context_ = best_plan.plan_context_;
final_physical_plan_desc.plan = final_plan;
} else {
NetworkTransfer transfer =
current_req.requireNetworkTransfer(best_plan.plan_context_);
if (transfer == NONE) {
final_physical_plan_desc.cost = best_plan.cost;
final_physical_plan_desc.plan_context_ = best_plan.plan_context_;
final_physical_plan_desc.plan = final_plan;
} else if ((transfer == OneToOne) || (transfer == Shuffle)) {
/* the input PlanContext should be transfered in the network to meet the
* requirement
* TODO: implement OneToOne Exchange
* */
ExchangeMerger::State state;
state.block_size_ = block_size;
state.child_ = best_plan.plan; // child_iterator;
state.exchange_id_ =
IDsGenerator::getInstance()->generateUniqueExchangeID();
state.schema_ = GetSchema(best_plan.plan_context_.attribute_list_);
std::vector<NodeID> upper_id_list;
if (requirement.hasRequiredLocations()) {
upper_id_list = requirement.getRequiredLocations();
} else {
if (requirement.hasRequiredPartitionFunction()) {
/* partition function contains the number of partitions*/
PartitionFunction* partitoin_function =
requirement.getPartitionFunction();
upper_id_list = std::vector<NodeID>(
NodeTracker::GetInstance()->GetNodeIDList().begin(),
NodeTracker::GetInstance()->GetNodeIDList().begin() +
partitoin_function->getNumberOfPartitions() - 1);
} else {
// TODO(wangli): decide the degree of parallelism
upper_id_list = NodeTracker::GetInstance()->GetNodeIDList();
}
}
state.upper_id_list_ = upper_id_list;
assert(requirement.hasReuiredPartitionKey());
state.partition_schema_ = partition_schema::set_hash_partition(
this->GetIdInAttributeList(best_plan.plan_context_.attribute_list_,
requirement.getPartitionKey()));
assert(state.partition_schema_.partition_key_index >= 0);
std::vector<NodeID> lower_id_list =
GetInvolvedNodeID(best_plan.plan_context_.plan_partitioner_);
state.lower_id_list_ = lower_id_list;
PhysicalOperatorBase* exchange = new ExchangeMerger(state);
best_plan.plan = exchange;
best_plan.cost += best_plan.plan_context_.GetAggregatedDatasize();
final_physical_plan_desc.cost = best_plan.cost;
final_physical_plan_desc.plan_context_ = best_plan.plan_context_;
final_physical_plan_desc.plan = exchange;
}
}
if (requirement.passLimits(final_physical_plan_desc.cost))
return true;
else
return false;
}
bool LogicalFilter::GetOptimalPhysicalPlan(
Requirement requirement, PhysicalPlanDescriptor& physical_plan_descriptor,
const unsigned& block_size) {
PhysicalPlanDescriptor physical_plan;
std::vector<PhysicalPlanDescriptor> candidate_physical_plans;
/* no requirement to the child*/
if (child_->GetOptimalPhysicalPlan(Requirement(), physical_plan)) {
NetworkTransfer transfer =
requirement.requireNetworkTransfer(physical_plan.plan_context_);
if (NONE == transfer) {
PhysicalFilter::State state;
state.block_size_ = block_size;
state.child_ = physical_plan.plan;
state.qual_ = condi_;
state.column_id_ = column_id_;
PlanContext plan_context = GetPlanContext();
state.schema_ = GetSchema(plan_context.attribute_list_);
PhysicalOperatorBase* filter = new PhysicalFilter(state);
physical_plan.plan = filter;
candidate_physical_plans.push_back(physical_plan);
} else if ((OneToOne == transfer) || (Shuffle == transfer)) {
/**
* The input plan context should be transfered in the network to meet the
* requirement.
* TODO(wangli): Implement OneToOne Exchange
* */
PhysicalFilter::State state_f;
state_f.block_size_ = block_size;
state_f.child_ = physical_plan.plan;
state_f.qual_ = condi_;
state_f.column_id_ = column_id_;
PlanContext plan_context = GetPlanContext();
state_f.schema_ = GetSchema(plan_context.attribute_list_);
PhysicalOperatorBase* filter = new PhysicalFilter(state_f);
physical_plan.plan = filter;
physical_plan.cost += physical_plan.plan_context_.GetAggregatedDatasize();
ExchangeMerger::State state;
state.block_size_ = block_size;
state.child_ = physical_plan.plan; // child_iterator;
state.exchange_id_ =
IDsGenerator::getInstance()->generateUniqueExchangeID();
state.schema_ = GetSchema(physical_plan.plan_context_.attribute_list_);
std::vector<NodeID> upper_id_list;
if (requirement.hasRequiredLocations()) {
upper_id_list = requirement.getRequiredLocations();
} else {
if (requirement.hasRequiredPartitionFunction()) {
// Partition function contains the number of partitions.
PartitionFunction* partitoin_function =
requirement.getPartitionFunction();
upper_id_list = std::vector<NodeID>(
NodeTracker::GetInstance()->GetNodeIDList().begin(),
NodeTracker::GetInstance()->GetNodeIDList().begin() +
partitoin_function->getNumberOfPartitions() - 1);
} else {
// TODO(wangli): decide the degree of parallelism
upper_id_list = NodeTracker::GetInstance()->GetNodeIDList();
}
}
state.upper_id_list_ = upper_id_list;
assert(requirement.hasReuiredPartitionKey());
state.partition_schema_ =
partition_schema::set_hash_partition(this->GetIdInAttributeList(
physical_plan.plan_context_.attribute_list_,
requirement.getPartitionKey()));
assert(state.partition_schema_.partition_key_index >= 0);
std::vector<NodeID> lower_id_list =
GetInvolvedNodeID(physical_plan.plan_context_.plan_partitioner_);
state.lower_id_list_ = lower_id_list;
PhysicalOperatorBase* exchange = new ExchangeMerger(state);
physical_plan.plan = exchange;
}
candidate_physical_plans.push_back(physical_plan);
}
if (child_->GetOptimalPhysicalPlan(requirement, physical_plan)) {
PhysicalFilter::State state;
state.block_size_ = block_size;
state.child_ = physical_plan.plan;
state.column_id_ = column_id_;
PlanContext plan_context = GetPlanContext();
state.schema_ = GetSchema(plan_context.attribute_list_);
PhysicalOperatorBase* filter = new PhysicalFilter(state);
physical_plan.plan = filter;
candidate_physical_plans.push_back(physical_plan);
}
physical_plan_descriptor =
GetBestPhysicalPlanDescriptor(candidate_physical_plans);
if (requirement.passLimits(physical_plan_descriptor.cost))
return true;
else
return false;
}
void DebugGlyph::print_requirement(const Requirement& r) {
printf(
"%.2f(+%.2f,-%.2f) @ %.2f",
r.natural(), r.stretch(), r.shrink(), r.alignment()
);
}