BlockStreamIteratorBase* EqualJoin::getIteratorTree(const unsigned& block_size){
if(dataflow_==0){
getDataflow();
}
BlockStreamJoinIterator* join_iterator;
BlockStreamIteratorBase* child_iterator_left=left_child_->getIteratorTree(block_size);
BlockStreamIteratorBase* child_iterator_right=right_child_->getIteratorTree(block_size);
Dataflow dataflow_left=left_child_->getDataflow();
Dataflow dataflow_right=right_child_->getDataflow();
BlockStreamJoinIterator::State state;
state.block_size_=block_size;
state.ht_nbuckets=1024*1024;
// state.ht_nbuckets=1024;
state.input_schema_left=getSchema(dataflow_left.attribute_list_);
state.input_schema_right=getSchema(dataflow_right.attribute_list_);
state.ht_schema=getSchema(dataflow_left.attribute_list_);
/* the bucket size is 64-byte-aligned */
// state.ht_bucketsize=((state.input_schema_left->getTupleMaxSize()-1)/64+1)*64;
/*
* In the initial implementation, I set the bucket size to be up round to cache line
* size, e.g., 64Bytes. Finally, I realized that different from aggregation, the hash
* table bucket in the build phase of hash join is filled very quickly and hence a
* a relatively large bucket size could reduce the number of overflowing buckets and
* avoid the random memory access caused by acceesing overflowing buckets.
*/
state.ht_bucketsize=128;
state.output_schema=getSchema(dataflow_->attribute_list_);
state.joinIndex_left=getLeftJoinKeyIndexList();
state.joinIndex_right=getRightJoinKeyIndexList();
state.payload_left=getLeftPayloadIndexList();
state.payload_right=getRightPayloadIndexList();
switch(join_police_){
case no_repartition:{
state.child_left=child_iterator_left;
state.child_right=child_iterator_right;
join_iterator=new BlockStreamJoinIterator(state);
break;
}
case left_repartition:{
// state.child_left
BlockStreamExpander::State expander_state;
expander_state.block_count_in_buffer_=EXPANDER_BUFFER_SIZE;
expander_state.block_size_=block_size;
expander_state.init_thread_count_=Config::initial_degree_of_parallelism;
expander_state.child_=child_iterator_left;
expander_state.schema_=getSchema(dataflow_left.attribute_list_);
BlockStreamIteratorBase* expander=new BlockStreamExpander(expander_state);
NodeTracker* node_tracker=NodeTracker::getInstance();
ExpandableBlockStreamExchangeEpoll::State exchange_state;
exchange_state.block_size_=block_size;
exchange_state.child_=expander;//child_iterator_left;
exchange_state.exchange_id_=IDsGenerator::getInstance()->generateUniqueExchangeID();
std::vector<NodeID> upper_id_list=getInvolvedNodeID(dataflow_->property_.partitioner);
exchange_state.upper_ip_list_=convertNodeIDListToNodeIPList(upper_id_list);
std::vector<NodeID> lower_id_list=getInvolvedNodeID(dataflow_left.property_.partitioner);
exchange_state.lower_ip_list_=convertNodeIDListToNodeIPList(lower_id_list);
const Attribute right_partition_key=dataflow_->property_.partitioner.getPartitionKey();
/* get the left attribute that is corresponding to the partition key.*/
Attribute left_partition_key=joinkey_pair_list_[getIndexInRightJoinKeyList(right_partition_key)].first;
exchange_state.partition_schema_=partition_schema::set_hash_partition(getIndexInAttributeList(dataflow_left.attribute_list_,left_partition_key));
// exchange_state.schema=getSchema(dataflow_left.attribute_list_,dataflow_right.attribute_list_);
exchange_state.schema_=getSchema(dataflow_left.attribute_list_);
BlockStreamIteratorBase* exchange=new ExpandableBlockStreamExchangeEpoll(exchange_state);
state.child_left=exchange;
state.child_right=child_iterator_right;
join_iterator=new BlockStreamJoinIterator(state);
break;
}
case right_repartition:{
BlockStreamExpander::State expander_state;
expander_state.block_count_in_buffer_=EXPANDER_BUFFER_SIZE;
expander_state.block_size_=block_size;
expander_state.init_thread_count_=Config::initial_degree_of_parallelism;
expander_state.child_=child_iterator_right;
expander_state.schema_=getSchema(dataflow_right.attribute_list_);
BlockStreamIteratorBase* expander=new BlockStreamExpander(expander_state);
NodeTracker* node_tracker=NodeTracker::getInstance();
ExpandableBlockStreamExchangeEpoll::State exchange_state;
exchange_state.block_size_=block_size;
exchange_state.child_=expander;
exchange_state.exchange_id_=IDsGenerator::getInstance()->generateUniqueExchangeID();
std::vector<NodeID> upper_id_list=getInvolvedNodeID(dataflow_->property_.partitioner);
exchange_state.upper_ip_list_=convertNodeIDListToNodeIPList(upper_id_list);
std::vector<NodeID> lower_id_list=getInvolvedNodeID(dataflow_right.property_.partitioner);
exchange_state.lower_ip_list_=convertNodeIDListToNodeIPList(lower_id_list);
const Attribute output_partition_key=dataflow_->property_.partitioner.getPartitionKey();
/* get the right attribute that is corresponding to the partition key.*/
Attribute right_repartition_key;
if(dataflow_->property_.partitioner.hasShadowPartitionKey()){
right_repartition_key=joinkey_pair_list_[getIndexInLeftJoinKeyList(output_partition_key,dataflow_->property_.partitioner.getShadowAttributeList())].second;
}
else{
right_repartition_key=joinkey_pair_list_[getIndexInLeftJoinKeyList(output_partition_key)].second;
}
exchange_state.partition_schema_=partition_schema::set_hash_partition(getIndexInAttributeList(dataflow_right.attribute_list_,right_repartition_key));
exchange_state.schema_=getSchema(dataflow_right.attribute_list_);
BlockStreamIteratorBase* exchange=new ExpandableBlockStreamExchangeEpoll(exchange_state);
state.child_left=child_iterator_left;
state.child_right=exchange;
join_iterator=new BlockStreamJoinIterator(state);
break;
}
case complete_repartition:{
/* build left input*/
BlockStreamExpander::State expander_state_l;
expander_state_l.block_count_in_buffer_=EXPANDER_BUFFER_SIZE;
expander_state_l.block_size_=block_size;
expander_state_l.init_thread_count_=Config::initial_degree_of_parallelism;
expander_state_l.child_=child_iterator_left;
expander_state_l.schema_=getSchema(dataflow_left.attribute_list_);
BlockStreamIteratorBase* expander_l=new BlockStreamExpander(expander_state_l);
ExpandableBlockStreamExchangeEpoll::State l_exchange_state;
l_exchange_state.block_size_=block_size;
l_exchange_state.child_=expander_l;
l_exchange_state.exchange_id_=IDsGenerator::getInstance()->generateUniqueExchangeID();
std::vector<NodeID> lower_id_list=getInvolvedNodeID(dataflow_left.property_.partitioner);
l_exchange_state.lower_ip_list_=convertNodeIDListToNodeIPList(lower_id_list);
std::vector<NodeID> upper_id_list=getInvolvedNodeID(dataflow_->property_.partitioner);
l_exchange_state.upper_ip_list_=convertNodeIDListToNodeIPList(upper_id_list);
const Attribute left_partition_key=dataflow_->property_.partitioner.getPartitionKey();
l_exchange_state.partition_schema_=partition_schema::set_hash_partition(getIndexInAttributeList(dataflow_left.attribute_list_,left_partition_key));
l_exchange_state.schema_=getSchema(dataflow_left.attribute_list_);
BlockStreamIteratorBase* l_exchange=new ExpandableBlockStreamExchangeEpoll(l_exchange_state);
/*build right input*/
BlockStreamExpander::State expander_state_r;
expander_state_r.block_count_in_buffer_=EXPANDER_BUFFER_SIZE;
expander_state_r.block_size_=block_size;
expander_state_r.init_thread_count_=Config::initial_degree_of_parallelism;
expander_state_r.child_=child_iterator_right;
expander_state_r.schema_=getSchema(dataflow_right.attribute_list_);
BlockStreamIteratorBase* expander_r=new BlockStreamExpander(expander_state_r);
ExpandableBlockStreamExchangeEpoll::State r_exchange_state;
r_exchange_state.block_size_=block_size;
r_exchange_state.child_=expander_r;
r_exchange_state.exchange_id_=IDsGenerator::getInstance()->generateUniqueExchangeID();
lower_id_list=getInvolvedNodeID(dataflow_right.property_.partitioner);
r_exchange_state.lower_ip_list_=convertNodeIDListToNodeIPList(lower_id_list);
upper_id_list=getInvolvedNodeID(dataflow_->property_.partitioner);
r_exchange_state.upper_ip_list_=convertNodeIDListToNodeIPList(upper_id_list);
const Attribute right_partition_key=joinkey_pair_list_[getIndexInLeftJoinKeyList(left_partition_key)].second;
r_exchange_state.partition_schema_=partition_schema::set_hash_partition(getIndexInAttributeList(dataflow_right.attribute_list_,right_partition_key));
r_exchange_state.schema_=getSchema(dataflow_right.attribute_list_);
BlockStreamIteratorBase* r_exchange=new ExpandableBlockStreamExchangeEpoll(r_exchange_state);
/* finally build the join iterator itself*/
state.child_left=l_exchange;
state.child_right=r_exchange;
join_iterator=new BlockStreamJoinIterator(state);
break;
}
default:{
break;
}
}
return join_iterator;
}
bool LogicalScan::GetOptimalPhysicalPlan(Requirement requirement,PhysicalPlanDescriptor& physical_plan_descriptor, const unsigned & block_size){
Dataflow dataflow=getDataflow();
NetworkTransfer transfer=requirement.requireNetworkTransfer(dataflow);
ExpandableBlockStreamProjectionScan::State state;
state.block_size_=block_size;
state.projection_id_=target_projection_->getProjectionID();
state.schema_=getSchema(dataflow_->attribute_list_);
state.sample_rate_=sample_rate_;
PhysicalPlan scan=new ExpandableBlockStreamProjectionScan(state);
if(transfer==NONE){
physical_plan_descriptor.plan=scan;
physical_plan_descriptor.dataflow=dataflow;
physical_plan_descriptor.cost+=0;
}
else{
physical_plan_descriptor.cost+=dataflow.getAggregatedDatasize();
ExpandableBlockStreamExchangeEpoll::State state;
state.block_size_=block_size;
state.child_=scan;//child_iterator;
state.exchange_id_=IDsGenerator::getInstance()->generateUniqueExchangeID();
state.schema_=getSchema(dataflow.attribute_list_);
std::vector<NodeID> lower_id_list=getInvolvedNodeID(dataflow.property_.partitioner);
state.lower_id_list_=lower_id_list;
std::vector<NodeID> upper_id_list;
if(requirement.hasRequiredLocations()){
upper_id_list=requirement.getRequiredLocations();
}
else{
if(requirement.hasRequiredPartitionFunction()){
/* partition function contains the number of partitions*/
PartitionFunction* partitoin_function=requirement.getPartitionFunction();
upper_id_list=std::vector<NodeID>(NodeTracker::getInstance()->getNodeIDList().begin(),NodeTracker::getInstance()->getNodeIDList().begin()+partitoin_function->getNumberOfPartitions()-1);
}
else{
//TODO: decide the degree of parallelism
upper_id_list=NodeTracker::getInstance()->getNodeIDList();
}
}
state.upper_id_list_=upper_id_list;
state.partition_schema_=partition_schema::set_hash_partition(getIndexInAttributeList(dataflow.attribute_list_,requirement.getPartitionKey()));
assert(state.partition_schema_.partition_key_index>=0);
BlockStreamIteratorBase* exchange=new ExpandableBlockStreamExchangeEpoll(state);
Dataflow new_dataflow;
new_dataflow.attribute_list_=dataflow.attribute_list_;
new_dataflow.property_.partitioner.setPartitionKey(requirement.getPartitionKey());
new_dataflow.property_.partitioner.setPartitionFunction(PartitionFunctionFactory::createBoostHashFunction(state.upper_id_list_.size()));
const unsigned total_size=dataflow.getAggregatedDatasize();
const unsigned degree_of_parallelism=state.upper_id_list_.size();
std::vector<DataflowPartition> dataflow_partition_list;
for(unsigned i=0;i<degree_of_parallelism;i++){
const NodeID location=upper_id_list[i];
/* Currently, the join output size cannot be predicted due to the absence of data statistics.
* We just use the magic number as following */
const unsigned datasize=total_size/degree_of_parallelism;
DataflowPartition dfp(i,datasize,location);
dataflow_partition_list.push_back(dfp);
}
new_dataflow.property_.partitioner.setPartitionList(dataflow_partition_list);
physical_plan_descriptor.plan=exchange;
physical_plan_descriptor.dataflow=new_dataflow;
physical_plan_descriptor.cost+=new_dataflow.getAggregatedDatasize();
}
if(requirement.passLimits(physical_plan_descriptor.cost))
return true;
else
return false;
}