void* ResultCollector::CollectResult(void* arg) { ResultCollector* Pthis = (ResultCollector*)arg; Pthis->state_.child_->Open(Pthis->state_.partition_offset_); BlockStreamBase* block_for_asking; if (false == Pthis->CreateBlockStream(block_for_asking)) { assert(false); return 0; } Pthis->block_buffer_->column_header_list_ = Pthis->state_.column_header_; unsigned long long start = 0; start = curtick(); while (Pthis->state_.child_->Next(block_for_asking)) { Pthis->block_buffer_->atomicAppendNewBlock(block_for_asking); if (false == Pthis->CreateBlockStream(block_for_asking)) { assert(false); return 0; } } Pthis->sema_input_complete_.post(); double eclipsed_seconds = getSecond(start); Pthis->block_buffer_->query_time_ = eclipsed_seconds; Pthis->block_buffer_->schema_ = Pthis->state_.input_->duplicateSchema(); Pthis->finished_thread_count_++; return 0; }
static void test_scan_filter_performance(int value) { unsigned long long int start=curtick(); TableDescriptor* table=Catalog::getInstance()->getTable("cj"); LogicalOperator* cj_scan=new LogicalScan(table->getProjectoin(0)); Filter::Condition filter_condition_1; filter_condition_1.add(table->getAttribute(3),AttributeComparator::GEQ,std::string("10107")); filter_condition_1.add(table->getAttribute(3),AttributeComparator::L,(void*)&value); LogicalOperator* filter_1=new Filter(filter_condition_1,cj_scan); const NodeID collector_node_id=0; LogicalOperator* root=new LogicalQueryPlanRoot(collector_node_id,filter_1,LogicalQueryPlanRoot::PERFORMANCE); BlockStreamPerformanceMonitorTop* executable_query_plan=(BlockStreamPerformanceMonitorTop*)root->getIteratorTree(1024*64); // executable_query_plan->print(); executable_query_plan->open(); while(executable_query_plan->next(0)); executable_query_plan->close(); // ResultSet *result_set=executable_query_plan->getResultSet(); const unsigned long int number_of_tuples=executable_query_plan->getNumberOfTuples(); printf("execution time: %4.4f seconds.\n",getSecond(start)); if(!print_test_name_result(number_of_tuples==26820,"Low selectivity filter")){ printf("\tExpected:26695 actual: %d\n",number_of_tuples); } // result_set->~ResultSet(); executable_query_plan->~BlockStreamIteratorBase(); root->~LogicalOperator(); }
static void query_select_sort() { /* * select sum(a+1)+count(a),b * from T * group by b * * notation: p a p s * */ unsigned long long int start=curtick(); TableDescriptor* table=Environment::getInstance()->getCatalog()->getTable("LINEITEM"); //===========================scan=========================== LogicalOperator* scan=new LogicalScan(table->getProjectoin(0)); //==========================project========================= vector< vector<ExpressionItem> >expr_list1; vector<ExpressionItem> expr1; ExpressionItem ei1_1; ExpressionItem ei1_2; ExpressionItem ei1_3; ei1_1.setVariable("LINEITEM.L_ORDERKEY"); ei1_2.setIntValue("1"); ei1_3.setOperator("+"); expr1.push_back(ei1_1); expr1.push_back(ei1_2); expr1.push_back(ei1_3); expr_list1.push_back(expr1); LogicalOperator* project1=new LogicalProject(scan,expr_list1); //==========================project========================= vector< vector<ExpressionItem> >expr_list2; ExpressionItem ei21_1; ei21_1.setVariable("LINEITEM.L_ORDERKEY+1"); vector<ExpressionItem> expr21; expr21.push_back(ei21_1); expr_list2.push_back(expr21); LogicalOperator* project2=new LogicalProject(project1,expr_list2); //============================sort========================== vector<LogicalSort::OrderByAttr*> vo; LogicalSort::OrderByAttr tmp=LogicalSort::OrderByAttr("LINEITEM.L_ORDERKEY+1",0); vo.push_back(&tmp); LogicalOperator* sort=new LogicalSort(project1,vo); //===========================root=========================== LogicalOperator* root=new LogicalQueryPlanRoot(0,sort,LogicalQueryPlanRoot::PRINT); cout<<"performance is ok!"<<endl; BlockStreamIteratorBase* physical_iterator_tree=root->getIteratorTree(64*1024); // physical_iterator_tree->print(); physical_iterator_tree->open(); while(physical_iterator_tree->next(0)); physical_iterator_tree->close(); printf("Q1: execution time: %4.4f second.\n",getSecond(start)); }
unsigned TimeOutReceiver::TimeOutWait(unsigned expected_message_count,unsigned time_out_in_ms){ unsigned long long int start=curtick(); unsigned count(0); while(count<expected_message_count&&getMilliSecond(start)<time_out_in_ms){ count+=Consume(expected_message_count-count); } return count; }
// TODO(Hanzhang): According to AVOID_CONTENTION_IN_SCAN, we choose the // strategy. We need finish case(1). bool PhysicalProjectionScan::Next(SegmentExecStatus* const exec_status, BlockStreamBase* block) { RETURN_IF_CANCELLED(exec_status); unsigned long long total_start = curtick(); if (!block->isIsReference()) { block->setIsReference(false); } #ifdef AVOID_CONTENTION_IN_SCAN ScanThreadContext* stc = reinterpret_cast<ScanThreadContext*>(GetContext()); if (NULL == stc) { stc = new ScanThreadContext(); InitContext(stc); } if (ExpanderTracker::getInstance()->isExpandedThreadCallBack( pthread_self())) { input_dataset_.AtomicPut(stc->assigned_data_); delete stc; destorySelfContext(); kPerfInfo->report_instance_performance_in_millibytes(); return false; } if (!stc->assigned_data_.empty()) { ChunkReaderIterator::block_accessor* ba = stc->assigned_data_.front(); stc->assigned_data_.pop_front(); ba->GetBlock(block); // whether delete InMemeryBlockAccessor::target_block_start_address // is depend on whether use copy in ba->getBlock(block); delete ba; kPerfInfo->processed_one_block(); return true; } else { if (input_dataset_.AtomicGet(stc->assigned_data_, Config::scan_batch)) { // case(1) return Next(block); } else { delete stc; destorySelfContext(); return false; } } #else if (ExpanderTracker::getInstance()->isExpandedThreadCallBack( pthread_self())) { return false; } // perf_info_->processed_one_block(); // case(2) RETURN_IF_CANCELLED(exec_status); return partition_reader_iterator_->NextBlock(block); #endif }
bool BlockStreamPerformanceMonitorTop::open(const PartitionOffset& partition_offset){ start_=curtick(); state_.child_->open(partition_offset); block_=BlockStreamBase::createBlock(state_.schema_,state_.block_size_); tuplecount_=0; int error; error=pthread_create(&report_tid_,NULL,report,this); if(error!=0){ std::cout<<"create threads error!"<<std::endl; } return true; }
bool PerformanceIteratorTop::open(){ state.child->open(); tuple=memalign(cacheline_size,state.schema->getTupleMaxSize()); tuplecount=0; int error; error=pthread_create(&report_tid,NULL,report,this); if(error!=0){ std::cout<<"create threads error!"<<std::endl; } start=curtick(); return true; }
bool BlockStreamNestLoopJoinIterator::open(const PartitionOffset& partition_offset) { RegisterExpandedThreadToAllBarriers(); // AtomicPushFreeHtBlockStream(BlockStreamBase::createBlock(state_.input_schema_left,state_.block_size_)); // AtomicPushFreeBlockStream(BlockStreamBase::createBlock(state_.input_schema_right,state_.block_size_)); unsigned long long int timer; bool winning_thread=false; if(tryEntryIntoSerializedSection(0))//the first thread of all need to do { ExpanderTracker::getInstance()->addNewStageEndpoint(pthread_self(),LocalStageEndPoint(stage_desc,"nest loop build",0)); winning_thread=true; timer=curtick(); // unsigned output_index=0; // for(unsigned i=0;i<state_.joinIndex_left.size();i++){ // joinIndex_left_to_output[i]=output_index; // output_index++; // } // for(unsigned i=0;i<state_.payload_left.size();i++){ // payload_left_to_output[i]=output_index; // output_index++; // } // for(unsigned i=0;i<state_.payload_right.size();i++){ // payload_right_to_output[i]=output_index; // output_index++; // } blockbuffer=new DynamicBlockBuffer(); } state_.child_left->open(partition_offset); barrierArrive(0); join_thread_context* jtc=new join_thread_context(); createBlockStream(jtc->block_for_asking_); while(state_.child_left->next(jtc->block_for_asking_)) { blockbuffer->atomicAppendNewBlock(jtc->block_for_asking_); createBlockStream(jtc->block_for_asking_); } delete jtc->block_for_asking_; if(ExpanderTracker::getInstance()->isExpandedThreadCallBack(pthread_self())){ unregisterExpandedThreadToAllBarriers(1); return true; } barrierArrive(1);//??ERROR // join_thread_context* jtc=new join_thread_context(); jtc->block_for_asking_=BlockStreamBase::createBlock(state_.input_schema_right,state_.block_size_); jtc->block_stream_iterator_=jtc->block_for_asking_->createIterator(); initContext(jtc); state_.child_right->open(partition_offset); return true; }
static void test_index_filter_performance(int value_high) { unsigned long long int start = curtick(); vector<IndexScanIterator::query_range> q_range; q_range.clear(); int value_low = 10107; // int value_high = 600257; TableDescriptor* table = Catalog::getInstance()->getTable("cj"); IndexScanIterator::query_range q; q.value_low = malloc(sizeof(int)); // newmalloc q.value_low = (void*)(&value_low); q.comp_low = GEQ; q.value_high = malloc(sizeof(int)); // newmalloc q.value_high = (void*)(&value_high); q.comp_high = L; q.c_type.type = t_int; q.c_type.operate = new OperateInt(); q_range.push_back(q); LogicalOperator* index_scan = new LogicalIndexScan(table->getProjectoin(0)->getProjectionID(), table->getAttribute("sec_code"), q_range); const NodeID collector_node_id = 0; LogicalOperator* root = new LogicalQueryPlanRoot( collector_node_id, index_scan, LogicalQueryPlanRoot::PERFORMANCE); // root->print(); PerformanceMonitor* executable_query_plan = (PerformanceMonitor*)root->GetPhysicalPlan(1024 * 64); executable_query_plan->Open(); while (executable_query_plan->Next(0)) ; executable_query_plan->Close(); // ResultSet* result_set = executable_query_plan->getResultSet(); const unsigned long int number_of_tuples = executable_query_plan->GetNumberOfTuples(); delete executable_query_plan; root->~LogicalOperator(); // cout << "Sec_code: " << value_low << "\t Result: " << number_of_tuples //<< endl; printf("execution time: %4.4f seconds.\n", getSecond(start)); if (!print_test_name_result(number_of_tuples == 26820, "Index Scan")) { printf("\tIndex Scan sec_code = %d, Expected:%d actual: %d\n", value_low, 26820, number_of_tuples); } }
int mainasdfaf234(int argc,const char** argv){ std::vector<column_type> column_list,column_list_; column_list.push_back(column_type(t_int)); Schema* input=new SchemaFix(column_list); BlockStreamSingleColumnScan::State bsscs_state("/home/claims/temp/Uniform_0_99.column",input); PhysicalOperatorBase* bsscs1=new BlockStreamSingleColumnScan(bsscs_state); PhysicalOperatorBase* bsscs2=new BlockStreamSingleColumnScan(bsscs_state); int f=atoi(argv[2]); AttributeComparator fA(column_type(t_int),Comparator::L,0,&f); std::vector<AttributeComparator> ComparatorList; ComparatorList.push_back(fA); BlockStreamFilter::State bsf_state(input,bsscs1,ComparatorList,4096); PhysicalOperatorBase* bsf=new BlockStreamFilter(bsf_state); // BlockStreamBase *block=new BlockStreamFix(4096,4); int choice=0; while(choice==0){ // bsf->open(); bsf->Open(); unsigned long long int start=curtick(); while(bsf->Next(block)){ block->setEmpty(); } printf("Time=%f Throughput=%f.\n",getSecond(start),1024/getSecond(start)); bsf->Close(); printf("Continue(0) or Not(1) ?\n"); scanf("%d",&choice); } }
bool PhysicalProjectionScan::Open(SegmentExecStatus* const exec_status, const PartitionOffset& kPartitionOffset) { RETURN_IF_CANCELLED(exec_status); RegisterExpandedThreadToAllBarriers(); if (TryEntryIntoSerializedSection()) { /* this is the first expanded thread*/ PartitionStorage* partition_handle_; if (NULL == (partition_handle_ = BlockManager::getInstance()->GetPartitionHandle( PartitionID(state_.projection_id_, kPartitionOffset)))) { LOG(ERROR) << PartitionID(state_.projection_id_, kPartitionOffset) .getName() .c_str() << CStrError(rNoPartitionIdScan) << std::endl; SetReturnStatus(false); } else { partition_reader_iterator_ = partition_handle_->CreateAtomicReaderIterator(); SetReturnStatus(true); } #ifdef AVOID_CONTENTION_IN_SCAN unsigned long long start = curtick(); ChunkReaderIterator* chunk_reader_it; ChunkReaderIterator::block_accessor* ba; while (chunk_reader_it = partition_reader_iterator_->NextChunk()) { while (chunk_reader_it->GetNextBlockAccessor(ba)) { ba->GetBlockSize(); input_dataset_.input_data_blocks_.push_back(ba); } } #endif ExpanderTracker::getInstance()->addNewStageEndpoint( pthread_self(), LocalStageEndPoint(stage_src, "Scan", 0)); perf_info_ = ExpanderTracker::getInstance()->getPerformanceInfo(pthread_self()); perf_info_->initialize(); } BarrierArrive(); return GetReturnStatus(); }
bool ResultCollector::Open(const PartitionOffset& part_offset) { state_.partition_offset_ = part_offset; if (sema_open_.try_wait()) { block_buffer_ = new ResultSet(); block_buffer_iterator_ = block_buffer_->createIterator(); open_finished_ = true; } else { while (!open_finished_) { usleep(1); } } registered_thread_count_++; if (true == g_thread_pool_used) { Environment::getInstance()->getThreadPool()->add_task(CollectResult, this); } else { pthread_t tid; pthread_create(&tid, NULL, CollectResult, this); } unsigned long long int start = curtick(); sema_input_complete_.wait(); block_buffer_->query_time_ = getSecond(start); return true; }
bool ExpandableBlockStreamExchangeEpoll::open(const PartitionOffset& partition_offset) { unsigned long long int start = curtick(); RegisterExpandedThreadToAllBarriers(); if (tryEntryIntoSerializedSection()) { debug_winner_thread++; nexhausted_lowers=0; this->partition_offset=partition_offset; nlowers=state.lower_id_list_.size(); for (unsigned i = 0; i < nlowers; i++) { debug_received_block[i] = 0; } socket_fd_lower_list = new int[nlowers]; //init -1 ---Yu for (int i = 0; i < nlowers; ++i) { socket_fd_lower_list[i] = -1; } buffer=new BlockStreamBuffer(state.block_size_,BUFFER_SIZE_IN_EXCHANGE,state.schema_); ExpanderTracker::getInstance()->addNewStageEndpoint(pthread_self(),LocalStageEndPoint(stage_src,"Exchange",buffer)); received_block_stream_=BlockStreamBase::createBlock(state.schema_,state.block_size_); block_for_socket_ = new BlockContainer*[nlowers]; for (unsigned i = 0; i < nlowers; i++) { block_for_socket_[i] = new BlockContainer(received_block_stream_->getSerializedBlockSize()); } if (PrepareTheSocket() == false) return false; if (SetSocketNonBlocking(sock_fd) == false) { return false; } logging_->log("[%ld,%d] Open: nexhausted lowers=%d, nlower=%d", state.exchange_id_, partition_offset, nexhausted_lowers, nlowers); if (RegisterExchange() == false) { logging_->elog("Register Exchange with ID=%d fails!", state.exchange_id_); } if(isMaster()){ /* According to a bug reported by dsc, the master exchangeupper should check whether other * uppers have registered to exchangeTracker. Otherwise, the lower may fail to connect to the * exchangeTracker of some uppers when the lower nodes receive the exchagnelower, as some uppers * have not register the exchange_id to the exchangeTracker. */ logging_->log("[%ld,%d] Synchronizing....", state.exchange_id_, partition_offset); checkOtherUpperRegistered(); logging_->log("[%ld,%d] Synchronized!", state.exchange_id_, partition_offset); logging_->log("[%ld,%d] This exchange is the master one, serialize the iterator subtree to the children...", state.exchange_id_, partition_offset); if (SerializeAndSendToMulti() == false) return false; } if (CreateReceiverThread() == false) { return false; } createPerformanceInfo(); } /* A synchronization barrier, in case of multiple expanded threads*/ barrierArrive(); return true; }
// receive each one block from all sender void* ExpandableBlockStreamExchangeEpoll::receiver(void* arg){ ExpandableBlockStreamExchangeEpoll* Pthis=(ExpandableBlockStreamExchangeEpoll*)arg; struct epoll_event event; struct epoll_event *events; int status; /** create epoll **/ Pthis->epoll_fd_ = epoll_create1(0); if (Pthis->epoll_fd_ == -1) { Pthis->logging_->elog("epoll create error!\n"); return 0; } event.data.fd = Pthis->sock_fd; event.events = EPOLLIN | EPOLLET; status = epoll_ctl(Pthis->epoll_fd_, EPOLL_CTL_ADD, Pthis->sock_fd, &event); if (status == -1) { Pthis->logging_->elog("epoll ctl error!\n"); return 0; } events=(epoll_event*)calloc(Pthis->nlowers,sizeof(epoll_event)); int fd_cur=0; ticks start=curtick(); std::vector<int> finish_times;//in ms while(true){ usleep(1); const int event_count = epoll_wait(Pthis->epoll_fd_, events, Pthis->nlowers, -1); for (int i = 0; i < event_count; i++) { if ((events[i].events & EPOLLERR) || (events[i].events & EPOLLHUP) || (!(events[i].events & EPOLLIN))) { if (errno == EINTR) { continue; } Pthis->logging_->elog("[%ld] epoll error,reason:%s\n", Pthis->state.exchange_id_, strerror(errno)); FileClose(events[i].data.fd); std::cout << "in " << __FILE__ << ":" << __LINE__; printf("-----for debug:close fd %d.\n", events[i].data.fd); continue; } else if (Pthis->sock_fd == events[i].data.fd) { /* We have a notification on the listening socket, which means one or more incoming connections.*/ while (true) { sockaddr in_addr; socklen_t in_len; int infd; char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV]; in_len = sizeof in_addr; infd = accept(Pthis->sock_fd, &in_addr, &in_len); if (infd == -1) { if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) { /* all the incoming connections are processed.*/ break; } else { Pthis->logging_->elog("accept error! "); break; } } status=getnameinfo(&in_addr,in_len,hbuf,sizeof(hbuf),sbuf,sizeof(sbuf),NI_NUMERICHOST|NI_NUMERICSERV); if(status==0){ Pthis->logging_->log("[%ld] Accepted connection on descriptor %d (host=%s, port=%s),id=%d\n",Pthis->state.exchange_id_, infd, hbuf, sbuf,Pthis->state.exchange_id_); Pthis->lower_ip_array.push_back(hbuf); Pthis->lower_sock_fd_to_index[infd]=Pthis->lower_ip_array.size()-1; assert(Pthis->lower_ip_array.size()<=Pthis->state.lower_id_list_.size()); } /*Make the incoming socket non-blocking and add it to the list of fds to monitor.*/ if (!Pthis->SetSocketNonBlocking(infd)) { return 0; } event.data.fd = infd; event.events = EPOLLIN | EPOLLET; status = epoll_ctl(Pthis->epoll_fd_, EPOLL_CTL_ADD, infd, &event); if (status == -1) { Pthis->logging_->elog("epoll_ctl"); return 0; } } continue; } else { /* We have data on the fd waiting to be read.*/ int done = 0; while (true) { int byte_received; int socket_fd_index=Pthis->lower_sock_fd_to_index[events[i].data.fd]; byte_received=read(events[i].data.fd, (char*)Pthis->block_for_socket_[socket_fd_index]->getBlock()+Pthis->block_for_socket_[socket_fd_index]->GetCurSize(), Pthis->block_for_socket_[socket_fd_index]->GetRestSize()); if(byte_received==-1){ if(errno==EAGAIN){ /*We have read all the data,so go back to the loop.*/ break; } Pthis->logging_->elog("read error!\n"); done = 1; } else if (byte_received == 0) { /* End of file. The remote has closed the connection.*/ done = 1; break; } /* The data is successfully read.*/ Pthis->block_for_socket_[socket_fd_index]->IncreaseActualSize(byte_received); if (Pthis->block_for_socket_[socket_fd_index]->GetRestSize() > 0) { /** the current block is not read entirely from the sender, so continue the loop to read.**/ continue; } /** a block is completely read. **/ Pthis->logging_->log("[%ld] The %d-th block is received from Lower[%s]", Pthis->state.exchange_id_, Pthis->debug_received_block[socket_fd_index], Pthis->lower_ip_array[socket_fd_index].c_str()); Pthis->debug_received_block[socket_fd_index]++; /** deserialize the data block from sender to the blockstreambase (received_block_stream_) **/ Pthis->received_block_stream_->deserialize((Block*) Pthis->block_for_socket_[socket_fd_index]); /** mark block_for_socket_[socket_fd_index] to be empty so that it can accommodate the subsequent data **/ Pthis->block_for_socket_[socket_fd_index]->reset(); /** In the current implementation, a empty block stream means End-Of-File**/ const bool eof=Pthis->received_block_stream_->Empty(); if(!eof){ /** the newly obtained data block is validate, so we insert it into the buffer and post * sem_new_block_or_eof_ so that all the threads waiting for the semaphore continue. **/ Pthis->buffer->insertBlock(Pthis->received_block_stream_); //??? why is all ,not 1 // multiple threads will still compete with lock Pthis->sem_new_block_or_eof_.post(Pthis->number_of_registered_expanded_threads_); } else { /** The newly obtained data block is the end-of-file. **/ Pthis->logging_->log("[%ld] *****This block is the last one.", Pthis->state.exchange_id_); finish_times.push_back((int)getMilliSecond(start)); /** update the exhausted senders count and post sem_new_block_or_eof_ so that all the * threads waiting for the semaphore continue. **/ Pthis->nexhausted_lowers++; Pthis->sem_new_block_or_eof_.post(Pthis->number_of_registered_expanded_threads_); if (Pthis->nexhausted_lowers == Pthis->nlowers) { /* * When all the exchange lowers are exhausted, notify the buffer * that the input data is completely received. */ Pthis->buffer->setInputComplete(); /* print the finish times */ for(unsigned i=0;i<finish_times.size();i++){ printf("%d\t",finish_times[i]); } printf("\t Var:%5.4f\n",get_stddev(finish_times)); } Pthis->logging_->log( "[%ld] <<<<<<<<<<<<<<<<nexhausted_lowers=%d>>>>>>>>>>>>>>>>exchange=(%d,%d)", Pthis->state.exchange_id_, Pthis->nexhausted_lowers, Pthis->state.exchange_id_, Pthis->partition_offset); /** tell the sender that all the block are consumed so that the sender can close the socket**/ Pthis->SendBlockAllConsumedNotification(events[i].data.fd); Pthis->logging_->log("[%ld] This notification (all the blocks in the socket buffer are consumed) is send to the lower[%s] exchange=(%d,%d).\n", Pthis->state.exchange_id_, Pthis->lower_ip_array[socket_fd_index].c_str(), Pthis->state.exchange_id_, Pthis->partition_offset); } } if (done) { Pthis->logging_->log("[%ld] Closed connection on descriptor %d[%s]\n", Pthis->state.exchange_id_, events[i].data.fd, Pthis->lower_ip_array[Pthis->lower_sock_fd_to_index[events[i].data.fd]].c_str()); /* Closing the descriptor will make epoll remove it from the set of descriptors which are monitored. */ FileClose(events[i].data.fd); } } } } }
/** * build a hash table first, which stores the tuple needed to be deleted in a *hash manner and accelerate the probe phase * */ bool PhysicalDeleteFilter::Open(SegmentExecStatus* const exec_status, const PartitionOffset& partition_offset) { #ifdef TIME startTimer(&timer); #endif RETURN_IF_CANCELLED(exec_status); RegisterExpandedThreadToAllBarriers(); int ret = rSuccess; int64_t timer; bool winning_thread = false; if (TryEntryIntoSerializedSection(0)) { winning_thread = true; ExpanderTracker::getInstance()->addNewStageEndpoint( pthread_self(), LocalStageEndPoint(stage_desc, "delete filter build", 0)); unsigned output_index = 0; for (unsigned i = 0; i < state_.filter_key_deleted_.size(); i++) { joinIndex_table_to_output_[i] = output_index; output_index++; } for (unsigned i = 0; i < state_.payload_base_.size(); i++) { payload_table_to_output_[i] = output_index; output_index++; } // start to create the hash table, including the used hash function, hash // table structure hash_ = PartitionFunctionFactory::createBoostHashFunction( state_.hashtable_bucket_num_); int64_t hash_table_build = curtick(); hashtable_ = new BasicHashTable( state_.hashtable_bucket_num_, state_.hashtable_bucket_size_, state_.input_schema_left_->getTupleMaxSize()); if (NULL == hashtable_) { return ret = rMemoryAllocationFailed; LOG(ERROR) << "hashtable allocation failed" << "[" << rMemoryAllocationFailed << "]" << endl; } #ifdef _DEBUG_ consumed_tuples_from_left = 0; #endif // start to create the join expression, based on which it is able to the // probe the deleted tuples // QNode* expr = createEqualJoinExpression( // state_.hashtable_schema_, state_.input_schema_right_, // state_.filter_key_deleted_, state_.filter_key_base_); // if (NULL == expr) { // ret = rSuccess; // LOG(ERROR) << "The generation of the enqual join expression for // delete " // "filter is failed" << endl; // } // ticks start = curtick(); // // // start to generate the dedicated function, based on which the probe // is // // eventually acted, including using llvm and the function pointer // if (Config::enable_codegen) { // eftt_ = getExprFuncTwoTuples(expr, state_.hashtable_schema_, // state_.input_schema_right_); // memcpy_ = getMemcpy(state_.hashtable_schema_->getTupleMaxSize()); // memcat_ = getMemcat(state_.hashtable_schema_->getTupleMaxSize(), // state_.input_schema_right_->getTupleMaxSize()); // } // if (eftt_) { // cff_ = PhysicalDeleteFilter::isMatchCodegen; // printf("Codegen(delete filter) succeed(%4.3fms)!\n", // getMilliSecond(start)); // } else { cff_ = PhysicalDeleteFilter::isMatch; // printf("Codegen(delete filter) failed!\n"); // } // delete expr; } /** * For performance concern, the following line should place just after * "RegisterNewThreadToAllBarriers();" * in order to accelerate the open response time. */ LOG(INFO) << "delete filter operator begin to open left child" << endl; state_.child_left_->Open(exec_status, partition_offset); LOG(INFO) << "delete filter operator finished opening left child" << endl; BarrierArrive(0); BasicHashTable::Iterator tmp_it = hashtable_->CreateIterator(); void* cur; void* tuple_in_hashtable; unsigned bn; void* key_in_input; void* key_in_hashtable; void* value_in_input; void* value_in_hashtable; // create the context for the multi-thread to build the hash table DeleteFilterThreadContext* dftc = CreateOrReuseContext(crm_numa_sensitive); const Schema* input_schema = state_.input_schema_left_->duplicateSchema(); // we used the filter_key_deleted_[0] here, because the data is partitioned // based on the first column in the join index const Operate* op = input_schema->getcolumn(state_.filter_key_deleted_[0]) .operate->duplicateOperator(); const unsigned buckets = state_.hashtable_bucket_num_; int64_t start = curtick(); int64_t processed_tuple_count = 0; LOG(INFO) << "delete filter operator begin to call left child's next()" << endl; RETURN_IF_CANCELLED(exec_status); while (state_.child_left_->Next(exec_status, dftc->l_block_for_asking_)) { RETURN_IF_CANCELLED(exec_status); delete dftc->l_block_stream_iterator_; dftc->l_block_stream_iterator_ = dftc->l_block_for_asking_->createIterator(); while (cur = dftc->l_block_stream_iterator_->nextTuple()) { #ifdef _DEBUG_ processed_tuple_count++; lock_.acquire(); consumed_tuples_from_left++; lock_.release(); #endif const void* key_addr = input_schema->getColumnAddess(state_.filter_key_deleted_[0], cur); bn = op->getPartitionValue(key_addr, buckets); tuple_in_hashtable = hashtable_->atomicAllocate(bn); if (memcpy_) memcpy_(tuple_in_hashtable, cur); else input_schema->copyTuple(cur, tuple_in_hashtable); } dftc->l_block_for_asking_->setEmpty(); } // printf("%d cycles per // tuple!\n",(curtick()-start)/processed_tuple_count); unsigned tmp = 0; #ifdef _DEBUG_ tuples_in_hashtable = 0; produced_tuples = 0; consumed_tuples_from_right = 0; #endif if (ExpanderTracker::getInstance()->isExpandedThreadCallBack( pthread_self())) { UnregisterExpandedThreadToAllBarriers(1); // printf("<<<<<<<<<<<<<<<<<Join open detected call back // signal!>>>>>>>>>>>>>>>>>\n"); return true; } BarrierArrive(1); // if(winning_thread){ //// hashtable->report_status(); //// printf("Hash Table Build time: %4.4f\n",getMilliSecond(timer)); // } // hashtable->report_status(); // printf("join open consume %d tuples\n",consumed_tuples_from_left); RETURN_IF_CANCELLED(exec_status); state_.child_right_->Open(exec_status, partition_offset); RETURN_IF_CANCELLED(exec_status); LOG(INFO) << "delete filter operator finished opening right child" << endl; return true; }
//int main_join_success(int argc, const char** argv){ int main_join_success22(int argc, const char** argv){ int master; printf("Master(0) or Slave(others)?\n"); scanf("%d",&master); if(master==0){ Environment::getInstance(true); // const unsigned block_size=atoi(argv[1]); // const unsigned thread_count=atoi(argv[2]); // const unsigned expander_buffer=atoi(argv[3]); // std::vector<std::string> upper_ip_list; // const unsigned lowers_1=atoi(argv[4]); // const unsigned lowers_2=atoi(argv[5]); const unsigned block_size=4096; const unsigned thread_count=3; const unsigned expander_buffer=3; std::vector<std::string> upper_ip_list; const unsigned lowers_1=3; const unsigned lowers_2=3; upper_ip_list.push_back("10.11.1.208"); std::vector<std::string> lower_ip_list; // lower_ip_list.push_back("10.11.1.201"); lower_ip_list.push_back("10.11.1.202"); lower_ip_list.push_back("10.11.1.203"); lower_ip_list.push_back("10.11.1.204"); lower_ip_list.push_back("10.11.1.205"); lower_ip_list.push_back("10.11.1.206"); lower_ip_list.push_back("10.11.1.207"); // lower_ip_list.push_back("10.11.1.208"); // lower_ip_list.push_back("10.11.1.209"); // lower_ip_list.push_back("10.11.1.210"); // lower_ip_list.push_back("10.11.1.211"); // lower_ip_list.push_back("10.11.1.214"); // std::vector<std::string> used_lowers; // for(unsigned i=0;i<lowers;i++){ // used_lowers.push_back(lower_ip_list[i]); // } std::vector<std::string> used_lowers_1; for(unsigned i=0;i<lowers_1;i++){ used_lowers_1.push_back(lower_ip_list[i]); } std::vector<std::string> used_lowers_2; for(unsigned i=lower_ip_list.size()-1;i>=lower_ip_list.size()-lowers_2;i--){ used_lowers_2.push_back(lower_ip_list[i]); } std::vector<column_type> column_list,column_list_; column_list.push_back(column_type(t_int)); Schema* schema=new SchemaFix(column_list); // ExpandableBlockStreamSingleColumnScan::State ebssc_state("/home/imdb/temp/Uniform_0_99.column",schema,block_size); // BlockStreamIteratorBase* ebssc=new ExpandableBlockStreamSingleColumnScan(ebssc_state); ExpandableBlockStreamSingleColumnScan::State ebssc_state1("/home/imdb/temp/zhanglei/join_2",schema,block_size); BlockStreamIteratorBase* ebssc1=new ExpandableBlockStreamSingleColumnScan(ebssc_state1); // ExpandableBlockStreamSingleColumnScan::State ebssc_state2("/home/imdb/temp/zhanglei/Uniform_0_99.column_10000",schema,block_size); // BlockStreamIteratorBase* ebssc2=new ExpandableBlockStreamSingleColumnScan(ebssc_state2); ExpandableBlockStreamSingleColumnScan::State ebssc_state3("/home/imdb/temp/zhanglei/join_2_cp",schema,block_size); BlockStreamIteratorBase* ebssc3=new ExpandableBlockStreamSingleColumnScan(ebssc_state3); // ExpandableBlockStreamSingleColumnScan::State ebssc_state4("/home/imdb/temp/zhanglei/Uniform_0_99.column_10000_copy",schema,block_size); // BlockStreamIteratorBase* ebssc4=new ExpandableBlockStreamSingleColumnScan(ebssc_state4); // FilterIterator::AttributeComparator filter1(column_type(t_int),Comparator::L,0,&f); // std::vector<FilterIterator::AttributeComparator> ComparatorList; // ComparatorList.push_back(filter1); // ExpandableBlockStreamFilter::State ebsf_state(schema,ebssc,ComparatorList,block_size); // BlockStreamIteratorBase* bbsf=new ExpandableBlockStreamFilter(ebsf_state); std::vector<Schema *> inputs; inputs.push_back(schema); inputs.push_back(schema); column_list_.push_back(column_type(t_int)); column_list_.push_back(column_type(t_int)); Schema* output=new SchemaFix(column_list_); std::vector<BlockStreamIteratorBase *> children_; children_.push_back(ebssc1); children_.push_back(ebssc3); ////////////////////////////////////////combined\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ BlockStreamCombinedIterator::State bsci_state1(inputs,output,children_); BlockStreamCombinedIterator *bsc1=new BlockStreamCombinedIterator(bsci_state1); BlockStreamExpander::State bse_state(output,bsc1,thread_count,block_size,expander_buffer); BlockStreamIteratorBase* bse=new BlockStreamExpander(bse_state); const int exchange_id=1; ExpandableBlockStreamBroadcastExchange::State ebse_state(output,bse,block_size,used_lowers_2,used_lowers_1,exchange_id); BlockStreamIteratorBase* ebse=new ExpandableBlockStreamBroadcastExchange(ebse_state); BlockStreamCombinedIterator::State bsci_state2(inputs,output,children_); BlockStreamCombinedIterator *bsc2=new BlockStreamCombinedIterator(bsci_state2); std::vector<column_type> column_list_join; column_list_join.push_back(column_type(t_int)); column_list_join.push_back(column_type(t_int)); column_list_join.push_back(column_type(t_int)); Schema* output_join=new SchemaFix(column_list_join); std::vector<unsigned> joinIndex_left; joinIndex_left.push_back(0); std::vector<unsigned> joinIndex_right; joinIndex_right.push_back(1); std::vector<unsigned> payload_left; payload_left.push_back(1); std::vector<unsigned> payload_right; payload_right.push_back(0); BlockStreamJoinIterator::State bsji_state;//(ebse,bsc2,output,output,output_join,joinIndex_left,joinIndex_right,payload_left,payload_right,100,1024,4096); BlockStreamJoinIterator* bsji=new BlockStreamJoinIterator(bsji_state); BlockStreamExpander::State bse_state_(output_join,bsji,thread_count,block_size,expander_buffer); BlockStreamIteratorBase* bse_=new BlockStreamExpander(bse_state_); const int exchange_id_1=0; ExpandableBlockStreamBroadcastExchange::State ebse_state_(output,bse_,block_size,used_lowers_1,upper_ip_list,exchange_id_1); BlockStreamIteratorBase* ebse_=new ExpandableBlockStreamBroadcastExchange(ebse_state_); BlockStreamExpander::State bse_state_1(output_join,ebse_,thread_count,block_size,expander_buffer); BlockStreamIteratorBase* bse_1=new BlockStreamExpander(bse_state_1); BlockStreamPerformanceMonitorTop::State bspfm_state(output_join,bse_1,block_size,1000); BlockStreamIteratorBase* bspfm=new BlockStreamPerformanceMonitorTop(bspfm_state); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// BlockStreamBase *block=new BlockStreamFix(block_size,12); volatile int choice; printf("Continue(1) or Not(0) ?\n"); scanf("%d",&choice); unsigned tuple_count=0; while(choice==1){ // bsf->open(); ebse_->open(); unsigned long long int start=curtick(); // tuple_count=0; while(ebse_->next(block)){ BlockStreamBase::BlockStreamTraverseIterator *it=block->createIterator(); void* tuple; while(tuple=it->nextTuple()){ printf("tuple:%d \n",*(int*)tuple); tuple_count++; } block->setEmpty(); } printf("Total tupls:%d\n",tuple_count); printf("Time=%f Throughput=%f.\n",getSecond(start),1024/getSecond(start)); ebse_->close(); printf("Continue(0) or Not(1) ?\n"); // getchar(); scanf("%d",&choice); printf("you input %d\n",choice); } } else{ Environment::getInstance(false); } printf("Waiting~~~~~....\n"); while(true){ sleep(1); } }
/** * first, block_for_socket_ for receive data from senders, then if one block is * enough, next serialize it and put it into all_merged_block_buffer. * epoll is good at listening every coming block for different socket. * */ void* ExchangeMerger::Receiver(void* arg) { ExchangeMerger* Pthis = reinterpret_cast<ExchangeMerger*>(arg); struct epoll_event event; struct epoll_event* events; int status; // create epoll Pthis->epoll_fd_ = epoll_create1(0); if (Pthis->epoll_fd_ == -1) { LOG(ERROR) << " exchange_id = " << Pthis->state_.exchange_id_ << " partition_offset = " << Pthis->partition_offset_ << " merger fail to create epoll!" << std::endl; return NULL; } event.data.fd = Pthis->sock_fd_; event.events = EPOLLIN | EPOLLET; status = epoll_ctl(Pthis->epoll_fd_, EPOLL_CTL_ADD, Pthis->sock_fd_, &event); if (-1 == status) { LOG(ERROR) << " exchange_id = " << Pthis->state_.exchange_id_ << " partition_offset = " << Pthis->partition_offset_ << " merger fail to create epoll_ctl!" << std::endl; return NULL; } events = reinterpret_cast<epoll_event*>( calloc(Pthis->lower_num_, sizeof(epoll_event))); int fd_cur = 0; ticks start = curtick(); std::vector<int> finish_times; // in ms while (true) { usleep(1); const int event_count = epoll_wait(Pthis->epoll_fd_, events, Pthis->lower_num_, -1); for (int i = 0; i < event_count; i++) { if ((events[i].events & EPOLLERR) || (events[i].events & EPOLLHUP) || (!(events[i].events & EPOLLIN))) { if (errno == EINTR) { continue; } LOG(WARNING) << " exchange_id = " << Pthis->state_.exchange_id_ << " partition_offset = " << Pthis->partition_offset_ << " epoll error,reason: " << strerror(errno) << " close fd = " << events[i].data.fd << std::endl; FileClose(events[i].data.fd); continue; } else if (Pthis->sock_fd_ == events[i].data.fd) { /* We have a notification on the listening socket, which means one or * more incoming connections. */ while (true) { sockaddr in_addr; socklen_t in_len; int infd; char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV]; in_len = sizeof in_addr; infd = accept(Pthis->sock_fd_, &in_addr, &in_len); if (infd == -1) { if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) { /* all the incoming connections are processed.*/ break; } else { LOG(WARNING) << " exchange_id = " << Pthis->state_.exchange_id_ << " partition_offset = " << Pthis->partition_offset_ << " epoll accept error, try again!" << std::endl; break; } } status = getnameinfo(&in_addr, in_len, hbuf, sizeof(hbuf), sbuf, sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV); if (0 == status) { LOG(INFO) << " exchange_id = " << Pthis->state_.exchange_id_ << " partition_offset = " << Pthis->partition_offset_ << " Accepted connection on descriptor " << infd << " host= " << hbuf << " port= " << sbuf << std::endl; Pthis->lower_ip_list_.push_back(hbuf); Pthis->lower_sock_fd_to_id_[infd] = Pthis->lower_ip_list_.size() - 1; assert(Pthis->lower_ip_list_.size() <= Pthis->state_.lower_id_list_.size()); } /*Make the incoming socket non-blocking and add it to the list of fds * to monitor.*/ if (!Pthis->SetSocketNonBlocking(infd)) { return 0; } event.data.fd = infd; event.events = EPOLLIN | EPOLLET; status = epoll_ctl(Pthis->epoll_fd_, EPOLL_CTL_ADD, infd, &event); if (-1 == status) { LOG(ERROR) << " exchange_id = " << Pthis->state_.exchange_id_ << " partition_offset = " << Pthis->partition_offset_ << " epoll_ctl error2" << std::endl; return NULL; } } continue; } else { /* We have data on the fd waiting to be read.*/ int done = 0; while (true) { int byte_received; int socket_fd_index = Pthis->lower_sock_fd_to_id_[events[i].data.fd]; byte_received = read(events[i].data.fd, (reinterpret_cast<char*>( Pthis->block_for_socket_[socket_fd_index]->getBlock())) + Pthis->block_for_socket_[socket_fd_index]->GetCurSize(), Pthis->block_for_socket_[socket_fd_index]->GetRestSize()); if (byte_received == -1) { if (errno == EAGAIN) { /*We have read all the data,so go back to the loop.*/ break; } LOG(WARNING) << " exchange_id = " << Pthis->state_.exchange_id_ << " partition_offset = " << Pthis->partition_offset_ << " merger read error!" << std::endl; done = 1; } else if (byte_received == 0) { /* End of file. The remote has closed the connection.*/ done = 1; break; } /* The data is successfully read.*/ Pthis->block_for_socket_[socket_fd_index]->IncreaseActualSize( byte_received); if (Pthis->block_for_socket_[socket_fd_index]->GetRestSize() > 0) { /** the current block is not read entirely from the Sender, so * continue the loop to read.**/ continue; } /** a block is completely read. **/ /** deserialize the data block from Sender to the blockstreambase * (block_for_deserialization) **/ Pthis->block_for_deserialization->deserialize( reinterpret_cast<Block*>( Pthis->block_for_socket_[socket_fd_index])); /** mark block_for_socket_[socket_fd_index] to be empty so that it can * accommodate the subsequent data **/ Pthis->block_for_socket_[socket_fd_index]->reset(); /** * In the current implementation, a empty block stream means * End-Of-File */ const bool eof = Pthis->block_for_deserialization->Empty(); if (!eof) { /** the newly obtained data block is validate, so we insert it into * the all_merged_block_buffer_ and post sem_new_block_or_eof_ so * that all the threads waiting for the semaphore continue. */ Pthis->all_merged_block_buffer_->insertBlock( Pthis->block_for_deserialization); //??? why is all ,not 1 // multiple threads will still compete with lock Pthis->sem_new_block_or_eof_.post( Pthis->number_of_registered_expanded_threads_); } else { /** The newly obtained data block is the end-of-file. **/ LOG(INFO) << " exchange_id = " << Pthis->state_.exchange_id_ << " partition_offset = " << Pthis->partition_offset_ << " This block is the last one." << std::endl; finish_times.push_back(static_cast<int>(getMilliSecond(start))); /** update the exhausted senders count and post *sem_new_block_or_eof_ so that all the * threads waiting for the semaphore continue. **/ Pthis->exhausted_lowers++; Pthis->sem_new_block_or_eof_.post( Pthis->number_of_registered_expanded_threads_); if (Pthis->exhausted_lowers == Pthis->lower_num_) { /* * When all the exchange lowers are exhausted, notify the * all_merged_block_buffer_ * that the input data is completely received. */ Pthis->all_merged_block_buffer_->setInputComplete(); /* print the finish times */ // for (unsigned i = 0; i < finish_times.size(); i++) // { // printf("%d\t", finish_times[i]); // } // printf("\t Var:%5.4f\n", get_stddev(finish_times)); } LOG(INFO) << " exchange_id = " << Pthis->state_.exchange_id_ << " partition_offset = " << Pthis->partition_offset_ << " exhausted lowers = " << Pthis->exhausted_lowers << " senders have exhausted" << std::endl; /** tell the Sender that all the block are consumed so that the * Sender can close the socket**/ Pthis->ReplyAllBlocksConsumed(events[i].data.fd); LOG(INFO) << " exchange_id = " << Pthis->state_.exchange_id_ << " partition_offset = " << Pthis->partition_offset_ << " This notification (all the blocks in the socket buffer " "are consumed) is replied to the lower " << Pthis->lower_ip_list_[socket_fd_index].c_str() << std::endl; } } if (done) { LOG(INFO) << " exchange_id = " << Pthis->state_.exchange_id_ << " partition_offset = " << Pthis->partition_offset_ << " Closed connection on descriptor " << events[i].data.fd << " " << Pthis->lower_ip_list_ [Pthis->lower_sock_fd_to_id_[events[i].data.fd]].c_str() << std::endl; /* Closing the descriptor will make epoll remove it from the set of descriptors which are monitored. */ FileClose(events[i].data.fd); } } } } }
/** * note the serialized block's size is different from others, it has tail info. * exchange merger is at the end of one segment of plan, so it's the "stage_src" * for this stage */ bool ExchangeMerger::Open(const PartitionOffset& partition_offset) { unsigned long long int start = curtick(); RegisterExpandedThreadToAllBarriers(); if (TryEntryIntoSerializedSection()) { // first arrived thread dose exhausted_lowers = 0; this->partition_offset_ = partition_offset; lower_num_ = state_.lower_id_list_.size(); socket_fd_lower_list_ = new int[lower_num_]; for (int i = 0; i < lower_num_; ++i) { socket_fd_lower_list_[i] = -1; } // buffer all deserialized blocks come from every socket all_merged_block_buffer_ = new BlockStreamBuffer( state_.block_size_, BUFFER_SIZE_IN_EXCHANGE, state_.schema_); ExpanderTracker::getInstance()->addNewStageEndpoint( pthread_self(), LocalStageEndPoint(stage_src, "Exchange", all_merged_block_buffer_)); // if one of block_for_socket is full, it will be deserialized into // block_for_deserialization and sended to all_merged_data_buffer block_for_deserialization = BlockStreamBase::createBlock(state_.schema_, state_.block_size_); // store block for each socket and the received block is serialized. block_for_socket_ = new BlockContainer* [lower_num_]; for (unsigned i = 0; i < lower_num_; ++i) { block_for_socket_[i] = new BlockContainer( block_for_deserialization->getSerializedBlockSize()); } if (PrepareSocket() == false) return false; if (SetSocketNonBlocking(sock_fd_) == false) { return false; } LOG(INFO) << "exchange_id = " << state_.exchange_id_ << " partition_offset = " << partition_offset << " Open: exhausted lower senders num = " << exhausted_lowers << " lower sender num = " << lower_num_ << std::endl; if (RegisterExchange() == false) { LOG(ERROR) << "Register Exchange with ID = " << state_.exchange_id_ << " fails!" << std::endl; } if (IsMaster()) { /* According to a bug reported by dsc, the master exchange upper should * check whether other uppers have registered to exchangeTracker. * Otherwise, the lower may fail to connect to the exchangeTracker of some * uppers when the lower nodes receive the exchange lower, as some uppers * have not register the exchange_id to the exchangeTracker. */ LOG(INFO) << " exchange_id = " << state_.exchange_id_ << " partition_offset = " << partition_offset << "Synchronizing...." << std::endl; IsOtherMergersRegistered(); LOG(INFO) << " exchange_id = " << state_.exchange_id_ << " partition_offset = " << partition_offset << " Synchronized! Then serialize and send its next segment " "plan to all its lower senders" << std::endl; if (SerializeAndSendPlan() == false) return false; } if (CreateReceiverThread() == false) { return false; } CreatePerformanceInfo(); } /// A synchronization barrier, in case of multiple expanded threads BarrierArrive(); return true; }
//int main(int argc,const char** argv){ int main_combine(int argc,const char** argv){ const unsigned block_size=BLOCK_SIZE_CAO; const unsigned thread_count=4; const unsigned expander_buffer=4; std::vector<column_type> column_list,column_list_; column_list.push_back(column_type(t_int)); Schema* schema=new SchemaFix(column_list); ExpandableBlockStreamSingleColumnScan::State ebssc_state1("/home/imdb/temp/Uniform_0_99.column",schema,block_size); BlockStreamIteratorBase* ebssc1=new ExpandableBlockStreamSingleColumnScan(ebssc_state1); ExpandableBlockStreamSingleColumnScan::State ebssc_state2("/home/imdb/temp/Uniform_0_99.column",schema,block_size); BlockStreamIteratorBase* ebssc2=new ExpandableBlockStreamSingleColumnScan(ebssc_state2); std::vector<Schema *> inputs; inputs.push_back(schema); inputs.push_back(schema); column_list_.push_back(column_type(t_int)); column_list_.push_back(column_type(t_int)); Schema* output=new SchemaFix(column_list_); std::vector<BlockStreamIteratorBase *> children_; children_.push_back(ebssc1); children_.push_back(ebssc2); BlockStreamCombinedIterator::State bsci_state(inputs,output,children_); BlockStreamCombinedIterator *bsc=new BlockStreamCombinedIterator(bsci_state); BlockStreamExpander::State bse_state(schema,bsc,thread_count,block_size,expander_buffer); BlockStreamIteratorBase* bse=new BlockStreamExpander(bse_state); BlockStreamBase *block=new BlockStreamFix(block_size,8); int choice=0; std::ostringstream ostr; boost::archive::text_oarchive oa(ostr); oa.register_type(static_cast<BlockStreamCombinedIterator *>(NULL)); oa.register_type(static_cast<BlockStreamExpander *>(NULL)); oa.register_type(static_cast<ExpandableBlockStreamSingleColumnScan *>(NULL)); Register_Schemas<boost::archive::text_oarchive>(oa); // Register_Iterators(oa); oa<<bse; std::cout<<"Serialization Result:"<<ostr.str()<<std::endl; std::istringstream istr(ostr.str()); boost::archive::text_iarchive ia(istr); BlockStreamIteratorBase* des; ia.register_type(static_cast<BlockStreamCombinedIterator *>(NULL)); ia.register_type(static_cast<BlockStreamExpander *>(NULL)); ia.register_type(static_cast<ExpandableBlockStreamSingleColumnScan *>(NULL)); Register_Schemas<boost::archive::text_iarchive>(ia); ia>>des; // return 1; while(choice==0){ // bsf->open(); des->open(); cout<<"after open!"<<endl; unsigned long long int start=curtick(); cout<<"ready for the next"<<endl; unsigned tuple_count=0; while(des->next(block)){ BlockStreamBase::BlockStreamTraverseIterator* it=block->createIterator(); while(it->nextTuple()){ tuple_count++; } block->setEmpty(); } printf("Time=%f Throughput=%f.\n tuple=%d",getSecond(start),1024/getSecond(start),tuple_count); des->close(); printf("Continue(0) or Not(1) ?\n"); scanf("%d",&choice); } }
static void query_select_aggregation_ing(){ /* * select sum(a+1)+count(a),b * from T * group by b * * notation: p a p s * */ unsigned long long int start=curtick(); TableDescriptor* table=Environment::getInstance()->getCatalog()->getTable("LINEITEM"); //===========================scan=========================== LogicalOperator* scan=new LogicalScan(table->getProjectoin(0)); //==========================project========================= vector< vector<ExpressionItem> >expr_list1; vector<ExpressionItem> expr0; vector<ExpressionItem> expr1; vector<ExpressionItem> expr2; vector<ExpressionItem> expr3; vector<ExpressionItem> expr4; vector<ExpressionItem> expr5; vector<ExpressionItem> expr6; vector<ExpressionItem> expr7; vector<ExpressionItem> expr8; vector<ExpressionItem> expr9; vector<ExpressionItem> expr10; vector<ExpressionItem> expr11; vector<ExpressionItem> expr12; vector<ExpressionItem> expr13; vector<ExpressionItem> expr14; vector<ExpressionItem> expr15; vector<ExpressionItem> expr16; vector<ExpressionItem> expr17; ExpressionItem ei1; ExpressionItem ei1_1; ExpressionItem ei1_2; ExpressionItem ei1_3; ExpressionItem ei1_4; ExpressionItem ei1_5; ExpressionItem ei1_6; ExpressionItem ei1_7; ExpressionItem ei1_8; ExpressionItem ei1_9; ExpressionItem ei2; ExpressionItem ei3; ExpressionItem ei4; ExpressionItem ei5; ExpressionItem ei6; ExpressionItem ei7; ExpressionItem ei8; ExpressionItem ei9; ExpressionItem ei10; ExpressionItem ei11; ExpressionItem ei12; ExpressionItem ei13; ExpressionItem ei14; ExpressionItem ei15; ExpressionItem ei16; ExpressionItem ei17; ei1_1.setVariable("LINEITEM.L_EXTENDEDPRICE"); ei1_2.setIntValue("1"); ei1_3.setVariable("LINEITEM.L_DISCOUNT"); ei1_4.setOperator("-"); ei1_5.setOperator("*"); ei1_6.setIntValue("1"); ei1_7.setVariable("LINEITEM.L_TEX"); ei1_8.setOperator("+"); ei1_9.setOperator("*"); ei1.setVariable("LINEITEM.row_id"); ei2.setVariable("LINEITEM.L_ORDERKEY"); ei3.setVariable("LINEITEM.L_PARTKEY"); ei4.setVariable("LINEITEM.L_SUPPKEY"); ei5.setVariable("LINEITEM.L_LINENUMBER"); ei6.setVariable("LINEITEM.L_QUANTITY"); ei7.setVariable("LINEITEM.L_EXTENDEDPRICE"); ei8.setVariable("LINEITEM.L_DISCOUNT"); ei9.setVariable("LINEITEM.L_TEX"); ei10.setVariable("LINEITEM.L_RETURNFLAG"); // ei10.size=1; ei11.setVariable("LINEITEM.L_LINESTATUS"); // ei11.size=1; ei13.setVariable("LINEITEM.L_COMMITDATE"); ei14.setVariable("LINEITEM.L_RECEIPTDATE"); ei15.setVariable("LINEITEM.L_SHIPINSTRUCT"); // ei15.size=25; ei16.setVariable("LINEITEM.L_SHIPMODE"); // ei16.size=10; ei17.setVariable("LINEITEM.L_COMMENT"); // ei17.size=44; expr1.push_back(ei1_1); expr1.push_back(ei1_2); expr1.push_back(ei1_3); expr1.push_back(ei1_4); expr1.push_back(ei1_5); // expr1.push_back(ei1_6); // expr1.push_back(ei1_7); // expr1.push_back(ei1_8); // expr1.push_back(ei1_9); // expr1.push_back(ei1); expr2.push_back(ei1_1); expr2.push_back(ei1_2); expr2.push_back(ei1_3); expr2.push_back(ei1_4); expr2.push_back(ei1_5); expr2.push_back(ei1_6); expr2.push_back(ei1_7); expr2.push_back(ei1_8); expr2.push_back(ei1_9); expr3.push_back(ei1_2); expr3.push_back(ei1_3); expr3.push_back(ei1_4); // expr3.push_back(ei3); expr4.push_back(ei4); expr5.push_back(ei5); expr6.push_back(ei6); expr7.push_back(ei7); expr8.push_back(ei8); expr9.push_back(ei9); expr10.push_back(ei10); expr11.push_back(ei11); expr12.push_back(ei12); expr13.push_back(ei13); expr14.push_back(ei14); expr15.push_back(ei15); expr16.push_back(ei16); expr17.push_back(ei17); expr0.push_back(ei1); // expr_list1.push_back(expr0); expr_list1.push_back(expr10); expr_list1.push_back(expr11); expr_list1.push_back(expr6); expr_list1.push_back(expr7); expr_list1.push_back(expr1); expr_list1.push_back(expr2); expr_list1.push_back(expr8); expr_list1.push_back(expr3); // expr_list1.push_back(expr3); // expr_list1.push_back(expr4); // expr_list1.push_back(expr5); // expr_list1.push_back(expr8); // expr_list1.push_back(expr9); // expr_list1.push_back(expr12); // expr_list1.push_back(expr13); // expr_list1.push_back(expr14); // expr_list1.push_back(expr15); // expr_list1.push_back(expr16); // expr_list1.push_back(expr17); LogicalOperator* project1=new LogicalProject(scan,expr_list1); // //========================aggregation======================= // std::vector<Attribute> group_by_attributes; // group_by_attributes.push_back(table->getAttribute("L_RETURNFLAG")); // group_by_attributes.push_back(table->getAttribute("L_LINESTATUS")); // std::vector<Attribute> aggregation_attributes; // aggregation_attributes.push_back(table->getAttribute("L_QUANTITY")); // aggregation_attributes.push_back(table->getAttribute("L_EXTENDEDPRICE")); // aggregation_attributes.push_back(table->getAttribute("L_DISCOUNT")); // aggregation_attributes.push_back(Attribute(ATTRIBUTE_ANY)); // std::vector<BlockStreamAggregationIterator::State::aggregation> aggregation_function; // // aggregation_function.push_back(BlockStreamAggregationIterator::State::sum); // aggregation_function.push_back(BlockStreamAggregationIterator::State::sum); // aggregation_function.push_back(BlockStreamAggregationIterator::State::sum); // aggregation_function.push_back(BlockStreamAggregationIterator::State::count); // LogicalOperator* aggregation=new Aggregation(group_by_attributes,aggregation_attributes,aggregation_function,project1); // // //==========================project========================= // vector< vector<ExpressionItem> >expr_list2; // LogicalOperator* project2=new LogicalProject(aggregation,expr_list2); //===========================root=========================== LogicalOperator* root=new LogicalQueryPlanRoot(0,project1,LogicalQueryPlanRoot::PERFORMANCE); cout<<"performance is ok!"<<endl; BlockStreamIteratorBase* physical_iterator_tree=root->getIteratorTree(64*1024); // physical_iterator_tree->print(); physical_iterator_tree->open(); while(physical_iterator_tree->next(0)); physical_iterator_tree->close(); printf("Q1: execution time: %4.4f second.\n",getSecond(start)); }
static void query_select_fzh() { /* * select sum(a+1)+count(a),b * from T * group by b * * notation: p a p s * */ unsigned long long int start=curtick(); TableDescriptor* table=Environment::getInstance()->getCatalog()->getTable("LINEITEM"); //===========================scan=========================== LogicalOperator* scan=new LogicalScan(table->getProjectoin(0)); //==========================project========================= vector< vector<ExpressionItem> >expr_list1; vector<ExpressionItem> expr1; vector<ExpressionItem> expr2; vector<ExpressionItem> expr3; vector<ExpressionItem> expr4; vector<ExpressionItem> expr5; vector<ExpressionItem> expr6; vector<ExpressionItem> expr7; vector<ExpressionItem> expr8; vector<ExpressionItem> expr9; vector<ExpressionItem> expr10; vector<ExpressionItem> expr11; vector<ExpressionItem> expr12; vector<ExpressionItem> expr13; vector<ExpressionItem> expr14; vector<ExpressionItem> expr15; vector<ExpressionItem> expr16; vector<ExpressionItem> expr17; ExpressionItem ei1; ExpressionItem ei1_1; ExpressionItem ei1_2; ExpressionItem ei1_3; ExpressionItem ei1_4; ExpressionItem ei1_5; ExpressionItem ei1_6; ExpressionItem ei1_7; ExpressionItem ei1_8; ExpressionItem ei1_9; ExpressionItem ei2; ExpressionItem ei3; ExpressionItem ei4; ExpressionItem ei5; ExpressionItem ei6; ExpressionItem ei7; ExpressionItem ei8; ExpressionItem ei9; ExpressionItem ei10; ExpressionItem ei11; ExpressionItem ei12; ExpressionItem ei13; ExpressionItem ei14; ExpressionItem ei15; ExpressionItem ei16; ExpressionItem ei17; ei1_1.setVariable("LINEITEM.row_id"); // ei1_2.setVariable("LINEITEM.L_ORDERKEY"); ei1_2.setIntValue("1"); ei1_3.setOperator("+"); expr1.push_back(ei1_1); expr1.push_back(ei1_2); expr1.push_back(ei1_3); expr_list1.push_back(expr1); LogicalOperator* project1=new LogicalProject(scan,expr_list1); //========================aggregation======================= std::vector<Attribute> group_by_attributes; group_by_attributes.push_back(table->getAttribute("L_RETURNFLAG")); group_by_attributes.push_back(table->getAttribute("L_LINESTATUS")); std::vector<Attribute> aggregation_attributes; aggregation_attributes.push_back(table->getAttribute("L_QUANTITY")); aggregation_attributes.push_back(table->getAttribute("L_EXTENDEDPRICE")); aggregation_attributes.push_back(table->getAttribute("L_DISCOUNT")); aggregation_attributes.push_back(Attribute(ATTRIBUTE_ANY)); std::vector<BlockStreamAggregationIterator::State::aggregation> aggregation_function; aggregation_function.push_back(BlockStreamAggregationIterator::State::sum); aggregation_function.push_back(BlockStreamAggregationIterator::State::sum); aggregation_function.push_back(BlockStreamAggregationIterator::State::sum); aggregation_function.push_back(BlockStreamAggregationIterator::State::count); LogicalOperator* aggregation=new Aggregation(group_by_attributes,aggregation_attributes,aggregation_function,project1); //==========================project========================= vector< vector<ExpressionItem> >expr_list2; ExpressionItem ei21_1; ei21_1.setVariable("LINEITEM.row_id+1"); vector<ExpressionItem> expr21; expr21.push_back(ei21_1); expr_list2.push_back(expr21); LogicalOperator* project2=new LogicalProject(project1,expr_list2); //===========================root=========================== LogicalOperator* root=new LogicalQueryPlanRoot(0,project2,LogicalQueryPlanRoot::PRINT); cout<<"performance is ok!"<<endl; BlockStreamIteratorBase* physical_iterator_tree=root->getIteratorTree(64*1024); // physical_iterator_tree->print(); physical_iterator_tree->open(); while(physical_iterator_tree->next(0)); physical_iterator_tree->close(); printf("Q1: execution time: %4.4f second.\n",getSecond(start)); }
static void query_select_sort_string() { /* * select sum(a+1)+count(a),b * from T * group by b * * notation: p a p s * */ unsigned long long int start=curtick(); TableDescriptor* table=Environment::getInstance()->getCatalog()->getTable("LINEITEM"); //===========================scan=========================== LogicalOperator* scan=new LogicalScan(table->getProjectoin(0)); //==========================project========================= vector< vector<ExpressionItem> >expr_list1; // vector<ExpressionItem> expr1; vector<ExpressionItem> expr2; vector<ExpressionItem> expr3; ExpressionItem ei1_1; ExpressionItem ei1_2; ExpressionItem ei1_3; ExpressionItem ei2; ExpressionItem ei3; // ei1_1.setVariable("LINEITEM.L_ORDERKEY"); // ei1_2.setIntValue("1"); // ei1_3.setOperator("+"); ei2.setVariable("LINEITEM.L_RETURNFLAG"); ei3.setVariable("LINEITEM.L_LINESTATUS"); // expr1.push_back(ei1_1); // expr1.push_back(ei1_2); // expr1.push_back(ei1_3); expr2.push_back(ei2); expr3.push_back(ei3); // expr_list1.push_back(expr1); expr_list1.push_back(expr2); expr_list1.push_back(expr3); LogicalOperator* project1=new LogicalProject(scan,expr_list1); //============================groupby========================== // std::vector<Attribute> group_by_attributes; // group_by_attributes.push_back(project1->getDataflow().getAttribute("LINEITEM.L_RETURNFLAG")); // group_by_attributes.push_back(project1->getDataflow().getAttribute("LINEITEM.L_LINESTATUS")); // std::vector<Attribute> aggregation_attributes; // aggregation_attributes.push_back(table->getAttribute("L_QUANTITY")); // aggregation_attributes.push_back(table->getAttribute("L_EXTENDEDPRICE")); // aggregation_attributes.push_back(table->getAttribute("L_DISCOUNT")); // aggregation_attributes.push_back(Attribute(ATTRIBUTE_ANY)); // std::vector<BlockStreamAggregationIterator::State::aggregation> aggregation_function; // aggregation_function.push_back(BlockStreamAggregationIterator::State::sum); // aggregation_function.push_back(BlockStreamAggregationIterator::State::sum); // aggregation_function.push_back(BlockStreamAggregationIterator::State::sum); // aggregation_function.push_back(BlockStreamAggregationIterator::State::count); // LogicalOperator* aggregation=new Aggregation(group_by_attributes,aggregation_attributes,aggregation_function,project1); //==========================project========================= // vector< vector<ExpressionItem> >expr_list2; // //// ExpressionItem ei21_1; // ExpressionItem ei22; // ExpressionItem ei23; //// ei21_1.setVariable("LINEITEM.L_ORDERKEY+1"); // ei22.setVariable("LINEITEM.L_RETURNFLAG"); // ei23.setVariable("LINEITEM.L_LINESTATUS"); //// vector<ExpressionItem> expr21; // vector<ExpressionItem> expr22; // vector<ExpressionItem> expr23; //// expr21.push_back(ei21_1); // expr22.push_back(ei22); // expr23.push_back(ei23); //// expr_list2.push_back(expr21); // expr_list2.push_back(expr22); // expr_list2.push_back(expr23); // LogicalOperator* project2=new LogicalProject(aggregation,expr_list2); //============================sort========================== vector<LogicalSort::OrderByAttr*>vo; // vo.push_back(&LogicalSort::OrderByAttr("LINEITEM.L_ORDERKEY+1")); //get_orderby_att(vo); // char * orderstr1="LINEITEM.L_LINESTATUS"; // char * orderstr2="LINEITEM.L_RETURNFLAG"; LogicalSort::OrderByAttr tmp1("LINEITEM.L_LINESTATUS",1); LogicalSort::OrderByAttr tmp2("LINEITEM.L_RETURNFLAG",0); vo.push_back(&tmp1); vo.push_back(&tmp2); for(int i=0;i<vo.size();i++) { printf("vo[%d]= %s len=%d aa=%x sa=%x\n",i,(vo[i])->ta_,strlen(vo[i]->ta_) ,vo[i],vo[i]->ta_); } LogicalOperator* sort1=new LogicalSort(project1,vo); sort1->print(); cout<<"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"<<endl; //===========================root=========================== LogicalOperator* root=new LogicalQueryPlanRoot(0,sort1,LogicalQueryPlanRoot::PRINT); root->print(); cout<<"performance is ok!"<<endl; getchar(); getchar(); // BlockStreamIteratorBase* physical_iterator_tree=root->getIteratorTree(64*1024); //// physical_iterator_tree->print(); // physical_iterator_tree->open(); // while(physical_iterator_tree->next(0)); // physical_iterator_tree->close(); printf("Q1: execution time: %4.4f second.\n",getSecond(start)); }
bool PhysicalSort::Open(const PartitionOffset &part_off) { /** * TODO(anyone): multi threads can be used to pipeline!!! */ swap_num_ = 0; temp_cur_ = 0; /** * first we can store all the data which will be bufferred * 1, buffer is the first phase. multi-threads will be applyed to the data * in the buffer. * 2, sort the data in the buffer, we choose quicksort to sort the records * by specifying the column to be sorted * 3, whether to register the buffer into the blockmanager. * */ BlockStreamBase *block_for_asking; state_.partition_offset_ = part_off; state_.child_->Open(state_.partition_offset_); if (sema_open_.try_wait()) { block_buffer_iterator_ = block_buffer_.createIterator(); open_finished_ = true; } else { while (!open_finished_) { usleep(1); } } if (CreateBlockStream(block_for_asking) == false) { LOG(ERROR) << "error in the create block stream!!!" << endl; return 0; } /** * phase 1: store the data in the buffer! * by using multi-threads to speed up */ unsigned block_offset = 0; unsigned tuple_count_sum = 0; BlockStreamBase::BlockStreamTraverseIterator *iterator_for_scan; while (state_.child_->Next(block_for_asking)) { tuple_count_sum += block_for_asking->getTuplesInBlock(); block_buffer_.atomicAppendNewBlock(block_for_asking); iterator_for_scan = block_buffer_.getBlock(block_offset)->createIterator(); void *tuple_ptr = 0; while ((tuple_ptr = iterator_for_scan->nextTuple()) != 0) { tuple_vector_.push_back(tuple_ptr); } block_offset++; if (CreateBlockStream(block_for_asking) == false) { LOG(ERROR) << "error in the create block stream!!!" << endl; return 0; } } /** * phase 2: sort the data in the buffer! * by using multi-threads to speed up? * TODO(anyone): whether to store the sorted data into the blockmanager. */ // cout<<"check the memory usage!!!"<<endl; unsigned long long int time = curtick(); // order(state_.orderbyKey_,tuple_count_sum); Order(); // cout<<"the tuple_count is: "<<tuple_count_sum<<"Total time: // "<<getSecond(time)<<" seconds, the swap num is: "<<swap_num<<endl; return true; }
/** * @brief Method description : describe the open method which gets results from * the left child and copy them into its local buffer, say the block buffer. the * block buffer is a dynamic block buffer since all the expanded threads will * share the same block buffer. */ bool PhysicalNestLoopJoin::Open(SegmentExecStatus *const exec_status, const PartitionOffset &partition_offset) { RETURN_IF_CANCELLED(exec_status); RegisterExpandedThreadToAllBarriers(); unsigned long long int timer; bool winning_thread = false; if (TryEntryIntoSerializedSection(0)) { // the first thread of all need to do ExpanderTracker::getInstance()->addNewStageEndpoint( pthread_self(), LocalStageEndPoint(stage_desc, "nest loop", 0)); winning_thread = true; timer = curtick(); block_buffer_ = new DynamicBlockBuffer(); if (state_.join_condi_.size() == 0) { join_condi_process_ = WithoutJoinCondi; } else { join_condi_process_ = WithJoinCondi; } LOG(INFO) << "[NestloopJoin]: [the first thread opens the nestloopJoin " "physical operator]" << std::endl; } RETURN_IF_CANCELLED(exec_status); state_.child_left_->Open(exec_status, partition_offset); RETURN_IF_CANCELLED(exec_status); BarrierArrive(0); NestLoopJoinContext *jtc = CreateOrReuseContext(crm_numa_sensitive); // create a new block to hold the results from the left child // and add results to the dynamic buffer // jtc->block_for_asking_ == BlockStreamBase::createBlock( // state_.input_schema_left_, // state_.block_size_); CreateBlockStream(jtc->block_for_asking_, state_.input_schema_left_); // auto temp = jtc->block_for_asking_->getBlock(); // cout << "temp start" << temp << endl; // // cout << "init block_for_asking_ : " << jtc->block_for_asking_->getBlock() // << " is reference : " << jtc->block_for_asking_->isIsReference() << // endl; while (state_.child_left_->Next(exec_status, jtc->block_for_asking_)) { if (exec_status->is_cancelled()) { if (NULL != jtc->block_for_asking_) { delete jtc->block_for_asking_; jtc->block_for_asking_ = NULL; } return false; } // cout << "after assgin start :" << jtc->block_for_asking_->getBlock() // << " is reference : " << jtc->block_for_asking_->isIsReference() // << endl; block_buffer_->atomicAppendNewBlock(jtc->block_for_asking_); // if (!jtc->block_for_asking_->isIsReference()) { CreateBlockStream(jtc->block_for_asking_, state_.input_schema_left_); // } else { // // cout << "temp after" << temp << endl; // // delete temp; // CreateBlockStream(jtc->block_for_asking_, // state_.input_schema_left_); // jtc->block_for_asking_->setIsReference(false); // } // cout << "new start :" << jtc->block_for_asking_->getBlock() // << " is reference : " << jtc->block_for_asking_->isIsReference() // << endl; } // cout << "buffer_size_ : " << block_buffer_->GetBufferSize() << endl; // the last block is created without storing the results from the left // child if (NULL != jtc->block_for_asking_) { delete jtc->block_for_asking_; jtc->block_for_asking_ = NULL; } // when the finished expanded thread finished its allocated work, it can be // called back here. What should be noticed that the callback meas the to // exit on the of the thread if (ExpanderTracker::getInstance()->isExpandedThreadCallBack( pthread_self())) { UnregisterExpandedThreadToAllBarriers(1); LOG(INFO) << "[NestloopJoin]: [the" << pthread_self() << "the thread is called to exit]" << std::endl; return true; // the } BarrierArrive(1); // ??ERROR // join_thread_context* jtc=new join_thread_context(); // jtc->block_for_asking_ == BlockStreamBase::createBlock( // state_.input_schema_right_, // state_.block_size_); CreateBlockStream(jtc->block_for_asking_, state_.input_schema_right_); jtc->block_for_asking_->setEmpty(); jtc->block_stream_iterator_ = jtc->block_for_asking_->createIterator(); jtc->buffer_iterator_ = block_buffer_->createIterator(); // underlying bug: as for buffer_iterator may be NULL, it's necessary to let // every buffer_iterator of each thread point to an empty block // jtc->buffer_stream_iterator_ = // jtc->buffer_iterator_.nextBlock()->createIterator(); InitContext(jtc); // rename this function, here means to store the thread // context in the operator context RETURN_IF_CANCELLED(exec_status); state_.child_right_->Open(exec_status, partition_offset); return true; }