bool PhysicalProjectionScan::Open(SegmentExecStatus* const exec_status, const PartitionOffset& kPartitionOffset) { RETURN_IF_CANCELLED(exec_status); RegisterExpandedThreadToAllBarriers(); if (TryEntryIntoSerializedSection()) { /* this is the first expanded thread*/ PartitionStorage* partition_handle_; if (NULL == (partition_handle_ = BlockManager::getInstance()->GetPartitionHandle( PartitionID(state_.projection_id_, kPartitionOffset)))) { LOG(ERROR) << PartitionID(state_.projection_id_, kPartitionOffset) .getName() .c_str() << CStrError(rNoPartitionIdScan) << std::endl; SetReturnStatus(false); } else { partition_reader_iterator_ = partition_handle_->CreateAtomicReaderIterator(); SetReturnStatus(true); } #ifdef AVOID_CONTENTION_IN_SCAN unsigned long long start = curtick(); ChunkReaderIterator* chunk_reader_it; ChunkReaderIterator::block_accessor* ba; while (chunk_reader_it = partition_reader_iterator_->NextChunk()) { while (chunk_reader_it->GetNextBlockAccessor(ba)) { ba->GetBlockSize(); input_dataset_.input_data_blocks_.push_back(ba); } } #endif ExpanderTracker::getInstance()->addNewStageEndpoint( pthread_self(), LocalStageEndPoint(stage_src, "Scan", 0)); perf_info_ = ExpanderTracker::getInstance()->getPerformanceInfo(pthread_self()); perf_info_->initialize(); } BarrierArrive(); return GetReturnStatus(); }
bool IndexScanIterator::Open(const PartitionOffset& partition_off) { AtomicPushBlockStream(BlockStreamBase::createBlockWithDesirableSerilaizedSize(state_.schema_, state_.block_size_)); if(TryEntryIntoSerializedSection()){ /* this is the first expanded thread*/ csb_index_list_ = IndexManager::getInstance()->getAttrIndex(state_.index_id_); PartitionStorage* partition_handle_; if((partition_handle_=BlockManager::getInstance()->getPartitionHandle(PartitionID(state_.projection_id_,partition_off)))==0){ printf("The partition[%s] does not exists!\n",PartitionID(state_.projection_id_,partition_off).getName().c_str()); SetReturnStatus(false); } else{ partition_reader_iterator_=partition_handle_->createAtomicReaderIterator(); // chunk_reader_iterator_ = partition_reader_iterator_->nextChunk(); } SetReturnStatus(true); } BarrierArrive(); return GetReturnStatus(); }
/** * delete the file associated with the Table table_name only * @param table_name * @return */ RetCode DropTableExec::DeleteTableFiles(const string& table_name) { RetCode ret = rSuccess; Catalog* local_catalog = Environment::getInstance()->getCatalog(); TableDescriptor* table_desc = local_catalog->getTable(table_name); // start to delete the files vector<vector<string>> write_path; for (int i = 0; i < table_desc->getNumberOfProjection(); i++) { vector<string> prj_write_path; prj_write_path.clear(); for (int j = 0; j < table_desc->getProjectoin(i) ->getPartitioner() ->getNumberOfPartitions(); ++j) { string path = PartitionID(table_desc->getProjectoin(i)->getProjectionID(), j).getPathAndName(); prj_write_path.push_back(path); } write_path.push_back(prj_write_path); // unbound the file in memory if (table_desc->getProjectoin(i)->getPartitioner()->allPartitionBound()) { Catalog::getInstance()->getBindingModele()->UnbindingEntireProjection( table_desc->getProjectoin(i)->getPartitioner()); } } TableFileConnector* connector = new TableFileConnector( Config::local_disk_mode ? FilePlatform::kDisk : FilePlatform::kHdfs, write_path); ret = connector->DeleteFiles(); if (ret != rSuccess) { ELOG(ret, "failed to delete the projections, when delete the file on table" + table_name); return ret; } return ret; }
HdfsLoader::HdfsLoader(const char c_separator,const char r_separator, std::vector<std::string> file_name, TableDescriptor* tableDescriptor, open_flag open_flag_) :table_descriptor_(tableDescriptor), col_separator(c_separator), row_separator(r_separator), file_list(file_name), open_flag_(open_flag_), block_size(64*1024) { if (open_flag_ == APPEND) row_id = table_descriptor_->getRowNumber(); else row_id = 0; table_schema = table_descriptor_->getSchema(); vector <unsigned> prj_index; for(int i = 0; i < table_descriptor_->getNumberOfProjection(); i++) { projection_schema.push_back(table_descriptor_->getProjectoin(i)->getSchema()); vector<string> prj_writepath; prj_writepath.clear(); for(int j=0; j<table_descriptor_->getProjectoin(i)->getPartitioner()->getNumberOfPartitions();j++){ string path = PartitionID(table_descriptor_->getProjectoin(i)->getProjectionID(),j).getPathAndName(); prj_writepath.push_back(path); } writepath.push_back(prj_writepath); Attribute partition_attribute = table_descriptor_->getProjectoin(i)->getPartitioner()->getPartitionKey(); int hash_key_index=table_descriptor_->getProjectoin(i)->getAttributeIndex(partition_attribute); partition_key_index.push_back(hash_key_index); PartitionFunction* pf = table_descriptor_->getProjectoin(i)->getPartitioner()->getPartitionFunction(); partition_functin_list_.push_back(pf); prj_index.clear(); for (int j = 0; j < table_descriptor_->getProjectoin(i)->getAttributeList().size(); j++) { prj_index.push_back(table_descriptor_->getProjectoin(i)->getAttributeList()[j].index); } SubTuple* st = new SubTuple(table_descriptor_->getSchema(), table_descriptor_->getProjectoin(i)->getSchema(), prj_index); sub_tuple_generator.push_back(st); } sblock = new Block(BLOCK_SIZE); ///////////////////////////////////assign 64k buffer to each PJ///////////////////////////// for(int i = 0; i < table_descriptor_->getNumberOfProjection(); i++) { vector<BlockStreamBase*> temp_v; vector<unsigned long> tmp_block_num; vector<unsigned long> tmp_tuple_count; for(int j = 0; j < table_descriptor_->getProjectoin(i)->getPartitioner()->getNumberOfPartitions(); j++) { temp_v.push_back(BlockStreamBase::createBlock(table_descriptor_->getProjectoin(i)->getSchema(), block_size-sizeof(unsigned))); if (open_flag_ == APPEND) { tmp_tuple_count.push_back(table_descriptor_->getProjectoin(i)->getPartitioner()->getPartitionCardinality(j)); tmp_block_num.push_back(table_descriptor_->getProjectoin(i)->getPartitioner()->getPartitionBlocks(j)); } else { tmp_tuple_count.push_back(0); tmp_block_num.push_back(0); } //ERROR: the init table_descriptor_->getProjectoin(i)->getPartitioner()->getPartitionChunks(j) is wrong!!! /*for testing*/ cout << "init number of partitions " << i << "\t" << j << "\t:" << table_descriptor_->getProjectoin(i)->getPartitioner()->getPartitionBlocks(j) << endl; } pj_buffer.push_back(temp_v); blocks_per_partition.push_back(tmp_block_num); tuples_per_partition.push_back(tmp_tuple_count); } /////////////////////////////////////////////////////////////////////////////////////////// }