bool PhysicalProjectionScan::Open(SegmentExecStatus* const exec_status,
const PartitionOffset& kPartitionOffset) {
RETURN_IF_CANCELLED(exec_status);
RegisterExpandedThreadToAllBarriers();
if (TryEntryIntoSerializedSection()) {
/* this is the first expanded thread*/
PartitionStorage* partition_handle_;
if (NULL ==
(partition_handle_ = BlockManager::getInstance()->GetPartitionHandle(
PartitionID(state_.projection_id_, kPartitionOffset)))) {
LOG(ERROR) << PartitionID(state_.projection_id_, kPartitionOffset)
.getName()
.c_str() << CStrError(rNoPartitionIdScan) << std::endl;
SetReturnStatus(false);
} else {
partition_reader_iterator_ =
partition_handle_->CreateAtomicReaderIterator();
SetReturnStatus(true);
}
#ifdef AVOID_CONTENTION_IN_SCAN
unsigned long long start = curtick();
ChunkReaderIterator* chunk_reader_it;
ChunkReaderIterator::block_accessor* ba;
while (chunk_reader_it = partition_reader_iterator_->NextChunk()) {
while (chunk_reader_it->GetNextBlockAccessor(ba)) {
ba->GetBlockSize();
input_dataset_.input_data_blocks_.push_back(ba);
}
}
#endif
ExpanderTracker::getInstance()->addNewStageEndpoint(
pthread_self(), LocalStageEndPoint(stage_src, "Scan", 0));
perf_info_ =
ExpanderTracker::getInstance()->getPerformanceInfo(pthread_self());
perf_info_->initialize();
}
BarrierArrive();
return GetReturnStatus();
}
bool IndexScanIterator::Open(const PartitionOffset& partition_off)
{
AtomicPushBlockStream(BlockStreamBase::createBlockWithDesirableSerilaizedSize(state_.schema_, state_.block_size_));
if(TryEntryIntoSerializedSection()){
/* this is the first expanded thread*/
csb_index_list_ = IndexManager::getInstance()->getAttrIndex(state_.index_id_);
PartitionStorage* partition_handle_;
if((partition_handle_=BlockManager::getInstance()->getPartitionHandle(PartitionID(state_.projection_id_,partition_off)))==0){
printf("The partition[%s] does not exists!\n",PartitionID(state_.projection_id_,partition_off).getName().c_str());
SetReturnStatus(false);
}
else{
partition_reader_iterator_=partition_handle_->createAtomicReaderIterator();
// chunk_reader_iterator_ = partition_reader_iterator_->nextChunk();
}
SetReturnStatus(true);
}
BarrierArrive();
return GetReturnStatus();
}
/**
* delete the file associated with the Table table_name only
* @param table_name
* @return
*/
RetCode DropTableExec::DeleteTableFiles(const string& table_name) {
RetCode ret = rSuccess;
Catalog* local_catalog = Environment::getInstance()->getCatalog();
TableDescriptor* table_desc = local_catalog->getTable(table_name);
// start to delete the files
vector<vector<string>> write_path;
for (int i = 0; i < table_desc->getNumberOfProjection(); i++) {
vector<string> prj_write_path;
prj_write_path.clear();
for (int j = 0; j < table_desc->getProjectoin(i)
->getPartitioner()
->getNumberOfPartitions();
++j) {
string path = PartitionID(table_desc->getProjectoin(i)->getProjectionID(),
j).getPathAndName();
prj_write_path.push_back(path);
}
write_path.push_back(prj_write_path);
// unbound the file in memory
if (table_desc->getProjectoin(i)->getPartitioner()->allPartitionBound()) {
Catalog::getInstance()->getBindingModele()->UnbindingEntireProjection(
table_desc->getProjectoin(i)->getPartitioner());
}
}
TableFileConnector* connector = new TableFileConnector(
Config::local_disk_mode ? FilePlatform::kDisk : FilePlatform::kHdfs,
write_path);
ret = connector->DeleteFiles();
if (ret != rSuccess) {
ELOG(ret,
"failed to delete the projections, when delete the file on table" +
table_name);
return ret;
}
return ret;
}
HdfsLoader::HdfsLoader(const char c_separator,const char r_separator, std::vector<std::string> file_name, TableDescriptor* tableDescriptor, open_flag open_flag_)
:table_descriptor_(tableDescriptor), col_separator(c_separator), row_separator(r_separator), file_list(file_name), open_flag_(open_flag_), block_size(64*1024)
{
if (open_flag_ == APPEND)
row_id = table_descriptor_->getRowNumber();
else
row_id = 0;
table_schema = table_descriptor_->getSchema();
vector <unsigned> prj_index;
for(int i = 0; i < table_descriptor_->getNumberOfProjection(); i++)
{
projection_schema.push_back(table_descriptor_->getProjectoin(i)->getSchema());
vector<string> prj_writepath;
prj_writepath.clear();
for(int j=0; j<table_descriptor_->getProjectoin(i)->getPartitioner()->getNumberOfPartitions();j++){
string path = PartitionID(table_descriptor_->getProjectoin(i)->getProjectionID(),j).getPathAndName();
prj_writepath.push_back(path);
}
writepath.push_back(prj_writepath);
Attribute partition_attribute = table_descriptor_->getProjectoin(i)->getPartitioner()->getPartitionKey();
int hash_key_index=table_descriptor_->getProjectoin(i)->getAttributeIndex(partition_attribute);
partition_key_index.push_back(hash_key_index);
PartitionFunction* pf = table_descriptor_->getProjectoin(i)->getPartitioner()->getPartitionFunction();
partition_functin_list_.push_back(pf);
prj_index.clear();
for (int j = 0; j < table_descriptor_->getProjectoin(i)->getAttributeList().size(); j++)
{
prj_index.push_back(table_descriptor_->getProjectoin(i)->getAttributeList()[j].index);
}
SubTuple* st = new SubTuple(table_descriptor_->getSchema(), table_descriptor_->getProjectoin(i)->getSchema(), prj_index);
sub_tuple_generator.push_back(st);
}
sblock = new Block(BLOCK_SIZE);
///////////////////////////////////assign 64k buffer to each PJ/////////////////////////////
for(int i = 0; i < table_descriptor_->getNumberOfProjection(); i++)
{
vector<BlockStreamBase*> temp_v;
vector<unsigned long> tmp_block_num;
vector<unsigned long> tmp_tuple_count;
for(int j = 0; j < table_descriptor_->getProjectoin(i)->getPartitioner()->getNumberOfPartitions(); j++)
{
temp_v.push_back(BlockStreamBase::createBlock(table_descriptor_->getProjectoin(i)->getSchema(), block_size-sizeof(unsigned)));
if (open_flag_ == APPEND) {
tmp_tuple_count.push_back(table_descriptor_->getProjectoin(i)->getPartitioner()->getPartitionCardinality(j));
tmp_block_num.push_back(table_descriptor_->getProjectoin(i)->getPartitioner()->getPartitionBlocks(j));
}
else {
tmp_tuple_count.push_back(0);
tmp_block_num.push_back(0);
}
//ERROR: the init table_descriptor_->getProjectoin(i)->getPartitioner()->getPartitionChunks(j) is wrong!!!
/*for testing*/ cout << "init number of partitions " << i << "\t" << j << "\t:" << table_descriptor_->getProjectoin(i)->getPartitioner()->getPartitionBlocks(j) << endl;
}
pj_buffer.push_back(temp_v);
blocks_per_partition.push_back(tmp_block_num);
tuples_per_partition.push_back(tmp_tuple_count);
}
///////////////////////////////////////////////////////////////////////////////////////////
}
