/*******#include "../physical_operator/ExpandableBlockStreamSingleColumnScan.h"
*****Local Disk Access******#include "../physical_operator/ExpandableBlockStreamSingleColumnScanDisk.h"
******/
int main_ld (int argc, char** argv)
{
const unsigned block_size = atoi(argv[1]);
const unsigned thread_count = atoi(argv[2]);
const unsigned expander_buffer = atoi(argv[3]);
const int columns_num = atoi(argv[4]);
unsigned long random_size = atoi(argv[5]); //number of columns which are randomly accessed
random_size = random_size*sizeof(int);
unsigned long scan_size = (unsigned long)4*1024*1024*1024-random_size*columns_num;
cout << "columns_num: " << columns_num << "\t random_size: " << random_size/sizeof(int) << endl;
std::vector<column_type> column_list,column_list_;
column_list.push_back(column_type(t_int));
for (int i = 0; i < columns_num; i++)
column_list_.push_back(column_type(t_int));
Schema* i_schema = new SchemaFix(column_list);
Schema* d_schema = new SchemaFix(column_list_);
// ExpandableBlockStreamSingleColumnScan::State ebssc_state1("/home/claims/temp/6_0.column", d_schema, block_size);
// BlockStreamIteratorBase* ebssc1 = new ExpandableBlockStreamSingleColumnScan(ebssc_state1);
// ebssc1->open();
// BlockStreamBase* block_=BlockStreamBase::createBlock(i_schema,block_size);
// while (ebssc1->next(block_)) {};
// ebssc1->close();
// block_->~BlockStreamBase();
ExpandableBlockStreamSingleColumnScanDisk::State ebsscsd_state("/home/claims/temp/5_0.column", d_schema, block_size, scan_size);
PhysicalOperatorBase* ebsscsd = new ExpandableBlockStreamSingleColumnScanDisk(ebsscsd_state);
ExpandableBlockStreamSingleColumnScan::State ebssc_state("/home/claims/temp/6_0.column", i_schema, block_size, random_size);
PhysicalOperatorBase* ebssc = new ExpandableBlockStreamSingleColumnScan(ebssc_state);
// BlockStreamExpander::State bse_state(i_schema,ebssc,thread_count,block_size,expander_buffer);
// BlockStreamIteratorBase* bse=new BlockStreamExpander(bse_state);
ExpandableBlockStreamRandomDiskAccess::State ebsrda_state("/home/claims/temp/Uniform_0_9999.column", ebssc, d_schema, i_schema, block_size);
PhysicalOperatorBase* ebsrda = new ExpandableBlockStreamRandomDiskAccess(ebsrda_state);
Expander::State bse_state1(d_schema,ebsscsd,thread_count,block_size,expander_buffer);
PhysicalOperatorBase* bse1=new Expander(bse_state1);
Expander::State bse_state2(d_schema,ebsrda,thread_count,block_size,expander_buffer);
PhysicalOperatorBase* bse2=new Expander(bse_state2);
BlockStreamPerformanceTest::State bspt_state(d_schema, bse1, bse2, block_size, 1000);
PhysicalOperatorBase* bspt = new BlockStreamPerformanceTest(bspt_state);
bspt->Open();
while (bspt->Next(0));
bspt->Close();
return 0;
}
unsigned long Analyzer::getDistinctCardinality(const AttributeID& attr_id) {
LogicalOperator* scan = new LogicalScan(
Catalog::getInstance()->getTable(attr_id.table_id)->getProjectoin(0));
std::vector<Attribute> group_by_attributes;
group_by_attributes.push_back(
Catalog::getInstance()->getTable(attr_id.table_id)->getAttribute(
attr_id.offset));
LogicalOperator* agg = new LogicalAggregation(
group_by_attributes, std::vector<Attribute>(),
std::vector<PhysicalAggregation::State::Aggregation>(), scan);
std::vector<Attribute> aggregation_attributes;
aggregation_attributes.push_back(Attribute(ATTRIBUTE_ANY));
std::vector<PhysicalAggregation::State::Aggregation> aggregation_function;
aggregation_function.push_back(PhysicalAggregation::State::kCount);
LogicalOperator* count_agg =
new LogicalAggregation(std::vector<Attribute>(), aggregation_attributes,
aggregation_function, agg);
LogicalOperator* root = new LogicalQueryPlanRoot(
0, count_agg, LogicalQueryPlanRoot::kResultCollector);
PhysicalOperatorBase* collector =
root->GetPhysicalPlan(1024 * 64 - sizeof(unsigned));
collector->Open();
collector->Next(0);
collector->Close();
ResultSet* resultset = collector->GetResultSet();
ResultSet::Iterator it = resultset->createIterator();
BlockStreamBase::BlockStreamTraverseIterator* b_it =
it.nextBlock()->createIterator();
const unsigned long distinct_cardinality = *(unsigned long*)b_it->nextTuple();
resultset->destory();
collector->~PhysicalOperatorBase();
root->~LogicalOperator();
return distinct_cardinality;
}
ResultSet* Executing::run_sql(std::string sql,std::string& error){
/*
* should call ExecuteLogicalQueryPlan. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! by yukai
* */
// char * test="select row_id,count(*) from cj where cj.row_id=1 group by cj.row_id;";
ResultSet* resultset=0;
Node* node=getparsetreeroot(sql.c_str());
if(node==0){
error="sql parser error!";
return resultset;
}
LogicalOperator* plan=parsetree2logicalplan(node);
if(plan==0){
error="query optimization error!";
return resultset;
}
plan->Print();
LogicalOperator* root=new LogicalQueryPlanRoot(0,plan,LogicalQueryPlanRoot::kResultCollector);
// unsigned long long int timer_start=curtick();
PhysicalOperatorBase* collector = root->GetPhysicalPlan(
1024 * 64 - sizeof(unsigned));
collector->Print();
collector->Open();
collector->Next(0);
collector->Close();
resultset = collector->GetResultSet();
root->~LogicalOperator();;
return resultset;
}
void Analyzer::compute_table_stat(const TableID& tab_id) {
TableDescriptor* table = Catalog::getInstance()->getTable(tab_id);
LogicalOperator* scan = new LogicalScan(table->getProjectoin(0));
std::vector<Attribute> group_by_attributes;
std::vector<Attribute> aggregation_attributes;
aggregation_attributes.push_back(Attribute(ATTRIBUTE_ANY));
std::vector<PhysicalAggregation::State::Aggregation> aggregation_function;
aggregation_function.push_back(PhysicalAggregation::State::kCount);
LogicalOperator* agg = new LogicalAggregation(
group_by_attributes, aggregation_attributes, aggregation_function, scan);
LogicalOperator* root =
new LogicalQueryPlanRoot(0, agg, LogicalQueryPlanRoot::kResultCollector);
PhysicalOperatorBase* collector =
root->GetPhysicalPlan(1024 * 64 - sizeof(unsigned));
collector->Open();
collector->Next(0);
collector->Close();
ResultSet* resultset = collector->GetResultSet();
ResultSet::Iterator it = resultset->createIterator();
BlockStreamBase::BlockStreamTraverseIterator* b_it =
it.nextBlock()->createIterator();
const unsigned long tuple_count = *(unsigned long*)b_it->nextTuple();
BlockStreamBase* block;
while (block = it.nextBlock()) {
BlockStreamBase::BlockStreamTraverseIterator* b_it =
block->createIterator();
}
TableStatistic* tab_stat = new TableStatistic();
tab_stat->number_of_tuples_ = tuple_count;
printf("Statistics for table %s is gathered!\n",
Catalog::getInstance()->getTable(tab_id)->getTableName().c_str());
tab_stat->print();
StatManager::getInstance()->setTableStatistic(tab_id, tab_stat);
resultset->destory();
root->~LogicalOperator();
}
static int test_CSBIndexBuilding ()
{
int master;
printf("~!OKOKO!!!!!\n");
printf("Master(0) or Slave(others)??\n");
scanf("%d",&master);
if(master!=0){
Environment::getInstance(false);
}
else{
Environment::getInstance(true);
ResourceManagerMaster *rmms=Environment::getInstance()->getResourceManagerMaster();
Catalog* catalog=Environment::getInstance()->getCatalog();
TableDescriptor* table_1=new TableDescriptor("cj",Environment::getInstance()->getCatalog()->allocate_unique_table_id());
table_1->addAttribute("row_id",data_type(t_u_long)); //0
table_1->addAttribute("trade_date",data_type(t_int));
table_1->addAttribute("order_no",data_type(t_u_long));
table_1->addAttribute("sec_code",data_type(t_int));
table_1->addAttribute("trade_dir",data_type(t_int));
table_1->addAttribute("order_type",data_type(t_int)); //5
vector<ColumnOffset> cj_proj0_index;
cj_proj0_index.push_back(0);
cj_proj0_index.push_back(1);
cj_proj0_index.push_back(2);
cj_proj0_index.push_back(3);
cj_proj0_index.push_back(4);
cj_proj0_index.push_back(5);
const int partition_key_index_1=2;
table_1->createHashPartitionedProjection(cj_proj0_index,"order_no",4); //G0
catalog->add_table(table_1);
////////////////////////////////////////
/* the following codes should be triggered by Load module*/
//////////////////ONE DAY////////////////////////////////////////////////
//cj_table
// 4 partitions partitioned by order_no
for(unsigned i=0;i<table_1->getProjectoin(0)->getPartitioner()->getNumberOfPartitions();i++){
catalog->getTable(0)->getProjectoin(0)->getPartitioner()->RegisterPartition(i,2);
}
/////////////////////////////////////////
ProjectionBinding *pb=new ProjectionBinding();
pb->BindingEntireProjection(catalog->getTable(0)->getProjectoin(0)->getPartitioner());
printf("ready(?)\n");
int input;
scanf("%d",&input);
vector<column_type> blc_column_list;
blc_column_list.push_back(column_type(t_u_long));
blc_column_list.push_back(column_type(t_int));
blc_column_list.push_back(column_type(t_u_long));
blc_column_list.push_back(column_type(t_int)); //sec_code for indexing
blc_column_list.push_back(column_type(t_int));
blc_column_list.push_back(column_type(t_int));
/*for testing the original data*/
// Schema* ps_schema = new SchemaFix(blc_column_list);
// ExpandableBlockStreamProjectionScan::State ps_state(catalog->getTable(0)->getProjectoin(0)->getProjectionID(), ps_schema, 64*1024);
// ExpandableBlockStreamIteratorBase* ps = new ExpandableBlockStreamProjectionScan(ps_state);
// ps->open();
// BlockStreamBase* block = BlockStreamBase::createBlockWithDesirableSerilaizedSize(ps_schema, 64*1024);
// void* tuple;
// while (ps->next(block))
// {
// BlockStreamBase::BlockStreamTraverseIterator* iterator = block->createIterator();
// while((tuple = iterator->nextTuple()) != 0)
// {
// ps_schema->displayTuple(tuple, " | ");
// sleep(1);
// }
//
// }
// ps->close();
/*for testing the original data*/
Schema* blc_schema = new SchemaFix(blc_column_list);
unsigned block_size = 64*1024;
bottomLayerCollecting::State blc_state(catalog->getTable(0)->getProjectoin(0)->getProjectionID(), blc_schema, 3, block_size);
PhysicalOperatorBase* blc = new bottomLayerCollecting(blc_state);
vector<column_type> bls_column_list;
bls_column_list.push_back(t_int); //chunk offset
bls_column_list.push_back(t_int); //sec_code
bls_column_list.push_back(t_u_smallInt); //chunk offset
bls_column_list.push_back(t_u_smallInt); //chunk offset
Schema* bls_schema = new SchemaFix(bls_column_list);
bottomLayerSorting::State bls_state(bls_schema, blc, block_size, catalog->getTable(0)->getProjectoin(0)->getProjectionID(), 3, "sec_code_index");
PhysicalOperatorBase* bls = new bottomLayerSorting(bls_state);
bls->Open();
BlockStreamBase* block;
while(bls->Next(block))
{
}
bls->Close();
//following for test the index manager~
test_index_manager_();
}
cout<<"Waiting~~~!~"<<endl;
while(true){
sleep(1);
}
return 0;
}
static int test_index_scan_iterator()
{
int master;
printf("~!OKOKO!!!!!\n");
printf("Master(0) or Slave(others)??\n");
scanf("%d",&master);
if(master!=0){
Environment::getInstance(false);
}
else{
Environment::getInstance(true);
ResourceManagerMaster *rmms=Environment::getInstance()->getResourceManagerMaster();
Catalog* catalog=Environment::getInstance()->getCatalog();
TableDescriptor* table_1=new TableDescriptor("cj",Environment::getInstance()->getCatalog()->allocate_unique_table_id());
table_1->addAttribute("row_id",data_type(t_u_long)); //0
table_1->addAttribute("trade_date",data_type(t_int));
table_1->addAttribute("order_no",data_type(t_u_long));
table_1->addAttribute("sec_code",data_type(t_int));
table_1->addAttribute("trade_dir",data_type(t_int));
table_1->addAttribute("order_type",data_type(t_int)); //5
vector<ColumnOffset> cj_proj0_index;
cj_proj0_index.push_back(0);
cj_proj0_index.push_back(1);
cj_proj0_index.push_back(2);
cj_proj0_index.push_back(3);
cj_proj0_index.push_back(4);
cj_proj0_index.push_back(5);
const int partition_key_index_1=2;
table_1->createHashPartitionedProjection(cj_proj0_index,"order_no",4); //G0
catalog->add_table(table_1);
////////////////////////////////////////
/* the following codes should be triggered by Load module*/
//////////////////ONE DAY////////////////////////////////////////////////
//cj_table
// 4 partitions partitioned by order_no
for(unsigned i=0;i<table_1->getProjectoin(0)->getPartitioner()->getNumberOfPartitions();i++){
catalog->getTable(0)->getProjectoin(0)->getPartitioner()->RegisterPartition(i,2);
}
/////////////////////////////////////////
ProjectionBinding *pb=new ProjectionBinding();
pb->BindingEntireProjection(catalog->getTable(0)->getProjectoin(0)->getPartitioner());
printf("ready(?)\n");
int input;
scanf("%d",&input);
vector<column_type> blc_column_list;
blc_column_list.push_back(column_type(t_u_long));
blc_column_list.push_back(column_type(t_int));
blc_column_list.push_back(column_type(t_u_long));
blc_column_list.push_back(column_type(t_int)); //sec_code for indexing
blc_column_list.push_back(column_type(t_int));
blc_column_list.push_back(column_type(t_int));
Schema* blc_schema = new SchemaFix(blc_column_list);
unsigned block_size = 64*1024;
bottomLayerCollecting::State blc_state(catalog->getTable(0)->getProjectoin(0)->getProjectionID(), blc_schema, 3, block_size);
// ExpandableBlockStreamIteratorBase* blc = new bottomLayerCollecting(blc_state);
PhysicalOperatorBase* blc = new bottomLayerCollecting(blc_state);
vector<column_type> bls_column_list;
bls_column_list.push_back(t_int); //chunk offset
bls_column_list.push_back(t_int); //sec_code
bls_column_list.push_back(t_u_smallInt); //chunk offset
bls_column_list.push_back(t_u_smallInt); //chunk offset
Schema* bls_schema = new SchemaFix(bls_column_list);
bottomLayerSorting::State bls_state(bls_schema, blc, block_size, catalog->getTable(0)->getProjectoin(0)->getProjectionID(), 3, "sec_code_index");
// ExpandableBlockStreamIteratorBase* bls = new bottomLayerSorting(bls_state);
PhysicalOperatorBase* bls = new bottomLayerSorting(bls_state);
bls->Open();
BlockStreamBase* block;
while(bls->Next(block))
{
}
bls->Close();
/********************************** CSB Plus Tree Index Building Finished! **********************************/
unsigned long index_id = 0;
vector<IndexScanIterator::query_range> q_range;
q_range.clear();
int value_low = 10107;
int value_high = 10110;
IndexScanIterator::query_range q1;
q1.value_low = malloc(sizeof(int)); //newmalloc
q1.value_low = (void*)(&value_low);
q1.comp_low = EQ;
q1.value_high = malloc(sizeof(int)); //newmalloc
q1.value_high = (void*)(&value_low);
q1.comp_high = EQ;
q1.c_type = t_int;
q_range.push_back(q1);
IndexScanIterator::query_range q2;
q2.value_low = malloc(sizeof(int)); //newmalloc
q2.value_low = (void*)(&value_high);
q2.comp_low = EQ;
q2.value_high = malloc(sizeof(int)); //newmalloc
q2.value_high = (void*)(&value_high);
q2.comp_high = EQ;
q2.c_type = t_int;
q_range.push_back(q2);
IndexScanIterator::State isi_state(catalog->getTable(0)->getProjectoin(0)->getProjectionID(), blc_schema, index_id, q_range, block_size);
PhysicalOperatorBase* isi = new IndexScanIterator(isi_state);
std::vector<std::string> attribute_name;
attribute_name.clear();
attribute_name.push_back("row_id");
attribute_name.push_back("trade_date");
attribute_name.push_back("order_no");
attribute_name.push_back("sec_code");
attribute_name.push_back("trade_dir");
attribute_name.push_back("order_type");
ResultPrinter::State bsp_state(blc_schema, isi, block_size, attribute_name, "\t");
PhysicalOperatorBase* bsp = new ResultPrinter(bsp_state);
bsp->Open();
while(bsp->Next(block))
{
}
bsp->Close();
}
cout<<"Waiting~~~!~"<<endl;
while(true){
sleep(1);
}
return 0;
}
void Analyzer::analyse(const AttributeID& attrID) {
Catalog* catalog = Catalog::getInstance();
TableDescriptor* table = catalog->getTable(attrID.table_id);
ProjectionDescriptor* projection = NULL;
unsigned pidSize = table->getNumberOfProjection();
const Attribute attr = table->getAttribute(attrID.offset);
for (unsigned i = 0; i < pidSize; ++i) {
if (table->getProjectoin(i)->hasAttribute(attr)) {
projection = table->getProjectoin(i);
break;
}
}
std::vector<Attribute> group_by_attributes;
std::vector<Attribute> aggregation_attributes;
group_by_attributes.push_back(attr);
aggregation_attributes.push_back(attr);
std::vector<PhysicalAggregation::State::Aggregation> aggregation_function;
aggregation_function.push_back(PhysicalAggregation::State::kCount);
LogicalOperator* sb_payload_scan = new LogicalScan(projection);
LogicalOperator* aggregation =
new LogicalAggregation(group_by_attributes, aggregation_attributes,
aggregation_function, sb_payload_scan);
const NodeID collector_node_id = 0;
LogicalOperator* root = new LogicalQueryPlanRoot(
collector_node_id, aggregation, LogicalQueryPlanRoot::kResultCollector);
PhysicalOperatorBase* collector =
root->GetPhysicalPlan(1024 * 64 - sizeof(unsigned));
collector->Open();
collector->Next(0);
collector->Close();
ResultSet* resultset = collector->GetResultSet();
ResultSet::Iterator it = resultset->createIterator();
BlockStreamBase* block;
void* tuple;
BlockStreamBase::BlockStreamTraverseIterator* block_it;
unsigned long valueCount = resultset->getNumberOftuples();
unsigned long tupleCount = 0;
TuplePtr* list = new TuplePtr[valueCount];
unsigned long i = 0;
while (block = (BlockStreamBase*)it.atomicNextBlock()) {
block_it = block->createIterator();
while (tuple = block_it->nextTuple()) {
list[i++] = tuple;
tupleCount += getFrequency(tuple, attr.attrType);
}
}
int magicNumber = 100;
StatisticOnTable* stat = new StatisticOnTable(magicNumber);
stat->setValueCount(valueCount);
stat->setTupleCount(tupleCount);
qsort_r(list, valueCount, sizeof(void*), compare,
(void*)(attr.attrType->operate));
mcvAnalyse(list, valueCount, attr, (Histogram*)stat);
equiDepthAnalyse(list, valueCount, attr, (Histogram*)stat);
// StatManager::getInstance()->addStat(attrID, stat);
StatManager::getInstance()->getTableStatistic(attrID.table_id);
delete list;
resultset->destory();
}
