/// Type-checked version of column_bytes.
///
/// \param name The name of the column to retrieve the size of.
///
/// \return The same as column_bytes if the value can be retrieved.
///
/// \throw error If the type of the cell to retrieve the size of is invalid.
/// \throw invalid_column_error If name is invalid.
int
sqlite::statement::safe_column_bytes(const char* name)
{
const int column = column_id(name);
if (column_type(column) != sqlite::type_blob &&
column_type(column) != sqlite::type_text)
throw sqlite::error(F("Column '%s' is not a blob or a string") % name);
return column_bytes(column);
}
/*******#include "../physical_operator/ExpandableBlockStreamSingleColumnScan.h"
*****Local Disk Access******#include "../physical_operator/ExpandableBlockStreamSingleColumnScanDisk.h"
******/
int main_ld (int argc, char** argv)
{
const unsigned block_size = atoi(argv[1]);
const unsigned thread_count = atoi(argv[2]);
const unsigned expander_buffer = atoi(argv[3]);
const int columns_num = atoi(argv[4]);
unsigned long random_size = atoi(argv[5]); //number of columns which are randomly accessed
random_size = random_size*sizeof(int);
unsigned long scan_size = (unsigned long)4*1024*1024*1024-random_size*columns_num;
cout << "columns_num: " << columns_num << "\t random_size: " << random_size/sizeof(int) << endl;
std::vector<column_type> column_list,column_list_;
column_list.push_back(column_type(t_int));
for (int i = 0; i < columns_num; i++)
column_list_.push_back(column_type(t_int));
Schema* i_schema = new SchemaFix(column_list);
Schema* d_schema = new SchemaFix(column_list_);
// ExpandableBlockStreamSingleColumnScan::State ebssc_state1("/home/claims/temp/6_0.column", d_schema, block_size);
// BlockStreamIteratorBase* ebssc1 = new ExpandableBlockStreamSingleColumnScan(ebssc_state1);
// ebssc1->open();
// BlockStreamBase* block_=BlockStreamBase::createBlock(i_schema,block_size);
// while (ebssc1->next(block_)) {};
// ebssc1->close();
// block_->~BlockStreamBase();
ExpandableBlockStreamSingleColumnScanDisk::State ebsscsd_state("/home/claims/temp/5_0.column", d_schema, block_size, scan_size);
PhysicalOperatorBase* ebsscsd = new ExpandableBlockStreamSingleColumnScanDisk(ebsscsd_state);
ExpandableBlockStreamSingleColumnScan::State ebssc_state("/home/claims/temp/6_0.column", i_schema, block_size, random_size);
PhysicalOperatorBase* ebssc = new ExpandableBlockStreamSingleColumnScan(ebssc_state);
// BlockStreamExpander::State bse_state(i_schema,ebssc,thread_count,block_size,expander_buffer);
// BlockStreamIteratorBase* bse=new BlockStreamExpander(bse_state);
ExpandableBlockStreamRandomDiskAccess::State ebsrda_state("/home/claims/temp/Uniform_0_9999.column", ebssc, d_schema, i_schema, block_size);
PhysicalOperatorBase* ebsrda = new ExpandableBlockStreamRandomDiskAccess(ebsrda_state);
Expander::State bse_state1(d_schema,ebsscsd,thread_count,block_size,expander_buffer);
PhysicalOperatorBase* bse1=new Expander(bse_state1);
Expander::State bse_state2(d_schema,ebsrda,thread_count,block_size,expander_buffer);
PhysicalOperatorBase* bse2=new Expander(bse_state2);
BlockStreamPerformanceTest::State bspt_state(d_schema, bse1, bse2, block_size, 1000);
PhysicalOperatorBase* bspt = new BlockStreamPerformanceTest(bspt_state);
bspt->Open();
while (bspt->Next(0));
bspt->Close();
return 0;
}
int mainasfasdf22(){
std::vector<column_type> column_list;
column_list.push_back(column_type(t_int));
Schema* input=new SchemaFix(column_list);
BlockStreamBase* bs=new BlockStreamFix(4096,4);
void* tuple;
int value=0;
int i=0;
while(tuple=bs->allocateTuple(4)){
*(int*)tuple=value++;
printf("insert %d\n",i++);
}
BlockStreamBase::BlockStreamTraverseIterator* bsti=bs->createIterator();
i=0;
while(tuple=bsti->nextTuple()){
printf("%d: Tuple=%d\n",i++,*(int*)tuple);
}
}
/// Returns a particular column in the result as a double.
///
/// \param index The column to retrieve.
///
/// \return A C string with the contents. Note that the pointer returned by
/// this call will be invalidated on the next call to any SQLite API function.
/// If you want to be extra safe, store the result in a std::string to not worry
/// about this.
std::string
sqlite::statement::column_text(const int index)
{
PRE(column_type(index) == type_text);
return reinterpret_cast< const char* >(::sqlite3_column_text(
_pimpl->stmt, index));
}
/// Returns a particular column in the result as a blob.
///
/// \param index The column to retrieve.
///
/// \return A block of memory with the blob contents. Note that the pointer
/// returned by this call will be invalidated on the next call to any SQLite API
/// function.
sqlite::blob
sqlite::statement::column_blob(const int index)
{
PRE(column_type(index) == type_blob);
return blob(::sqlite3_column_blob(_pimpl->stmt, index),
::sqlite3_column_bytes(_pimpl->stmt, index));
}
/// Type-checked version of column_double.
///
/// \param name The name of the column to retrieve.
///
/// \return The same as column_double if the value can be retrieved.
///
/// \throw error If the type of the cell to retrieve is invalid.
/// \throw invalid_column_error If name is invalid.
double
sqlite::statement::safe_column_double(const char* name)
{
const int column = column_id(name);
if (column_type(column) != sqlite::type_float)
throw sqlite::error(F("Column '%s' is not a float") % name);
return column_double(column);
}
row::column_type row::column(size_t nPosition) const
{
if (nPosition >= size()) {
throw no_such_column_exception();
}
return column_type(make_shared<sqlite::column>(stmt_, nPosition));
}
/// Type-checked version of column_int64.
///
/// \param name The name of the column to retrieve.
///
/// \return The same as column_int64 if the value can be retrieved.
///
/// \throw error If the type of the cell to retrieve is invalid.
/// \throw invalid_column_error If name is invalid.
int64_t
sqlite::statement::safe_column_int64(const char* name)
{
const int column = column_id(name);
if (column_type(column) != sqlite::type_integer)
throw sqlite::error(F("Column '%s' is not an integer") % name);
return column_int64(column);
}
row::column_type row::column(size_t nPosition) const
{
if (nPosition >= size() || row_ == -1) {
throw no_such_column_exception();
}
return column_type(make_shared<postgres::column>(stmt_, row_, nPosition));
}
/// Type-checked version of column_blob.
///
/// \param name The name of the column to retrieve.
///
/// \return The same as column_blob if the value can be retrieved.
///
/// \throw error If the type of the cell to retrieve is invalid.
/// \throw invalid_column_error If name is invalid.
sqlite::blob
sqlite::statement::safe_column_blob(const char* name)
{
const int column = column_id(name);
if (column_type(column) != sqlite::type_blob)
throw sqlite::error(F("Column '%s' is not a blob") % name);
return column_blob(column);
}
/// Type-checked version of column_text.
///
/// \param name The name of the column to retrieve.
///
/// \return The same as column_text if the value can be retrieved.
///
/// \throw error If the type of the cell to retrieve is invalid.
/// \throw invalid_column_error If name is invalid.
std::string
sqlite::statement::safe_column_text(const char* name)
{
const int column = column_id(name);
if (column_type(column) != sqlite::type_text)
throw sqlite::error(_pimpl->db.db_filename(),
F("Column '%s' is not a string") % name);
return column_text(column);
}
/// Type-checked version of column_int.
///
/// \param name The name of the column to retrieve.
///
/// \return The same as column_int if the value can be retrieved.
///
/// \throw error If the type of the cell to retrieve is invalid.
/// \throw invalid_column_error If name is invalid.
int
sqlite::statement::safe_column_int(const char* name)
{
const int column = column_id(name);
if (column_type(column) != sqlite::type_integer)
throw sqlite::error(_pimpl->db.db_filename(),
F("Column '%s' is not an integer") % name);
return column_int(column);
}
int mainasdfaf234(int argc,const char** argv){
std::vector<column_type> column_list,column_list_;
column_list.push_back(column_type(t_int));
Schema* input=new SchemaFix(column_list);
BlockStreamSingleColumnScan::State bsscs_state("/home/claims/temp/Uniform_0_99.column",input);
PhysicalOperatorBase* bsscs1=new BlockStreamSingleColumnScan(bsscs_state);
PhysicalOperatorBase* bsscs2=new BlockStreamSingleColumnScan(bsscs_state);
int f=atoi(argv[2]);
AttributeComparator fA(column_type(t_int),Comparator::L,0,&f);
std::vector<AttributeComparator> ComparatorList;
ComparatorList.push_back(fA);
BlockStreamFilter::State bsf_state(input,bsscs1,ComparatorList,4096);
PhysicalOperatorBase* bsf=new BlockStreamFilter(bsf_state);
//
BlockStreamBase *block=new BlockStreamFix(4096,4);
int choice=0;
while(choice==0){
// bsf->open();
bsf->Open();
unsigned long long int start=curtick();
while(bsf->Next(block)){
block->setEmpty();
}
printf("Time=%f Throughput=%f.\n",getSecond(start),1024/getSecond(start));
bsf->Close();
printf("Continue(0) or Not(1) ?\n");
scanf("%d",&choice);
}
}
bool
cql::cql_result_metadata_t::column_type(const std::string& column,
cql::cql_column_type_enum& output) const
{
if (_global_keyspace_name.empty() || _global_table_name.empty()) {
return false;
}
return column_type(_global_keyspace_name, _global_table_name, column, output);
}
static void produce_column_header_csv(void *csv_metadata, const char *column, readstat_variable_t* var) {
struct csv_metadata *c = (struct csv_metadata *)csv_metadata;
metadata_column_type_t coltype = column_type(c->json_md, column, c->output_format);
if (coltype == METADATA_COLUMN_TYPE_DATE) {
var->type = READSTAT_TYPE_STRING;
} else if (coltype == METADATA_COLUMN_TYPE_NUMERIC) {
var->type = READSTAT_TYPE_DOUBLE;
} else if (coltype == METADATA_COLUMN_TYPE_STRING) {
var->type = READSTAT_TYPE_STRING;
}
}
void produce_column_header_sav(void *csv_metadata, const char *column, readstat_variable_t* var) {
struct csv_metadata *c = (struct csv_metadata *)csv_metadata;
metadata_column_type_t coltype = column_type(c->json_md, column, c->output_format);
if (coltype == METADATA_COLUMN_TYPE_DATE) {
var->type = READSTAT_TYPE_DOUBLE;
snprintf(var->format, sizeof(var->format), "%s", "EDATE40");
} else if (coltype == METADATA_COLUMN_TYPE_NUMERIC) {
var->type = READSTAT_TYPE_DOUBLE;
snprintf(var->format, sizeof(var->format), "F8.%d", get_decimals(c->json_md, column));
} else if (coltype == METADATA_COLUMN_TYPE_STRING) {
var->type = READSTAT_TYPE_STRING;
}
}
bool
cql::cql_result_metadata_t::column_type(const std::string& keyspace,
const std::string& table,
const std::string& column,
cql::cql_column_type_enum& output) const
{
column_name_idx_t::const_iterator it = _column_name_idx.find(column_name_t(keyspace, table, column));
if(it != _column_name_idx.end()) {
column_type(it->second, output);
return true;
}
return false;
}
ColumnType Statement::GetDeclaredColumnType(int col) const
{
std::string column_type(sqlite3_column_decltype(GetStatement(), col));
std::transform(column_type.begin(), column_type.end(), column_type.begin(), tolower);
if (column_type == "integer")
return COLUMN_TYPE_INTEGER;
else if (column_type == "float")
return COLUMN_TYPE_FLOAT;
else if (column_type == "text")
return COLUMN_TYPE_TEXT;
else if (column_type == "blob")
return COLUMN_TYPE_BLOB;
return COLUMN_TYPE_NULL;
}
void column_type_udf(sqlite3_context* ctx, int nargs, sqlite3_value** values)
{
sqlite3 *db;
const char *table, *column, *type;
db=(sqlite3*)sqlite3_user_data(ctx);
table = sqlite3_value_text(values[0]);
column = sqlite3_value_text(values[1]);
/* Get declared type from schema */
type = column_type(db, table, column);
/* Return type */
sqlite3_result_text(ctx, type, (int)strlen(type), free);
}
int simple_stable::insert(const dtype & key, const istr & column, const dtype & value, bool append)
{
int r;
bool increment = !ct_data->find(key, column).exists();
if(increment)
{
/* this will check that the type matches */
r = adjust_column(column, 1, value.type);
if(r < 0)
return r;
}
else if(column_type(column) != value.type)
return -EINVAL;
r = ct_data->insert(key, column, value.flatten(), append);
if(r < 0 && increment)
adjust_column(column, -1, value.type);
return r;
}
int install_type_trigger( sqlite3* db, sqlite3_context* ctx,
const char* table, const char* column )
{
int rc;
char buf[256];
char *tmp;
const char *type, *sql, *emsg;
char* err;
type = column_type(db, table, column);
if(type == NULL) {
emsg = "column has no declared type";
sqlite3_result_error(ctx, emsg, (int)strlen(emsg));
sqlite3_free((void*)type);
return 1;
}
/* Check to see if corresponding validation function exists */
sql = "select validate_%s(null)";
tmp = sqlite3_mprintf(sql, type);
rc = sqlite3_exec(db, tmp, NULL, NULL, &err);
sqlite3_free(tmp);
if(rc != SQLITE_OK && err != NULL) {
emsg = "No validator exists for column type";
sqlite3_result_error(ctx, emsg, (int)strlen(emsg));
sqlite3_free((void*)type);
sqlite3_free(err);
return 1;
}
/* Create INSERT trigger */
sql = "CREATE TRIGGER %s_insert_%s_typecheck_tr \n"
"BEFORE INSERT ON %s \n"
"BEGIN \n"
" SELECT CASE \n"
" WHEN(SELECT validate_%s(new.%s) != 1) \n"
" THEN RAISE(ABORT, 'invalid %s value for %s.%s') \n"
" END; \n"
"END;";
tmp = sqlite3_mprintf(sql, table, column, table, type,
column, type, table, column);
rc = sqlite3_exec(db, tmp, NULL, NULL, &err);
sqlite3_free(tmp);
if(rc != SQLITE_OK && err != NULL) {
strncpy(&buf[0], err, 255);
buf[256] = '\0';
sqlite3_result_error(ctx, &buf[0], (int)strlen(&buf[0]));
sqlite3_free((void*)type);
return 1;
}
/* Create UPDATE trigger */
sql = "CREATE TRIGGER %s_update_%s_typecheck_tr \n"
"BEFORE UPDATE OF %s ON %s \n"
"FOR EACH ROW BEGIN \n"
" SELECT CASE \n"
" WHEN(SELECT validate_%s(new.%s) != 1) \n"
" THEN RAISE(ABORT, 'invalid %s value for %s.%s') \n"
" END; \n"
"END;";
tmp = sqlite3_mprintf(sql, table, column, column, table,
type, column, type, table, column);
rc = sqlite3_exec(db, tmp, NULL, NULL, &err);
sqlite3_free(tmp);
sqlite3_free((void*)type);
if(rc != SQLITE_OK && err != NULL) {
strncpy(&buf[0], err, 255);
buf[256] = '\0';
sqlite3_result_error(ctx, &buf[0], (int)strlen(&buf[0]));
sqlite3_free(err);
return 1;
}
return 0;
}
//int main(int argc,const char** argv){
int main_combine(int argc,const char** argv){
const unsigned block_size=BLOCK_SIZE_CAO;
const unsigned thread_count=4;
const unsigned expander_buffer=4;
std::vector<column_type> column_list,column_list_;
column_list.push_back(column_type(t_int));
Schema* schema=new SchemaFix(column_list);
ExpandableBlockStreamSingleColumnScan::State ebssc_state1("/home/imdb/temp/Uniform_0_99.column",schema,block_size);
BlockStreamIteratorBase* ebssc1=new ExpandableBlockStreamSingleColumnScan(ebssc_state1);
ExpandableBlockStreamSingleColumnScan::State ebssc_state2("/home/imdb/temp/Uniform_0_99.column",schema,block_size);
BlockStreamIteratorBase* ebssc2=new ExpandableBlockStreamSingleColumnScan(ebssc_state2);
std::vector<Schema *> inputs;
inputs.push_back(schema);
inputs.push_back(schema);
column_list_.push_back(column_type(t_int));
column_list_.push_back(column_type(t_int));
Schema* output=new SchemaFix(column_list_);
std::vector<BlockStreamIteratorBase *> children_;
children_.push_back(ebssc1);
children_.push_back(ebssc2);
BlockStreamCombinedIterator::State bsci_state(inputs,output,children_);
BlockStreamCombinedIterator *bsc=new BlockStreamCombinedIterator(bsci_state);
BlockStreamExpander::State bse_state(schema,bsc,thread_count,block_size,expander_buffer);
BlockStreamIteratorBase* bse=new BlockStreamExpander(bse_state);
BlockStreamBase *block=new BlockStreamFix(block_size,8);
int choice=0;
std::ostringstream ostr;
boost::archive::text_oarchive oa(ostr);
oa.register_type(static_cast<BlockStreamCombinedIterator *>(NULL));
oa.register_type(static_cast<BlockStreamExpander *>(NULL));
oa.register_type(static_cast<ExpandableBlockStreamSingleColumnScan *>(NULL));
Register_Schemas<boost::archive::text_oarchive>(oa);
// Register_Iterators(oa);
oa<<bse;
std::cout<<"Serialization Result:"<<ostr.str()<<std::endl;
std::istringstream istr(ostr.str());
boost::archive::text_iarchive ia(istr);
BlockStreamIteratorBase* des;
ia.register_type(static_cast<BlockStreamCombinedIterator *>(NULL));
ia.register_type(static_cast<BlockStreamExpander *>(NULL));
ia.register_type(static_cast<ExpandableBlockStreamSingleColumnScan *>(NULL));
Register_Schemas<boost::archive::text_iarchive>(ia);
ia>>des;
// return 1;
while(choice==0){
// bsf->open();
des->open();
cout<<"after open!"<<endl;
unsigned long long int start=curtick();
cout<<"ready for the next"<<endl;
unsigned tuple_count=0;
while(des->next(block)){
BlockStreamBase::BlockStreamTraverseIterator* it=block->createIterator();
while(it->nextTuple()){
tuple_count++;
}
block->setEmpty();
}
printf("Time=%f Throughput=%f.\n tuple=%d",getSecond(start),1024/getSecond(start),tuple_count);
des->close();
printf("Continue(0) or Not(1) ?\n");
scanf("%d",&choice);
}
}
int main_asdfasdf(){
Environment::getInstance(true);
ResourceManagerMaster *rmms=Environment::getInstance()->getResourceManagerMaster();
Catalog* catalog=Environment::getInstance()->getCatalog();
// rmms->RegisterNewSlave("192.168.1.1");
// rmms->RegisterNewSlave("192.168.1.2");
// rmms->RegisterNewSlave("192.168.1.3");
// rmms->RegisterNewSlave("192.168.1.4");
// rmms->RegisterNewSlave("192.168.1.5");
// rmms->RegisterDiskBuget(0,10000);
// rmms->RegisterDiskBuget(1,10000);
// rmms->RegisterDiskBuget(2,10000);
// rmms->RegisterDiskBuget(3,10000);
// rmms->RegisterDiskBuget(4,10000);
/////////////////////////////////////Create table left/////////////////////
TableDescriptor* table_1=new TableDescriptor("Left",Environment::getInstance()->getCatalog()->allocate_unique_table_id());
table_1->addAttribute("Name",data_type(t_string),3);
table_1->addAttribute("Age",data_type(t_int));
table_1->addAttribute("Gender",data_type(t_int));
table_1->addAttribute("Score",data_type(t_int));
vector<ColumnOffset> index_1;
index_1.push_back(0);
index_1.push_back(1);
index_1.push_back(3);
const int partition_key_index_1=3;
table_1->createHashPartitionedProjection(index_1,partition_key_index_1,3);
catalog->add_table(table_1);
////////////////////////////////////Create table right//////////////////////////
TableDescriptor* table_2=new TableDescriptor("right",Environment::getInstance()->getCatalog()->allocate_unique_table_id());
table_2->addAttribute("Name",data_type(t_string),10);
table_2->addAttribute("Age",data_type(t_int));
table_2->addAttribute("Gender",data_type(t_int));
table_2->addAttribute("Score",data_type(t_int));
vector<ColumnOffset> index_2;
index_2.push_back(0);
index_2.push_back(1);
index_2.push_back(3);
const int partition_key_index_2=3;
table_2->createHashPartitionedProjection(index_2,partition_key_index_2,3);
catalog->add_table(table_2);
///////////////////////////////////////////////////////////
///////////////////////////////////////
////////////////////////////////////////
/* the following codes should be triggered by Load module*/
for(unsigned i=0;i<table_1->getProjectoin(0)->getPartitioner()->getNumberOfPartitions();i++){
catalog->getTable(0)->getProjectoin(0)->getPartitioner()->RegisterPartition(i,5);
}
for(unsigned i=0;i<table_2->getProjectoin(0)->getPartitioner()->getNumberOfPartitions();i++){
catalog->getTable(1)->getProjectoin(0)->getPartitioner()->RegisterPartition(i,4);
}
////////////////////////////////////////
ProjectionBinding *pb=new ProjectionBinding();
pb->BindingEntireProjection(catalog->getTable(0)->getProjectoin(0)->getPartitioner());
pb->BindingEntireProjection(catalog->getTable(1)->getProjectoin(0)->getPartitioner());
////scan////////
std::vector<unsigned> index_list_1;
index_list_1.push_back(0);
index_list_1.push_back(1);
index_list_1.push_back(3);
LogicalOperator* scan_1=new LogicalScan(table_1->getAttributes(index_list_1));
////////filter////////
int f=0;
FilterIterator::AttributeComparator filter1(column_type(t_int),Comparator::EQ,2,&f);
std::vector<FilterIterator::AttributeComparator> ComparatorList;
ComparatorList.push_back(filter1);
LogicalOperator* filter=new Filter(ComparatorList,scan_1);
////scan/////////
std::vector<unsigned> index_list_2;
index_list_2.push_back(0);
index_list_2.push_back(1);
index_list_2.push_back(3);
LogicalOperator* scan_2=new LogicalScan(table_2->getAttributes(index_list_2));
//////////////////
////Join////////
EqualJoin::JoinPair joinpair(table_1->getAttribute(3),table_2->getAttribute(3));
std::vector<EqualJoin::JoinPair> pair_list;
pair_list.push_back(joinpair);
LogicalOperator* join=new EqualJoin(pair_list,filter,scan_2);
Dataflow final_dataflow=join->getDataflow();
printf("Total communication cost: %d\n",final_dataflow.property_.commnication_cost);
printf("Waiting~\n");
while(true){
sleep(1);
}
}
bool Sqlite3Table::select(sqlite3 * db,
QueryData & inQuerydata,
std::vector<QueryData> & outQuerydata) const
{
// exit_on_error(sqlite3_step(statement), __LINE__);
std::vector<std::string> all_columns (extract_column_names(inQuerydata));
std::stringstream ss;
ss << "SELECT * FROM " + mName;
if(!all_columns.empty())
{
ss << + " WHERE ";
}
bool first_add(true);
for(auto it(all_columns.begin()); it != all_columns.end(); it++)
{
if(!first_add)
{
ss << " AND ";
first_add = false;
}
ss << *it << " = @" << *it;
}
ss << ";";
// Prepare the statement
sqlite3_stmt * statement;
bool _success(true);
_success = (_success && checkIfSqlError(sqlite3_prepare_v2(db, ss.str().c_str(),-1/*null-terminated*/,&statement,NULL), __FILE__, __LINE__));
// PERFORM BINDING
for(auto it(inQuerydata.mStringColumns.begin()); it != inQuerydata.mStringColumns.end(); it++)
_success = (_success && checkIfSqlError(mColumns.find(it->first)->second->bind(statement, &it->second), __FILE__, __LINE__));
for(auto it(inQuerydata.mIntColumns.begin()); it != inQuerydata.mIntColumns.end(); it++)
_success = (_success && checkIfSqlError(mColumns.find(it->first)->second->bind(statement, &it->second), __FILE__, __LINE__));
for(auto it(inQuerydata.mDoubleColumns.begin()); it != inQuerydata.mDoubleColumns.end(); it++)
_success = (_success && checkIfSqlError(mColumns.find(it->first)->second->bind(statement, &it->second), __FILE__, __LINE__));
for(auto it(inQuerydata.mNullColumns.begin()); it != inQuerydata.mNullColumns.end(); it++)
_success = (_success && checkIfSqlError(mColumns.find(*it)->second->bind(statement, nullptr), __FILE__, __LINE__));
// Fill return data
int _columnCount(sqlite3_column_count(statement));
while(sqlite3_step(statement) == SQLITE_ROW)
{
QueryData row_result;
for(int c (0); c < _columnCount; c++)
{
std::string column_name(sqlite3_column_name(statement, c));
Sqlite3Column * column(mColumns.find(column_name)->second.get());
Sqlite3Type column_type(column->getType());
if(column_type == Sqlite3Null::_NAME)
{
row_result.mNullColumns.push_back(column_name);
}
else if(column_type == Sqlite3Integer::_NAME)
{
int * v = new int;
column->value(statement, c, v);
row_result.mIntColumns.emplace(column_name, *v);
delete v;
}
else if(column_type == Sqlite3Real::_NAME)
{
double * v = new double;
column->value(statement, c, v);
row_result.mDoubleColumns.emplace(column_name, *v);
delete v;
}
else if(column_type == Sqlite3Text::_NAME)
{
std::string * v = new std::string;
column->value(statement, c, v);
row_result.mStringColumns.emplace(column_name, *v);
delete v;
}
else
{
_success = false;
std::cerr << "Unimplemented type!" << std::endl;
}
}
outQuerydata.push_back(row_result);
}
sqlite3_finalize(statement);
return _success;
}
void bit_vector_nearest_neighbor_base::fill_schema(
vector<column_type>& schema) {
bit_vector_column_id_ = schema.size();
schema.push_back(column_type(column_type::bit_vector_type, bitnum_));
}
int maincdcdajij(int argc,char* argv[])
{
///////////////////////////////////////////Scan////////////////////////////////////////|
std::vector<column_type> column_list;
column_list.push_back(column_type(t_int));
Schema* input=new SchemaFix(column_list);
Schema* output=new SchemaFix(column_list);
SingleColumnScanIterator::State SCstate1("/home/imdb/temp/1_0.column",input, output);
Iterator* scs1=new SingleColumnScanIterator(SCstate1);
///////////////////////////////////////////Exchange////////////////////////////////////|
printf("argc: %d\n", argc);
int block_size;
if(argc==2)
{
block_size=atoi(argv[1]);
}
else
{
block_size=64;
}
std::vector<std::string> uppernode;
// std::vector<std::pair<std::string,std::string> > lowernode;
std::string p1("10.11.1.204");
std::string p2("10.11.1.205");
// pair<std::string,std::string> p3("10.11.1.206","4242");
// pair<std::string,std::string> p4("10.11.1.207","4242");
// pair<std::string,std::string> p5("10.11.1.208","4242");
uppernode.push_back(p1);
uppernode.push_back(p2);
// lowernode.push_back(p3);
// lowernode.push_back(p4);
// lowernode.push_back(p5);
// ExchangeIteratorWithWideDependency::State EIstate(input, scs1,block_size,lowernode,uppernode);
ExchangeIteratorLowerWithWideDependency::State EIstate(input, scs1,uppernode,block_size);
Iterator* ei=new ExchangeIteratorLowerWithWideDependency(EIstate);
///////////////////////////////////////Print///////////////////////////////////////////|
PrintIterator::State PIstate(input,ei);
Iterator *pi=new PrintIterator(PIstate);
//-------------------------------------------------------------------------------------|
std::ostringstream ostr;
boost::archive::text_oarchive oa(ostr);
// // oa.register_type(static_cast<SingleColumnScanIterator *>(NULL));
// Register_Schemas<boost::archive::text_oarchive>(oa);
// Register_Iterators(oa);
// oa<<pi;
// std::cout<<"Serialization Result:"<<ostr.str()<<std::endl;
IteratorMessage IM(pi);
oa<<IM;
cout<<"serialized: "<<ostr.str()<<endl;
return 1;
}
static int testSort(){
int master;
printf("Master(0) or Slave(others)??\n");
scanf("%d",&master);
if(master!=0){
Environment::getInstance(false);
}
else{
Environment::getInstance(true);
ResourceManagerMaster *rmms=Environment::getInstance()->getResourceManagerMaster();
Catalog* catalog=Environment::getInstance()->getCatalog();
TableDescriptor* table_1=new TableDescriptor("cj",Environment::getInstance()->getCatalog()->allocate_unique_table_id());
/**
* @li: I change the following code such that the scheme matches those in cj's first projection
*/
table_1->addAttribute("row_id",data_type(t_u_long)); //0
table_1->addAttribute("trade_date",data_type(t_int));
table_1->addAttribute("order_no",data_type(t_u_long));
table_1->addAttribute("sec_code",data_type(t_int));
table_1->addAttribute("trade_dir",data_type(t_int));
table_1->addAttribute("order_type",data_type(t_int));
vector<ColumnOffset> cj_proj0;
cj_proj0.push_back(0);
cj_proj0.push_back(1);
cj_proj0.push_back(2);
cj_proj0.push_back(3);
cj_proj0.push_back(4);
cj_proj0.push_back(5);
const int partition_key_index_1=0;
table_1->createHashPartitionedProjection(cj_proj0,"row_id",1); //G0
catalog->add_table(table_1);
for(unsigned i=0;i<table_1->getProjectoin(0)->getPartitioner()->getNumberOfPartitions();i++){
catalog->getTable(0)->getProjectoin(0)->getPartitioner()->RegisterPartition(i,1);
}
LogicalOperator* scan=new LogicalScan(table_1->getProjectoin(0));
Filter::Condition filter_condition_1;
filter_condition_1.add(table_1->getAttribute("row_id"),AttributeComparator::L,std::string("100000"));
LogicalOperator* filter_1=new Filter(filter_condition_1,scan);
vector<LogicalSort::OrderByAttr *> oas;
LogicalSort::OrderByAttr *od0=new LogicalSort::OrderByAttr("cj","row_id");
LogicalSort::OrderByAttr *od1=new LogicalSort::OrderByAttr("cj","order_no");
LogicalSort::OrderByAttr *od2=new LogicalSort::OrderByAttr("cj","sec_code");
oas.push_back(od2);
oas.push_back(od1);
oas.push_back(od0);
LogicalOperator* sort=new LogicalSort(scan,oas);
BlockStreamIteratorBase *sort_=sort->getIteratorTree(64*1024-sizeof(unsigned));
BlockStreamPrint::State print_state;
print_state.block_size_=64*1024-sizeof(unsigned);
print_state.child_=sort_;
vector<column_type> column_list;
column_list.push_back(column_type(t_u_long));
column_list.push_back(column_type(t_int));
column_list.push_back(column_type(t_u_long));
column_list.push_back(column_type(t_int));
column_list.push_back(column_type(t_int));
column_list.push_back(column_type(t_int));
print_state.schema_=new SchemaFix(column_list);
print_state.spliter_="-|-";
BlockStreamIteratorBase* print=new BlockStreamPrint(print_state);
print->open();
print->next(0);
print->close();
// executable_query_plan->open();
// executable_query_plan->next(0);
// executable_query_plan->close();
}
cout<<"Waiting~~~!~"<<endl;
while(true){
sleep(1);
}
return 0;
}
static int test_index_scan_iterator()
{
int master;
printf("~!OKOKO!!!!!\n");
printf("Master(0) or Slave(others)??\n");
scanf("%d",&master);
if(master!=0){
Environment::getInstance(false);
}
else{
Environment::getInstance(true);
ResourceManagerMaster *rmms=Environment::getInstance()->getResourceManagerMaster();
Catalog* catalog=Environment::getInstance()->getCatalog();
TableDescriptor* table_1=new TableDescriptor("cj",Environment::getInstance()->getCatalog()->allocate_unique_table_id());
table_1->addAttribute("row_id",data_type(t_u_long)); //0
table_1->addAttribute("trade_date",data_type(t_int));
table_1->addAttribute("order_no",data_type(t_u_long));
table_1->addAttribute("sec_code",data_type(t_int));
table_1->addAttribute("trade_dir",data_type(t_int));
table_1->addAttribute("order_type",data_type(t_int)); //5
vector<ColumnOffset> cj_proj0_index;
cj_proj0_index.push_back(0);
cj_proj0_index.push_back(1);
cj_proj0_index.push_back(2);
cj_proj0_index.push_back(3);
cj_proj0_index.push_back(4);
cj_proj0_index.push_back(5);
const int partition_key_index_1=2;
table_1->createHashPartitionedProjection(cj_proj0_index,"order_no",4); //G0
catalog->add_table(table_1);
////////////////////////////////////////
/* the following codes should be triggered by Load module*/
//////////////////ONE DAY////////////////////////////////////////////////
//cj_table
// 4 partitions partitioned by order_no
for(unsigned i=0;i<table_1->getProjectoin(0)->getPartitioner()->getNumberOfPartitions();i++){
catalog->getTable(0)->getProjectoin(0)->getPartitioner()->RegisterPartition(i,2);
}
/////////////////////////////////////////
ProjectionBinding *pb=new ProjectionBinding();
pb->BindingEntireProjection(catalog->getTable(0)->getProjectoin(0)->getPartitioner());
printf("ready(?)\n");
int input;
scanf("%d",&input);
vector<column_type> blc_column_list;
blc_column_list.push_back(column_type(t_u_long));
blc_column_list.push_back(column_type(t_int));
blc_column_list.push_back(column_type(t_u_long));
blc_column_list.push_back(column_type(t_int)); //sec_code for indexing
blc_column_list.push_back(column_type(t_int));
blc_column_list.push_back(column_type(t_int));
Schema* blc_schema = new SchemaFix(blc_column_list);
unsigned block_size = 64*1024;
bottomLayerCollecting::State blc_state(catalog->getTable(0)->getProjectoin(0)->getProjectionID(), blc_schema, 3, block_size);
// ExpandableBlockStreamIteratorBase* blc = new bottomLayerCollecting(blc_state);
PhysicalOperatorBase* blc = new bottomLayerCollecting(blc_state);
vector<column_type> bls_column_list;
bls_column_list.push_back(t_int); //chunk offset
bls_column_list.push_back(t_int); //sec_code
bls_column_list.push_back(t_u_smallInt); //chunk offset
bls_column_list.push_back(t_u_smallInt); //chunk offset
Schema* bls_schema = new SchemaFix(bls_column_list);
bottomLayerSorting::State bls_state(bls_schema, blc, block_size, catalog->getTable(0)->getProjectoin(0)->getProjectionID(), 3, "sec_code_index");
// ExpandableBlockStreamIteratorBase* bls = new bottomLayerSorting(bls_state);
PhysicalOperatorBase* bls = new bottomLayerSorting(bls_state);
bls->Open();
BlockStreamBase* block;
while(bls->Next(block))
{
}
bls->Close();
/********************************** CSB Plus Tree Index Building Finished! **********************************/
unsigned long index_id = 0;
vector<IndexScanIterator::query_range> q_range;
q_range.clear();
int value_low = 10107;
int value_high = 10110;
IndexScanIterator::query_range q1;
q1.value_low = malloc(sizeof(int)); //newmalloc
q1.value_low = (void*)(&value_low);
q1.comp_low = EQ;
q1.value_high = malloc(sizeof(int)); //newmalloc
q1.value_high = (void*)(&value_low);
q1.comp_high = EQ;
q1.c_type = t_int;
q_range.push_back(q1);
IndexScanIterator::query_range q2;
q2.value_low = malloc(sizeof(int)); //newmalloc
q2.value_low = (void*)(&value_high);
q2.comp_low = EQ;
q2.value_high = malloc(sizeof(int)); //newmalloc
q2.value_high = (void*)(&value_high);
q2.comp_high = EQ;
q2.c_type = t_int;
q_range.push_back(q2);
IndexScanIterator::State isi_state(catalog->getTable(0)->getProjectoin(0)->getProjectionID(), blc_schema, index_id, q_range, block_size);
PhysicalOperatorBase* isi = new IndexScanIterator(isi_state);
std::vector<std::string> attribute_name;
attribute_name.clear();
attribute_name.push_back("row_id");
attribute_name.push_back("trade_date");
attribute_name.push_back("order_no");
attribute_name.push_back("sec_code");
attribute_name.push_back("trade_dir");
attribute_name.push_back("order_type");
ResultPrinter::State bsp_state(blc_schema, isi, block_size, attribute_name, "\t");
PhysicalOperatorBase* bsp = new ResultPrinter(bsp_state);
bsp->Open();
while(bsp->Next(block))
{
}
bsp->Close();
}
cout<<"Waiting~~~!~"<<endl;
while(true){
sleep(1);
}
return 0;
}
void euclid_lsh::fill_schema(vector<column_type>& schema) {
first_column_id_ = schema.size();
schema.push_back(column_type(column_type::bit_vector_type, hash_num_));
schema.push_back(column_type(column_type::float_type));
}
//int main_join_success(int argc, const char** argv){
int main_join_success22(int argc, const char** argv){
int master;
printf("Master(0) or Slave(others)?\n");
scanf("%d",&master);
if(master==0){
Environment::getInstance(true);
// const unsigned block_size=atoi(argv[1]);
// const unsigned thread_count=atoi(argv[2]);
// const unsigned expander_buffer=atoi(argv[3]);
// std::vector<std::string> upper_ip_list;
// const unsigned lowers_1=atoi(argv[4]);
// const unsigned lowers_2=atoi(argv[5]);
const unsigned block_size=4096;
const unsigned thread_count=3;
const unsigned expander_buffer=3;
std::vector<std::string> upper_ip_list;
const unsigned lowers_1=3;
const unsigned lowers_2=3;
upper_ip_list.push_back("10.11.1.208");
std::vector<std::string> lower_ip_list;
// lower_ip_list.push_back("10.11.1.201");
lower_ip_list.push_back("10.11.1.202");
lower_ip_list.push_back("10.11.1.203");
lower_ip_list.push_back("10.11.1.204");
lower_ip_list.push_back("10.11.1.205");
lower_ip_list.push_back("10.11.1.206");
lower_ip_list.push_back("10.11.1.207");
// lower_ip_list.push_back("10.11.1.208");
// lower_ip_list.push_back("10.11.1.209");
// lower_ip_list.push_back("10.11.1.210");
// lower_ip_list.push_back("10.11.1.211");
// lower_ip_list.push_back("10.11.1.214");
// std::vector<std::string> used_lowers;
// for(unsigned i=0;i<lowers;i++){
// used_lowers.push_back(lower_ip_list[i]);
// }
std::vector<std::string> used_lowers_1;
for(unsigned i=0;i<lowers_1;i++){
used_lowers_1.push_back(lower_ip_list[i]);
}
std::vector<std::string> used_lowers_2;
for(unsigned i=lower_ip_list.size()-1;i>=lower_ip_list.size()-lowers_2;i--){
used_lowers_2.push_back(lower_ip_list[i]);
}
std::vector<column_type> column_list,column_list_;
column_list.push_back(column_type(t_int));
Schema* schema=new SchemaFix(column_list);
// ExpandableBlockStreamSingleColumnScan::State ebssc_state("/home/imdb/temp/Uniform_0_99.column",schema,block_size);
// BlockStreamIteratorBase* ebssc=new ExpandableBlockStreamSingleColumnScan(ebssc_state);
ExpandableBlockStreamSingleColumnScan::State ebssc_state1("/home/imdb/temp/zhanglei/join_2",schema,block_size);
BlockStreamIteratorBase* ebssc1=new ExpandableBlockStreamSingleColumnScan(ebssc_state1);
// ExpandableBlockStreamSingleColumnScan::State ebssc_state2("/home/imdb/temp/zhanglei/Uniform_0_99.column_10000",schema,block_size);
// BlockStreamIteratorBase* ebssc2=new ExpandableBlockStreamSingleColumnScan(ebssc_state2);
ExpandableBlockStreamSingleColumnScan::State ebssc_state3("/home/imdb/temp/zhanglei/join_2_cp",schema,block_size);
BlockStreamIteratorBase* ebssc3=new ExpandableBlockStreamSingleColumnScan(ebssc_state3);
// ExpandableBlockStreamSingleColumnScan::State ebssc_state4("/home/imdb/temp/zhanglei/Uniform_0_99.column_10000_copy",schema,block_size);
// BlockStreamIteratorBase* ebssc4=new ExpandableBlockStreamSingleColumnScan(ebssc_state4);
// FilterIterator::AttributeComparator filter1(column_type(t_int),Comparator::L,0,&f);
// std::vector<FilterIterator::AttributeComparator> ComparatorList;
// ComparatorList.push_back(filter1);
// ExpandableBlockStreamFilter::State ebsf_state(schema,ebssc,ComparatorList,block_size);
// BlockStreamIteratorBase* bbsf=new ExpandableBlockStreamFilter(ebsf_state);
std::vector<Schema *> inputs;
inputs.push_back(schema);
inputs.push_back(schema);
column_list_.push_back(column_type(t_int));
column_list_.push_back(column_type(t_int));
Schema* output=new SchemaFix(column_list_);
std::vector<BlockStreamIteratorBase *> children_;
children_.push_back(ebssc1);
children_.push_back(ebssc3);
////////////////////////////////////////combined\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
BlockStreamCombinedIterator::State bsci_state1(inputs,output,children_);
BlockStreamCombinedIterator *bsc1=new BlockStreamCombinedIterator(bsci_state1);
BlockStreamExpander::State bse_state(output,bsc1,thread_count,block_size,expander_buffer);
BlockStreamIteratorBase* bse=new BlockStreamExpander(bse_state);
const int exchange_id=1;
ExpandableBlockStreamBroadcastExchange::State ebse_state(output,bse,block_size,used_lowers_2,used_lowers_1,exchange_id);
BlockStreamIteratorBase* ebse=new ExpandableBlockStreamBroadcastExchange(ebse_state);
BlockStreamCombinedIterator::State bsci_state2(inputs,output,children_);
BlockStreamCombinedIterator *bsc2=new BlockStreamCombinedIterator(bsci_state2);
std::vector<column_type> column_list_join;
column_list_join.push_back(column_type(t_int));
column_list_join.push_back(column_type(t_int));
column_list_join.push_back(column_type(t_int));
Schema* output_join=new SchemaFix(column_list_join);
std::vector<unsigned> joinIndex_left;
joinIndex_left.push_back(0);
std::vector<unsigned> joinIndex_right;
joinIndex_right.push_back(1);
std::vector<unsigned> payload_left;
payload_left.push_back(1);
std::vector<unsigned> payload_right;
payload_right.push_back(0);
BlockStreamJoinIterator::State bsji_state;//(ebse,bsc2,output,output,output_join,joinIndex_left,joinIndex_right,payload_left,payload_right,100,1024,4096);
BlockStreamJoinIterator* bsji=new BlockStreamJoinIterator(bsji_state);
BlockStreamExpander::State bse_state_(output_join,bsji,thread_count,block_size,expander_buffer);
BlockStreamIteratorBase* bse_=new BlockStreamExpander(bse_state_);
const int exchange_id_1=0;
ExpandableBlockStreamBroadcastExchange::State ebse_state_(output,bse_,block_size,used_lowers_1,upper_ip_list,exchange_id_1);
BlockStreamIteratorBase* ebse_=new ExpandableBlockStreamBroadcastExchange(ebse_state_);
BlockStreamExpander::State bse_state_1(output_join,ebse_,thread_count,block_size,expander_buffer);
BlockStreamIteratorBase* bse_1=new BlockStreamExpander(bse_state_1);
BlockStreamPerformanceMonitorTop::State bspfm_state(output_join,bse_1,block_size,1000);
BlockStreamIteratorBase* bspfm=new BlockStreamPerformanceMonitorTop(bspfm_state);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
BlockStreamBase *block=new BlockStreamFix(block_size,12);
volatile int choice;
printf("Continue(1) or Not(0) ?\n");
scanf("%d",&choice);
unsigned tuple_count=0;
while(choice==1){
// bsf->open();
ebse_->open();
unsigned long long int start=curtick();
// tuple_count=0;
while(ebse_->next(block)){
BlockStreamBase::BlockStreamTraverseIterator *it=block->createIterator();
void* tuple;
while(tuple=it->nextTuple()){
printf("tuple:%d \n",*(int*)tuple);
tuple_count++;
}
block->setEmpty();
}
printf("Total tupls:%d\n",tuple_count);
printf("Time=%f Throughput=%f.\n",getSecond(start),1024/getSecond(start));
ebse_->close();
printf("Continue(0) or Not(1) ?\n");
// getchar();
scanf("%d",&choice);
printf("you input %d\n",choice);
}
}
else{
Environment::getInstance(false);
}
printf("Waiting~~~~~....\n");
while(true){
sleep(1);
}
}