int ObMetaTable3::MyIterator::add_scan_columns(ObScanParam &scan_param)
{
int ret = OB_SUCCESS;
if (OB_SUCCESS != (ret = get_table_schema()))
{
TBSYS_LOG(WARN, "failed to get table schema, err=%d", ret);
}
else
{
ObMetaTableColumnSchema col_schema(table_schema_);
for (int64_t i = 0; i < col_schema.get_columns_num(); ++i)
{
uint64_t cid = col_schema.get_cid_by_idx(i);
if (OB_INVALID_ID == cid)
{
TBSYS_LOG(ERROR, "failed to get column id, i=%ld", i);
}
else if (OB_SUCCESS != (ret = scan_param.add_column(cid)))
{
TBSYS_LOG(WARN, "failed to add column, err=%d", ret);
break;
}
}
}
return ret;
}
int MutatorBuilder::build_total_scan_param(ObScanParam &scan_param, PageArena<char> &allocer,
const int64_t table_start_version,
const bool using_id, const int64_t table_pos)
{
int ret = OB_SUCCESS;
const ObSchema &schema = schema_mgr_.begin()[table_pos];
int64_t cur_rowkey_info = get_cur_rowkey_info_(schema, using_id, rb_array_[table_pos]->get_prefix_start());
std::pair<ObString,ObString> key_range = rb_array_[table_pos]->get_rowkey4total_scan(cur_rowkey_info, allocer);
ObRange range;
range.start_key_ = key_range.first;
range.end_key_ = key_range.second;
const ObColumnSchema *iter = NULL;
for (iter = schema.column_begin(); iter != schema.column_end(); iter++)
{
if (using_id)
{
scan_param.add_column(iter->get_id());
}
else
{
ObString column_name;
column_name.assign_ptr(const_cast<char*>(iter->get_name()), static_cast<int32_t>(strlen(iter->get_name())));
scan_param.add_column(column_name);
}
}
if (using_id)
{
scan_param.set(schema.get_table_id(), ObString(), range);
}
else
{
ObString table_name;
table_name.assign_ptr(const_cast<char*>(schema.get_table_name()), static_cast<int32_t>(strlen(schema.get_table_name())));
scan_param.set(OB_INVALID_ID, table_name, range);
}
ObVersionRange version_range;
version_range.start_version_ = table_start_version;
version_range.border_flag_.set_max_value();
version_range.border_flag_.set_inclusive_start();
scan_param.set_version_range(version_range);
return ret;
}
void prepare_scan_param(ObScanParam &scan_param, const ObSchema &schema,
const int64_t table_start_version, const bool using_id)
{
ObRange range;
range.border_flag_.set_min_value();
range.border_flag_.set_max_value();
const ObColumnSchema *iter = NULL;
for (iter = schema.column_begin(); iter != schema.column_end(); iter++)
{
if (using_id)
{
scan_param.add_column(iter->get_id());
}
else
{
ObString column_name;
column_name.assign_ptr(const_cast<char*>(iter->get_name()), strlen(iter->get_name()));
scan_param.add_column(column_name);
}
}
if (using_id)
{
scan_param.set(schema.get_table_id(), ObString(), range);
}
else
{
ObString table_name;
table_name.assign(const_cast<char*>(schema.get_table_name()), strlen(schema.get_table_name()));
scan_param.set(OB_INVALID_ID, table_name, range);
}
ObVersionRange version_range;
version_range.start_version_ = table_start_version;
version_range.border_flag_.set_max_value();
version_range.border_flag_.set_inclusive_start();
scan_param.set_version_range(version_range);
}
int TaskFactory::add_all_columns(const uint64_t table_id, ObScanParam & param)
{
int ret = OB_SUCCESS;
int32_t count = 0;
const ObColumnSchemaV2 * column_schema = schema_->get_table_schema(table_id, count);
if ((NULL == column_schema) || (0 == count))
{
ret = OB_ERROR;
TBSYS_LOG(ERROR, "check column schema or schema count failed:column[%p], count[%d]",
column_schema, count);
}
else
{
ObString column;
const char * column_name = NULL;
for (int32_t i = 0; i < count; ++i)
{
column_name = column_schema->get_name();
if (column_name != NULL)
{
TBSYS_LOG(DEBUG, "TableId:%ld, COLUMN=%s", table_id, column_name);
column.assign(const_cast<char *>(column_name), (int32_t)strlen(column_name));
ret = param.add_column(column);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "add column failed:table[%lu], column[%s], ret[%d]",
table_id, column_name, ret);
ret = OB_ERROR;
break;
}
}
else
{
ret = OB_ERROR;
TBSYS_LOG(ERROR, "check column name failed:table[%lu], name[%s]", table_id, column_name);
break;
}
++column_schema;
}
}
return ret;
}
int scan_test_case_common(
BaseClient& client,
const ObServer& server,
const int64_t table_id,
const char* start_key_ptr,
const char* end_key_ptr,
const int64_t input_key_size,
const bool is_hex
)
{
ObScanParam input;
int64_t key_size = input_key_size;
if (is_hex)
{
key_size = input_key_size / 2;
}
char start_key[input_key_size];
char end_key[input_key_size];
char range_str[OB_RANGE_STR_BUFSIZ];
memset(start_key, 0, input_key_size);
memset(end_key, 0, input_key_size);
if (is_hex)
{
str_to_hex(start_key_ptr, static_cast<int32_t>(input_key_size), start_key, static_cast<int32_t>(key_size));
str_to_hex(end_key_ptr, static_cast<int32_t>(input_key_size), end_key, static_cast<int32_t>(key_size));
}
else
{
memcpy(start_key, start_key_ptr, key_size);
memcpy(end_key, end_key_ptr, key_size);
}
ObString ob_table_name(0, 0, NULL);
ObRange range;
range.table_id_= 0;
range.start_key_.assign_ptr(start_key, static_cast<int32_t>(key_size));
range.end_key_.assign_ptr(end_key, static_cast<int32_t>(key_size));
range.border_flag_.set_inclusive_start();
range.border_flag_.set_inclusive_end();
range.to_string(range_str, OB_RANGE_STR_BUFSIZ);
range.hex_dump();
fprintf(stderr, "input scan range = %s\n", range_str);
input.set(table_id, ob_table_name, range);
input.set_scan_size(100000);
input.add_column((uint64_t)0ull);
ObScanner scanner;
int64_t start = tbsys::CTimeUtil::getTime();
int ret = rpc_cs_scan(client.client_, server, input, scanner);
int64_t end = tbsys::CTimeUtil::getTime();
fprintf(stderr,"rpc_cs_scan time consume:%ld\n", end - start);
if (OB_SUCCESS != ret)
{
fprintf(stderr,"ret:%d\n", ret);
}
else
{
dump_scanner(scanner);
}
return ret;
}
int scan_test_case(
BaseClient& client,
const ObServer& server,
const uint64_t table_id,
const int64_t user_id,
const int8_t item_type,
const int32_t column_size
)
{
ObScanParam input;
const int32_t key_size = 17;
char start_key[key_size] ;
char end_key[key_size] ;
int64_t pos = 0;
encode_i64(start_key, key_size, pos, user_id);
if (item_type <= 1)
encode_i8(start_key, key_size, pos, item_type);
else
encode_i8(start_key, key_size, pos, 0x0);
memset(start_key + pos, 0x0, key_size - pos);
pos = 0;
encode_i64(end_key, key_size, pos, user_id);
if ( item_type <= 1)
encode_i8(end_key, key_size, pos, item_type);
else
encode_i8(end_key, key_size, pos, static_cast<int8_t>(0xFF));
memset(end_key + pos, 0xFF, key_size - pos);
hex_dump(start_key, key_size);
hex_dump(end_key, key_size);
ObString ob_table_name;
ObRange range;
range.table_id_= table_id;
range.start_key_.assign_ptr(start_key, key_size);
range.end_key_.assign_ptr(end_key, key_size);
range.border_flag_.set_inclusive_start();
range.border_flag_.set_inclusive_end();
range.hex_dump();
input.set(table_id, ob_table_name, range);
input.set_scan_size(100000);
for (int i = 0; i < column_size; ++i)
{
input.add_column(i + 2);
}
ObScanner scanner;
int64_t start = tbsys::CTimeUtil::getTime();
int ret = rpc_cs_scan(client.client_, server, input, scanner);
int64_t end = tbsys::CTimeUtil::getTime();
fprintf(stderr,"rpc_cs_scan time consume:%ld\n", end - start);
if (OB_SUCCESS != ret)
{
fprintf(stderr,"ret:%d\n", ret);
}
else
{
dump_scanner(scanner);
}
return ret;
}
void read_scan_param(const char *fname, const char *section, PageArena<char> &allocer, ObScanParam &scan_param)
{
TBSYS_CONFIG.load(fname);
const char *str = NULL;
int32_t len = 0;
str = TBSYS_CONFIG.getString(section, SCAN_FROZEN_ONLY);
if (0 == strcmp("YES", str))
{
//scan_param.set_is_read_frozen_only(true);
}
str = TBSYS_CONFIG.getString(section, SCAN_REVERSE);
if (NULL == str
|| 0 != strcmp("YES", str))
{
scan_param.set_scan_direction(ObScanParam::FORWARD);
}
else
{
scan_param.set_scan_direction(ObScanParam::BACKWARD);
}
ObString table_name;
sprintf(buffer, SCAN_TABLE_NAME);
str = getAndCopySting_(allocer, section, buffer, len);
table_name.assign_ptr(const_cast<char*>(str), len);
uint64_t table_id = OB_INVALID_ID;
if (NULL == str)
{
table_id = TBSYS_CONFIG.getInt(section, SCAN_TABLE_ID);
}
else
{
table_id = OB_INVALID_ID;
}
int32_t columns_num = TBSYS_CONFIG.getInt(section, SCAN_COLUMNS_NUM);
for (int32_t i = 0; i < columns_num; i++)
{
ObString column_name;
sprintf(buffer, SCAN_COLUMN_NAME_KEY_FORMAT, i);
str = getAndCopySting_(allocer, section, buffer, len);
column_name.assign_ptr(const_cast<char*>(str), len);
if (NULL != str)
{
scan_param.add_column(column_name);
}
else
{
sprintf(buffer, SCAN_COLUMN_ID_KEY_FORMAT, i);
uint64_t column_id = TBSYS_CONFIG.getInt(section, buffer);
scan_param.add_column(column_id);
}
}
ObRange range;
range.table_id_ = table_id;
str = getAndCopySting_(allocer, section, SCAN_START_KEY, len);
range.start_key_.assign_ptr(const_cast<char*>(str), len);
if (0 == strcmp("MIN_KEY", str))
{
range.border_flag_.set_min_value();
}
if (0 == strcmp("MAX_KEY", str))
{
range.border_flag_.set_max_value();
}
str = getAndCopySting_(allocer, section, SCAN_END_KEY, len);
range.end_key_.assign_ptr(const_cast<char*>(str), len);
if (0 == strcmp("MIN_KEY", str))
{
range.border_flag_.set_min_value();
}
if (0 == strcmp("MAX_KEY", str))
{
range.border_flag_.set_max_value();
}
str = getAndCopySting_(allocer, section, SCAN_INCLUSIVE_START, len);
if (0 == strcmp("YES", str))
{
range.border_flag_.set_inclusive_start();
}
str = getAndCopySting_(allocer, section, SCAN_INCLUSIVE_END, len);
if (0 == strcmp("YES", str))
{
range.border_flag_.set_inclusive_end();
}
scan_param.set(table_id, table_name, range);
}
void init_decode_param(ObStringBuf &buf, ObScanParam &org_scan_param)
{
UNUSED(buf);
const char *c_ptr;
ObString str;
ObNewRange q_range;
q_range.set_whole_range();
q_range.border_flag_.set_inclusive_start();
q_range.border_flag_.set_inclusive_end();
c_ptr = "collect_info";
str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
org_scan_param.reset();
EXPECT_EQ(org_scan_param.set(OB_INVALID_ID, str,q_range), OB_SUCCESS);
/// basic columns
bool is_return = false;
c_ptr = "item_collect_count";
str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
EXPECT_EQ(org_scan_param.add_column(str,is_return), OB_SUCCESS );
is_return = false;
c_ptr = "item_category";
str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
EXPECT_EQ(org_scan_param.add_column(str,is_return), OB_SUCCESS );
is_return = true;
c_ptr = "item_price";
str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
EXPECT_EQ(org_scan_param.add_column(str,is_return), OB_SUCCESS );
/// composite columns
c_ptr = "`item_collect_count`*`item_price`";
str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
EXPECT_EQ(org_scan_param.add_column(str,str, is_return), OB_SUCCESS );
/// where condition
c_ptr = "`item_price` > 10";
str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
EXPECT_EQ(org_scan_param.add_where_cond(str), OB_SUCCESS );
/// groupby::group by columns
is_return = false;
c_ptr = "item_category";
str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
EXPECT_EQ(org_scan_param.get_group_by_param().add_groupby_column(str,is_return), OB_SUCCESS);
/// aggregate columns
is_return = true;
c_ptr = "item_price";
str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
EXPECT_EQ(org_scan_param.get_group_by_param().add_aggregate_column(str,str, SUM, is_return), OB_SUCCESS);
/// aggregate columns
is_return = false;
c_ptr = "item_collect_count";
str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
EXPECT_EQ(org_scan_param.get_group_by_param().add_aggregate_column(str,str, SUM, is_return), OB_SUCCESS);
/// composite columns
c_ptr = "`item_collect_count`*`item_price`";
str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
EXPECT_EQ(org_scan_param.get_group_by_param().add_column(str, str,is_return), OB_SUCCESS );
/// having condtion
c_ptr = "`item_price` > 10";
str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
EXPECT_EQ(org_scan_param.get_group_by_param().add_having_cond(str), OB_SUCCESS );
/// orderby
c_ptr = "item_price";
str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
EXPECT_EQ(org_scan_param.add_orderby_column(str),OB_SUCCESS);
}
int MutatorBuilder::build_scan_param(ObScanParam &scan_param, PageArena<char> &allocer,
const int64_t table_start_version, const bool using_id, int64_t &table_pos, int64_t &prefix)
{
int ret = OB_SUCCESS;
scan_param.reset();
static int64_t row_info_array[OB_MAX_TABLE_NUMBER];
static int64_t counter = 0;
static const int64_t ROW_INFO_CACHE_FLUSH_PERIOD = 100;
//static const int64_t ROW_INFO_CACHE_FLUSH_PERIOD = 1000000000;
if (0 == (counter++ % ROW_INFO_CACHE_FLUSH_PERIOD))
{
memset(row_info_array, 0, sizeof(row_info_array));
}
int64_t rand = 0;
lrand48_r(&table_rand_seed_, &rand);
table_pos = range_rand(0, table_num_ - 1, rand);
const ObSchema &schema = schema_mgr_.begin()[table_pos];
if (0 == row_info_array[table_pos]
&& 0 == (row_info_array[table_pos] = get_cur_rowkey_info_(schema, using_id, rb_array_[table_pos]->get_prefix_start())))
{
ret = OB_ERROR;
}
else
{
ObRange range;
std::pair<ObString, ObString> key_range = rb_array_[table_pos]->get_rowkey4scan(row_info_array[table_pos], allocer, prefix);
range.start_key_ = key_range.first;
range.end_key_ = key_range.second;
if (using_id)
{
scan_param.add_column(SEED_COLUMN_ID);
}
else
{
ObString column_name;
column_name.assign_ptr((char*)SEED_COLUMN_NAME, static_cast<int32_t>(strlen(SEED_COLUMN_NAME)));
scan_param.add_column(column_name);
}
if (using_id)
{
scan_param.add_column(CELL_NUM_COLUMN_ID);
}
else
{
ObString column_name;
column_name.assign_ptr((char*)CELL_NUM_COLUMN_NAME, static_cast<int32_t>(strlen(CELL_NUM_COLUMN_NAME)));
scan_param.add_column(column_name);
}
lrand48_r(&table_rand_seed_, &rand);
int64_t column_num = schema.column_end() - schema.column_begin();
for (int64_t i = 0; i < column_num; i++)
{
lrand48_r(&table_rand_seed_, &rand);
int64_t column_pos = range_rand(0, column_num - 1, rand);
//int64_t column_pos = 0;
const ObColumnSchema &column_schema = schema.column_begin()[column_pos];
if (using_id)
{
scan_param.add_column(column_schema.get_id());
}
else
{
ObString column_name;
column_name.assign_ptr(const_cast<char*>(column_schema.get_name()), static_cast<int32_t>(strlen(column_schema.get_name())));
scan_param.add_column(column_name);
}
}
if (using_id)
{
scan_param.set(schema.get_table_id(), ObString(), range);
}
else
{
ObString table_name;
table_name.assign_ptr(const_cast<char*>(schema.get_table_name()), static_cast<int32_t>(strlen(schema.get_table_name())));
scan_param.set(OB_INVALID_ID, table_name, range);
}
ObVersionRange version_range;
version_range.start_version_ = table_start_version;
version_range.border_flag_.set_max_value();
version_range.border_flag_.set_inclusive_start();
scan_param.set_version_range(version_range);
char* is_read_consistency_cfg = getenv("is_read_consistency");
bool is_read_consistency = (NULL != is_read_consistency_cfg && 0 == strcmp(is_read_consistency_cfg, "true"));
scan_param.set_is_read_consistency(is_read_consistency);
}
return ret;
}
int ObScanParamLoader::get_org_scan_param(ObScanParam ¶m)
{
ObRange range;
ObString column;
ObString table_name;
ObPostfixExpression comp;
if (config_loaded_)
{
TBSYS_LOG(INFO, "creating org scan param");
param.reset();
range.reset();
range.table_id_ = table_id_;
range.start_key_ = range_start_;
if (range_start_inclusive_)
range.border_flag_.set_inclusive_start();
else
range.border_flag_.unset_inclusive_start();
if (range_start_min_)
range.border_flag_.set_min_value();
range.end_key_ = range_end_;
if (range_end_inclusive_)
range.border_flag_.set_inclusive_end();
else
range.border_flag_.unset_inclusive_end();
if (range_end_max_)
range.border_flag_.set_max_value();
param.set(table_id_, table_name_, range, true);
param.set_scan_direction(scan_direction_==0?ObScanParam::FORWARD:ObScanParam::BACKWARD);
param.set_limit_info(limit_offset_, limit_count_);
int i = 0;
for (i = 0; i < column_id_count_; i++)
{
param.add_column(column_id_[i], column_is_return_[i]);
}
for (i = 0; i < complex_column_id_count_; i++)
{
param.add_column(complex_column_id_[i],complex_column_id_[i], complex_column_is_return_[i]);
}
for (i = 0; i < orderby_column_id_count_; i++)
{
param.add_orderby_column(orderby_column_id_[i],
orderby_column_order_[i]==0?ObScanParam::ASC:ObScanParam::DESC);
}
if (where_cond_.length() > 0 && where_cond_.ptr() != NULL)
{
param.add_where_cond(where_cond_);
}
// groupby param
for (i = 0; i < groupby_column_id_count_; i++)
{
param.get_group_by_param().add_groupby_column(groupby_column_id_[i], groupby_column_is_return_[i]);
}
for (i = 0; i < groupby_complex_column_id_count_; i++)
{
param.get_group_by_param().add_column(groupby_complex_column_id_[i],
groupby_complex_column_id_[i],
groupby_complex_column_is_return_[i]);
}
if (having_cond_.length() > 0 && having_cond_.ptr() != NULL)
{
param.get_group_by_param().add_having_cond(having_cond_);
}
/// FIXME: SUM占位, 需要添加OP域
for (i = 0; i < agg_column_id_count_; i++)
{
param.get_group_by_param().add_aggregate_column(agg_column_id_[i],
agg_column_as_name_[i],
(ObAggregateFuncType)agg_column_op_[i],
agg_column_is_return_[i]);
}
}
return OB_SUCCESS;
}