int GetNotExistRows::form_get_param(ObGetParam &get_param, const uint32_t min_key_include, const uint32_t max_key_include,
void *&arg)
{
UNUSED(min_key_include);
int err = OB_SUCCESS;
arg = NULL;
const int32_t get_cell_count = 32;
const uint32_t random_selector = 1024*16;
uint32_t start_key_val = static_cast<uint32_t>(max_key_include + 1 + random()%random_selector);
uint32_t key;
get_param.reset(true);
ObCellInfo cell;
cell.table_name_ = msolap::target_table_name;
cell.column_name_ = ObGetParam::OB_GET_ALL_COLUMN_NAME;
for (int32_t i = 0; (OB_SUCCESS == err) && (i < get_cell_count); i++, start_key_val ++)
{
key = htonl(start_key_val);
cell.row_key_.assign((char*)&key,sizeof(key));
if (OB_SUCCESS != (err = get_param.add_cell(cell)))
{
TBSYS_LOG(WARN,"fail to add cell to ObGetParam [err:%d]", err);
}
}
if (OB_SUCCESS == err)
{
arg = &get_param;
}
return err;
}
int MultiGetP::form_get_param(ObGetParam &get_param, const uint32_t min_key_include, const uint32_t max_key_include,
void *&arg)
{
int err = OB_SUCCESS;
arg = NULL;
get_param.reset(true);
char c_name;
ObCellInfo cell;
cell.table_name_ = msolap::target_table_name;
cell.column_name_.assign(&c_name, sizeof(c_name));
int64_t cell_count = get_cell_count;
if (cell_count > max_key_include - min_key_include + 1)
{
cell_count = max_key_include - min_key_include + 1;
}
int64_t random_selector = (max_key_include - min_key_include + 1)/cell_count;
/// bool get_last_cell = false;
uint32_t start_key = min_key_include;
uint32_t start_key_val = 0;
char strkey[32];
for (int64_t cell_idx = 0;
(OB_SUCCESS == err) && (start_key <= max_key_include) && (cell_idx < cell_count);
cell_idx ++,start_key = static_cast<int32_t>(start_key + random_selector))
{
uint32_t cur_key = static_cast<uint32_t>(start_key + random() % random_selector);
c_name = msolap::olap_get_column_name(msolap::min_column_id + random()%(msolap::max_column_id - msolap::min_column_id + 1));
start_key_val = htonl(cur_key);
snprintf(strkey, 32, "%d", start_key_val);
cell.row_key_ = make_rowkey(strkey, &allocator_);
assert(cell.column_name_.ptr()[0] <= 'z' && cell.column_name_.ptr()[0] >= 'a');
if (OB_SUCCESS != (err = get_param.add_cell(cell)))
{
TBSYS_LOG(WARN,"fail to add cell to ObGetParam [err:%d,c_name:%c]", err, c_name);
}
}
if (OB_SUCCESS == err)
{
arg = &get_param;
}
return err;
}
int ObSKeyInfoKey::build_get_param(ObGetParam& get_param)
{
int ret = OB_SUCCESS;
ObCellInfo cell_info;
ObRowkey rowkey;
ObObj rowkey_obj;
ObVersionRange ver_range;
int64_t pos = 0;
if (NULL == buffer_)
{
buffer_ = reinterpret_cast<char*>(ob_malloc(BUF_SIZE, ObModIds::OB_SKEY_INFO_KEY));
}
ret = serialization::encode_i64(buffer_, BUF_SIZE, pos, skey_version_);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(ERROR, "encode failed pkey_version=%ld", skey_version_);
}
else
{
rowkey_obj.set_varchar(ObString(0, static_cast<int32_t>(pos), buffer_));
rowkey.assign(&rowkey_obj, 1);
get_param.reset();
get_param.set_is_result_cached(true);
cell_info.table_id_ = SKEY_TABLE_ID;
cell_info.row_key_ = rowkey;
cell_info.column_id_ = SKEY_COL_ID;
ret = get_param.add_cell(cell_info);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(ERROR, "Get param add cell failed table_id[%lu]", cell_info.table_id_);
}
else
{
ver_range.start_version_ = 0;
ver_range.end_version_ = INT64_MAX - 1;
ver_range.border_flag_.set_inclusive_start();
ver_range.border_flag_.set_inclusive_end();
get_param.set_version_range(ver_range);
}
}
return ret;
}
int GetSingleRowAllColumn::form_get_param(ObGetParam &get_param, const uint32_t min_key_include, const uint32_t max_key_include,
void *&arg)
{
int err = OB_SUCCESS;
arg = NULL;
uint32_t start_key_val = htonl(static_cast<int32_t>(random()%(max_key_include - min_key_include + 1) + min_key_include));
get_param.reset(true);
ObCellInfo cell;
cell.table_name_ = msolap::target_table_name;
cell.column_name_ = ObGetParam::OB_GET_ALL_COLUMN_NAME;
cell.row_key_.assign((char*)&start_key_val,sizeof(start_key_val));
if (OB_SUCCESS != (err = get_param.add_cell(cell)))
{
TBSYS_LOG(WARN,"fail to add cell to ObGetParam [err:%d]", err);
}
if (OB_SUCCESS == err)
{
arg = &get_param;
}
return err;
}
// maybe not find a merge server
int TaskFactory::get_table_tablet(const char * table_name, const uint64_t table_id, uint64_t & count)
{
int ret = OB_SUCCESS;
ObScanParam scan_param;
uint64_t max_len = 0;
if (NULL == table_name)
{
TBSYS_LOG(ERROR, "check table name failed:table[%s], id[%lu]", table_name, table_id);
ret = OB_ERROR;
}
else
{
ret = init_scan_param(table_name, table_id, max_len, scan_param);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init scan param failed:table[%s], ret[%d]", table_name, ret);
}
}
if (OB_SUCCESS == ret)
{
// for the first table tablet
ObRowkey row_key;
ObObj temp_buffer[common::OB_MAX_ROWKEY_COLUMN_NUMBER];
row_key.assign(temp_buffer, common::OB_MAX_ROWKEY_COLUMN_NUMBER);
ObGetParam param;
ObScanner scanner;
ObServer server;
ObRowkey start_key;
ObRowkey end_key;
ObCellInfo * cell = NULL;
ObScannerIterator iter;
bool row_change = false;
ObString name;
name.assign(const_cast<char*>(table_name), (int32_t)strlen(table_name));
ObCellInfo temp_cell;
temp_cell.table_id_ = table_id;
temp_cell.column_id_ = 0;
common::ModuleArena allocator;
while ((OB_SUCCESS == ret) && (!row_key.is_max_row()))
{
param.reset();
param.set_is_read_consistency(false);
temp_cell.row_key_ = row_key;
ret = param.add_cell(temp_cell);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "add cell failed:ret[%d]", ret);
break;
}
ret = rpc_->get(root_server_, timeout_, param, scanner);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "get root table for tablet failed:table[%lu], ret[%d]", table_id, ret);
break;
}
else
{
// skip the first row
iter = scanner.begin();
++iter;
while ((iter != scanner.end())
&& (OB_SUCCESS == (ret = iter.get_cell(&cell, &row_change))) && !row_change)
{
if (NULL == cell)
{
TBSYS_LOG(ERROR, "%s", "check cell failed");
ret = OB_INNER_STAT_ERROR;
break;
}
ret = cell->row_key_.deep_copy(start_key, allocator);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(WARN, "deep copy the rowkey failed:ret[%d]", ret);
}
++iter;
}
}
// scanner iter end
if (ret == OB_SUCCESS)
{
int64_t ip = 0;
int64_t port = 0;
int64_t version = 0;
TabletLocation list;
for (++iter; iter != scanner.end(); ++iter)
{
ret = iter.get_cell(&cell, &row_change);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret);
break;
}
else if (row_change) // && (iter != last_iter))
{
TaskInfo task;
task.set_table_id(table_id);
task.set_table_name(table_name);
ret = init_new_task(name, start_key, end_key, scan_param, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret);
break;
}
else
{
ret = insert_new_task(list, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret);
break;
}
++count;
}
list.clear();
start_key = end_key;
}
else
{
cell->row_key_.deep_copy(end_key, allocator);
if ((cell->column_name_.compare("1_ms_port") == 0)
|| (cell->column_name_.compare("2_ms_port") == 0)
|| (cell->column_name_.compare("3_ms_port") == 0))
{
ret = cell->value_.get_int(port);
}
else if ((cell->column_name_.compare("1_ipv4") == 0)
|| (cell->column_name_.compare("2_ipv4") == 0)
|| (cell->column_name_.compare("3_ipv4") == 0))
{
ret = cell->value_.get_int(ip);
}
else if ((cell->column_name_.compare("1_tablet_version") == 0)
|| (cell->column_name_.compare("2_tablet_version") == 0)
|| (cell->column_name_.compare("3_tablet_version") == 0))
{
ret = cell->value_.get_int(version);
if (OB_SUCCESS == ret)
{
if (0 == port || port == 0)
{
TBSYS_LOG(WARN, "%s", "check port or ip failed");
}
else
{
server.set_ipv4_addr(int32_t(ip), int32_t(port));
ObTabletLocation addr(version, server);
if (OB_SUCCESS != (ret = list.add(addr)))
{
TBSYS_LOG(ERROR, "add addr failed:server[%d], port[%d], ret[%d]",
server.get_ipv4(), server.get_port(), ret);
}
else
{
TBSYS_LOG(DEBUG, "add addr succ:server[%s], version:%ld", server.to_cstring(), version);
}
}
ip = port = version = 0;
}
}
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret);
break;
}
}
}
// for the last row
if ((OB_SUCCESS == ret) && (start_key != end_key))
{
TaskInfo task;
task.set_table_id(table_id);
task.set_table_name(table_name);
ret = init_new_task(name, start_key, end_key, scan_param, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret);
}
else
{
ret = insert_new_task(list, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret);
}
++count;
}
if (OB_SUCCESS == ret)
{
// copy all the end key objs to row_key
for (int64_t i = 0; i < end_key.length(); ++i)
{
const_cast<ObObj *> (row_key.ptr())[i] = end_key.ptr()[i];
row_key.assign(const_cast<ObObj *> (row_key.ptr()), end_key.length());
}
// modify last row key for next get root table
if (!end_key.is_max_row())
{
const_cast<ObObj *>(row_key.ptr())[end_key.length()].set_max_value();
row_key.assign(const_cast<ObObj *> (row_key.ptr()), end_key.length() + 1);
}
}
}
list.clear();
}
}
}
return ret;
}
int MutatorBuilder::build_get_param(ObGetParam &get_param, PageArena<char> &allocer,
const int64_t table_start_version, const bool using_id,
const int64_t row_num, const int64_t cell_num_per_row)
{
int ret = OB_SUCCESS;
get_param.reset();
UNUSED(table_start_version);
static int64_t row_info_array[OB_MAX_TABLE_NUMBER];
static int64_t suffix_num_array[OB_MAX_TABLE_NUMBER];
static int64_t counter = 0;
static const int64_t ROW_INFO_CACHE_FLUSH_PERIOD = 100;
//static const int64_t ROW_INFO_CACHE_FLUSH_PERIOD = 1000000000;
if (0 == (counter++ % ROW_INFO_CACHE_FLUSH_PERIOD))
{
memset(row_info_array, 0, sizeof(row_info_array));
memset(suffix_num_array, 0, sizeof(suffix_num_array));
}
for (int64_t i = 0; i < row_num; i++)
{
int64_t rand = 0;
lrand48_r(&table_rand_seed_, &rand);
int64_t table_pos = range_rand(0, table_num_ - 1, rand);
const ObSchema &cur_schema = schema_mgr_.begin()[table_pos];
if (0 == row_info_array[table_pos]
&& 0 == (row_info_array[table_pos] = get_cur_rowkey_info_(cur_schema, using_id, rb_array_[table_pos]->get_prefix_start())))
{
assert(OB_SUCCESS == ret);
ret = OB_ERROR;
break;
}
if (0 == suffix_num_array[table_pos]
&& 0 == (suffix_num_array[table_pos] = get_suffix_num_(cur_schema, using_id, rb_array_[table_pos]->get_prefix_start())))
{
assert(OB_SUCCESS == ret);
ret = OB_ERROR;
break;
}
ObCellInfo ci;
ci.row_key_ = rb_array_[table_pos]->get_random_rowkey(row_info_array[table_pos], suffix_num_array[table_pos], allocer);
if (using_id)
{
ci.table_id_ = cur_schema.get_table_id();
}
else
{
ci.table_name_.assign_ptr(const_cast<char*>(cur_schema.get_table_name()), static_cast<int32_t>(strlen(cur_schema.get_table_name())));
}
if (using_id)
{
ci.column_id_ = SEED_COLUMN_ID;
ci.column_name_.assign_ptr(NULL, 0);
}
else
{
ci.column_id_ = OB_INVALID_ID;
ci.column_name_.assign_ptr((char*)SEED_COLUMN_NAME, static_cast<int32_t>(strlen(SEED_COLUMN_NAME)));
}
get_param.add_cell(ci);
if (using_id)
{
ci.column_id_ = CELL_NUM_COLUMN_ID;
ci.column_name_.assign_ptr(NULL, 0);
}
else
{
ci.column_id_ = OB_INVALID_ID;
ci.column_name_.assign_ptr((char*)CELL_NUM_COLUMN_NAME, static_cast<int32_t>(strlen(CELL_NUM_COLUMN_NAME)));
}
get_param.add_cell(ci);
lrand48_r(&table_rand_seed_, &rand);
int64_t cell_num = range_rand(1, cell_num_per_row, rand);
int64_t column_num = cur_schema.column_end() - cur_schema.column_begin();
for (int64_t j = 0; j < cell_num; j++)
{
lrand48_r(&table_rand_seed_, &rand);
int64_t column_pos = range_rand(0, column_num - 1, rand);
//int64_t column_pos = 0;
const ObColumnSchema &column_schema = cur_schema.column_begin()[column_pos];
if (using_id)
{
ci.column_id_ = column_schema.get_id();
ci.column_name_.assign_ptr(NULL, 0);
}
else
{
ci.column_id_ = OB_INVALID_ID;
ci.column_name_.assign_ptr(const_cast<char*>(column_schema.get_name()), static_cast<int32_t>(strlen(column_schema.get_name())));
}
get_param.add_cell(ci);
}
}
ObVersionRange version_range;
version_range.start_version_ = table_start_version_;
version_range.border_flag_.set_max_value();
version_range.border_flag_.set_inclusive_start();
get_param.set_version_range(version_range);
char* is_read_consistency_cfg = getenv("is_read_consistency");
bool is_read_consistency = (NULL != is_read_consistency_cfg && 0 == strcmp(is_read_consistency_cfg, "true"));
get_param.set_is_read_consistency(is_read_consistency);
assert(get_param.get_cell_size() > 0);
return ret;
}
// maybe not find a merge server
int TaskFactory::get_table_tablet(const char * table_name, const uint64_t table_id, uint64_t & count)
{
int ret = OB_SUCCESS;
ObScanParam scan_param;
uint64_t max_len = 0;
if (NULL == table_name)
{
TBSYS_LOG(ERROR, "check table name failed:table[%s], id[%lu]", table_name, table_id);
ret = OB_ERROR;
}
else
{
ret = init_scan_param(table_name, table_id, max_len, scan_param);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init scan param failed:table[%s], ret[%d]", table_name, ret);
}
}
if (OB_SUCCESS == ret)
{
TaskInfo task;
task.set_table_id(table_id);
task.set_table_name(table_name);
// for the first table tablet
ObString row_key;
char temp_buffer[1];
memset(temp_buffer, 0, sizeof(temp_buffer));
row_key.assign(temp_buffer, sizeof(temp_buffer));
ObGetParam param;
ObScanner scanner;
ObServer server;
ObString start_key;
ObString end_key;
ObCellInfo * cell = NULL;
ObScannerIterator iter;
bool row_change = false;
ObString name;
name.assign(const_cast<char*>(table_name), strlen(table_name));
ObCellInfo temp_cell;
temp_cell.table_id_ = table_id;
temp_cell.column_id_ = 0;
const uint64_t MAX_LEN = 1024;
char last_tablet_rowkey[MAX_LEN] = "";
const int32_t MAX_SERVER_ADDR_SIZE = 128;
char server_addr[MAX_SERVER_ADDR_SIZE];
while ((OB_SUCCESS == ret)
&& (!is_max_rowkey(max_len, row_key.ptr(), row_key.length() - 1)))
{
param.reset();
param.set_is_read_consistency(false);
temp_cell.row_key_ = row_key;
ret = param.add_cell(temp_cell);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "add cell failed:ret[%d]", ret);
break;
}
ret = rpc_->get(root_server_, timeout_, param, scanner);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "get root table for tablet failed:table[%lu], ret[%d]", table_id, ret);
break;
}
else
{
// skip the first row
iter = scanner.begin();
++iter;
while ((iter != scanner.end())
&& (OB_SUCCESS == (ret = iter.get_cell(&cell, &row_change))) && !row_change)
{
if (NULL == cell)
{
TBSYS_LOG(ERROR, "%s", "check cell failed");
ret = OB_INNER_STAT_ERROR;
break;
}
start_key.assign(cell->row_key_.ptr(), cell->row_key_.length());
++iter;
}
}
if (ret == OB_SUCCESS)
{
int64_t ip = 0;
int64_t port = 0;
int64_t version = 0;
TabletLocation list;
for (++iter; iter != scanner.end(); ++iter)
{
ret = iter.get_cell(&cell, &row_change);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret);
break;
}
else if (row_change) // && (iter != last_iter))
{
ret = init_new_task(name, start_key, end_key, scan_param, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret);
break;
}
else
{
ret = insert_new_task(list, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret);
break;
}
++count;
}
list.clear();
start_key = end_key;
end_key.assign(cell->row_key_.ptr(), cell->row_key_.length());
}
else
{
end_key.assign(cell->row_key_.ptr(), cell->row_key_.length());
if ((cell->column_name_.compare("1_ms_port") == 0)
|| (cell->column_name_.compare("2_ms_port") == 0)
|| (cell->column_name_.compare("3_ms_port") == 0))
{
ret = cell->value_.get_int(port);
}
else if ((cell->column_name_.compare("1_ipv4") == 0)
|| (cell->column_name_.compare("2_ipv4") == 0)
|| (cell->column_name_.compare("3_ipv4") == 0))
{
ret = cell->value_.get_int(ip);
}
else if ((cell->column_name_.compare("1_tablet_version") == 0)
|| (cell->column_name_.compare("2_tablet_version") == 0)
|| (cell->column_name_.compare("3_tablet_version") == 0))
{
ret = cell->value_.get_int(version);
if (OB_SUCCESS == ret)
{
if (0 == port || port == 0)
{
TBSYS_LOG(WARN, "%s", "check port or ip failed");
}
else
{
server.set_ipv4_addr(ip, port);
ObTabletLocation addr(version, server);
if (OB_SUCCESS != (ret = list.add(addr)))
{
TBSYS_LOG(ERROR, "add addr failed:server[%d], port[%d], ret[%d]",
server.get_ipv4(), server.get_port(), ret);
}
else
{
server.to_string(server_addr, MAX_SERVER_ADDR_SIZE);
TBSYS_LOG(DEBUG, "add addr succ:server[%s], version:%ld", server_addr, version);
}
}
ip = port = version = 0;
}
}
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret);
break;
}
}
}
// for the last row
if ((OB_SUCCESS == ret) && (start_key != end_key))
{
ret = init_new_task(name, start_key, end_key, scan_param, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret);
}
else
{
ret = insert_new_task(list, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret);
}
++count;
}
if (OB_SUCCESS == ret)
{
// modify last row key for next get root table
uint64_t len = end_key.length();
if (MAX_LEN > len + 1)
{
memcpy(last_tablet_rowkey, end_key.ptr(), len);
last_tablet_rowkey[len] = 0;
row_key.assign(last_tablet_rowkey, len + 1);
}
else
{
TBSYS_LOG(ERROR, "check end key failed:len[%lu]", len);
}
}
}
list.clear();
}
}
}
return ret;
}
