int get_test_case_common(
BaseClient& client,
const ObServer& server,
const uint64_t table_id,
const char* start_key,
const int64_t column_count
)
{
ObGetParam input;
int32_t key_size = static_cast<int32_t>(strlen(start_key));
UNUSED(column_count);
ObCellInfo cell;
cell.table_id_ = table_id;
cell.row_key_.assign((char*)start_key, key_size);
cell.column_id_ = 0; //get full row
input.add_cell(cell);
ObScanner scanner;
int64_t start = tbsys::CTimeUtil::getTime();
int ret = rpc_cs_get(client.client_, server, input, scanner);
int64_t end = tbsys::CTimeUtil::getTime();
fprintf(stderr,"rpc_cs_get time consume:%ld\n", end - start);
if (OB_SUCCESS != ret)
{
fprintf(stderr,"ret:%d\n", ret);
}
else
{
dump_scanner(scanner);
}
return ret;
}
int GetNotExistRows::form_get_param(ObGetParam &get_param, const uint32_t min_key_include, const uint32_t max_key_include,
void *&arg)
{
UNUSED(min_key_include);
int err = OB_SUCCESS;
arg = NULL;
const int32_t get_cell_count = 32;
const uint32_t random_selector = 1024*16;
uint32_t start_key_val = static_cast<uint32_t>(max_key_include + 1 + random()%random_selector);
uint32_t key;
get_param.reset(true);
ObCellInfo cell;
cell.table_name_ = msolap::target_table_name;
cell.column_name_ = ObGetParam::OB_GET_ALL_COLUMN_NAME;
for (int32_t i = 0; (OB_SUCCESS == err) && (i < get_cell_count); i++, start_key_val ++)
{
key = htonl(start_key_val);
cell.row_key_.assign((char*)&key,sizeof(key));
if (OB_SUCCESS != (err = get_param.add_cell(cell)))
{
TBSYS_LOG(WARN,"fail to add cell to ObGetParam [err:%d]", err);
}
}
if (OB_SUCCESS == err)
{
arg = &get_param;
}
return err;
}
int MockChunkServer::handle_get_table(ObPacket * ob_packet)
{
int ret = OB_SUCCESS;
ObDataBuffer* data = ob_packet->get_buffer();
if (NULL == data)
{
ret = OB_ERROR;
}
ObGetParam param;
if (OB_SUCCESS == ret)
{
ret = param.deserialize(data->get_data(), data->get_capacity(), data->get_position());
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "%s", "check param failed");
}
}
tbnet::Connection* connection = ob_packet->get_connection();
ThreadSpecificBuffer::Buffer* thread_buffer = response_packet_buffer_.get_buffer();
if (NULL == thread_buffer)
{
ret = OB_ERROR;
}
else
{
thread_buffer->reset();
ObDataBuffer out_buffer(thread_buffer->current(), thread_buffer->remain());
ObResultCode result_msg;
result_msg.result_code_ = ret;
ret = result_msg.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position());
// fake data cell
ObCellInfo cell;
ObScanner scanner;
ObRowkey row_key;
ObString column_name;
char temp[256] = "";
cell.table_id_ = 101;
for (uint64_t i = 0; i < 10; ++i)
{
snprintf(temp, 256, "chunk_%lu_get_row_key:%lu", i, i);
row_key = make_rowkey(temp, &allocator_);
cell.row_key_ = row_key;
cell.column_id_ = i + 1;
cell.value_.set_int(2234 + i);
scanner.add_cell(cell);
}
scanner.set_is_req_fullfilled(true, 1);
int32_t channel_id = ob_packet->getChannelId();
ret = scanner.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position());
//
ret = send_response(OB_GET_RESPONSE, 1, out_buffer, connection, channel_id);
}
TBSYS_LOG(INFO, "handle get table result:ret[%d]", ret);
return ret;
}
int ObTransferSSTableQuery::get(const ObGetParam& get_param,
ObSSTableReader& reader,
ObIterator* iterator)
{
int ret = OB_SUCCESS;
int64_t cell_size = get_param.get_cell_size();
int64_t row_size = get_param.get_row_size();
ObSSTableGetter* sstable_getter = dynamic_cast<ObSSTableGetter*>(iterator);
const ObGetParam::ObRowIndex* row_index = get_param.get_row_index();
static __thread const ObSSTableReader* sstable_reader = NULL;
if (!inited_)
{
TBSYS_LOG(WARN, "transfer sstable query has not inited");
ret = OB_ERROR;
}
else if (cell_size <= 0 || NULL == row_index || row_size <= 0
|| row_size > OB_MAX_GET_ROW_NUMBER)
{
TBSYS_LOG(WARN, "invalid get param, cell_size=%ld, row_index=%p, row_size=%ld",
cell_size, row_index, row_size);
ret = OB_INVALID_ARGUMENT;
}
else if (NULL == sstable_getter)
{
TBSYS_LOG(WARN, "get thread local instance of sstable getter failed");
ret = OB_ERROR;
}
if (OB_SUCCESS == ret)
{
if (!reader.empty())
{
sstable_reader = &reader;
}
else
{
sstable_reader = NULL;
}
ret = sstable_getter->init(block_cache_, block_index_cache_,
get_param, &sstable_reader, 1, true,
sstable_row_cache_);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "failed to init sstable getter, ret=%d", ret);
}
else
{
// do nothing
}
}
return ret;
}
int ObPermInfoKey::build_get_param(ObGetParam & get_param)
{
int ret = OB_SUCCESS;
ObCellInfo cell_info;
ObRowkey rowkey;
ObObj rowkey_obj;
ObVersionRange ver_range;
int64_t pos = 0;
if (NULL == buffer_)
{
buffer_ = reinterpret_cast<char*>(ob_malloc(BUF_SIZE, ObModIds::OB_PERM_INFO));
}
if (NULL != buffer_)
{
int64_t encode_pos = pos;
ret = serialization::encode_i64(buffer_, BUF_SIZE, encode_pos, table_id_);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(ERROR, "encode failed table_id=%ld", table_id_);
}
else
{
memcpy(buffer_ + encode_pos, user_name_, length_);
encode_pos += length_;
rowkey_obj.set_varchar(
ObString(0, static_cast<int32_t>(encode_pos - pos), buffer_ + pos));
rowkey.assign(&rowkey_obj, 1);
get_param.set_is_result_cached(true);
cell_info.table_id_ = PERM_TABLE_ID;
cell_info.row_key_ = rowkey;
cell_info.column_id_ = PERM_COL_ID;
ret = get_param.add_cell(cell_info);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(ERROR, "Get param add cell failed table_id[%lu]", cell_info.table_id_);
}
else
{
ver_range.start_version_ = 0;
ver_range.end_version_ = INT64_MAX - 1;
ver_range.border_flag_.set_inclusive_start();
ver_range.border_flag_.set_inclusive_end();
get_param.set_version_range(ver_range);
}
}
}
else
{
TBSYS_LOG(ERROR, "malloc key buffer failed");
ret = OB_ERROR;
}
return ret;
}
int ObMergerScannerEncoder::encode(const ObGetParam & get_param, const ObScanner & org_scanner,
ObScanner & encoded_scanner)
{
int ret = OB_SUCCESS;
ObCellInfo * cur_cell = NULL;
ObCellInfo * encode_cell = NULL;
int64_t cell_index = 0;
ObScanner & input = const_cast<ObScanner &> (org_scanner);
while (OB_SUCCESS == (ret = input.next_cell()))
{
ret = input.get_cell(&cur_cell);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(WARN, "get cell from org scanner failed:ret[%d]", ret);
break;
}
// output(*cur_cell);
encode_cell = get_param[cell_index];
if (NULL == encode_cell)
{
ret = OB_ERROR;
TBSYS_LOG(WARN, "check encode cell failed:index[%ld]", cell_index);
break;
}
encode_cell->value_ = cur_cell->value_;
ret = encoded_scanner.add_cell(*encode_cell);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(WARN, "add cell to encoded scanner failed:ret[%d]", ret);
break;
}
++cell_index;
}
// double check size
if (cell_index != get_param.get_cell_size())
{
ret = OB_ERROR;
TBSYS_LOG(WARN, "check result scanner cell count not equal get param cell size:"
"count[%ld], size[%ld]", cell_index, get_param.get_cell_size());
}
input.reset_iter();
// fill all data succ
if (OB_ITER_END == ret)
{
ret = OB_SUCCESS;
}
return ret;
}
void read_get_param(const char *fname, const char *section, PageArena<char> &allocer, ObGetParam &get_param)
{
init_opt_map_();
init_obj_type_map_();
TBSYS_CONFIG.load(fname);
const char *str = NULL;
int32_t cell_infos_num = TBSYS_CONFIG.getInt(section, CELL_INFOS_NUM);
str = TBSYS_CONFIG.getString(section, GET_FROZEN_ONLY);
if (0 == strcmp("YES", str))
{
//get_param.set_is_read_frozen_only(true);
}
for (int32_t i = 0; i < cell_infos_num; i++)
{
const char *str = NULL;
int32_t len = 0;
ObCellInfo *ci = (ObCellInfo*)allocer.alloc(sizeof(ObCellInfo));
ci->reset();
// table name
sprintf(buffer, TABLE_NAME_KEY_FORMAT, i);
str = getAndCopySting_(allocer, section, buffer, len);
ci->table_name_.assign_ptr(const_cast<char*>(str), len);
// table id
if (NULL == str)
{
sprintf(buffer, TABLE_ID_KEY_FORMAT, i);
ci->table_id_ = TBSYS_CONFIG.getInt(section, buffer);
}
else
{
ci->table_id_ = OB_INVALID_ID;
}
// row key
sprintf(buffer, ROW_KEY_KEY_FORMAT, i);
str = getAndCopySting_(allocer, section, buffer, len);
ci->row_key_.assign_ptr(const_cast<char*>(str), len);
// column name
sprintf(buffer, COLUMN_NAME_KEY_FORMAT, i);
str = getAndCopySting_(allocer, section, buffer, len);
ci->column_name_.assign_ptr(const_cast<char*>(str), len);
// column id
if (NULL == str)
{
sprintf(buffer, COLUMN_ID_KEY_FORMAT, i);
ci->column_id_ = TBSYS_CONFIG.getInt(section, buffer);
}
else
{
ci->column_id_ = OB_INVALID_ID;
}
get_param.add_cell(*ci);
}
}
int ObTabletJoin::compose_get_param(uint64_t table_id, const ObString &rowkey, ObGetParam &get_param)
{
int ret = OB_SUCCESS;
ObCellInfo cell_info;
cell_info.table_id_ = table_id;
cell_info.row_key_ = rowkey;
JoinInfo join_info;
for(int64_t i=0; (OB_SUCCESS == ret) && i<table_join_info_.join_column_.count();i++)
{
if(OB_SUCCESS != (ret = table_join_info_.join_column_.at(i, join_info)))
{
TBSYS_LOG(WARN, "get join info fail:ret[%d], i[%ld]", ret, i);
}
else
{
cell_info.column_id_ = join_info.right_column_id_;
if(OB_SUCCESS != (ret = get_param.add_cell(cell_info)))
{
TBSYS_LOG(WARN, "add cell info to get_param fail:ret[%d]", ret);
}
}
}
return ret;
}
int ObFinalResult::get(const ObGetParam &get_param, ObScanner &scanner,bool ret_null)
{
int err = OB_SUCCESS;
ObRowInfo row_info;
ObCellInfo cur_cell;
map<ObRowInfo, vector<ObCellInfo> >::iterator target;
bool prev_row_not_exist = false;
for (int64_t get_idx = 0 ; OB_SUCCESS == err && get_idx < get_param.get_cell_size(); get_idx ++)
{
row_info.table_name_ = get_param[get_idx]->table_name_;
row_info.row_key_ = get_param[get_idx]->row_key_;
cur_cell = *(get_param[get_idx]);
target = row_infos_->lower_bound(row_info);
if (get_idx > 0
&& prev_row_not_exist
&& get_param[get_idx]->table_name_ == get_param[get_idx-1]->table_name_
&& get_param[get_idx]->row_key_ == get_param[get_idx-1]->row_key_
)
{
continue;
}
if (row_infos_->end() == target
|| row_info < target->first
|| target->second.size() == 0)
{
prev_row_not_exist = true;
cur_cell.value_.set_ext(ObActionFlag::OP_ROW_DOES_NOT_EXIST);
}
else
{
prev_row_not_exist = false;
vector<ObCellInfo>::iterator beg = target->second.begin();
for (;beg != target->second.end(); beg ++)
{
if (beg->column_name_ == cur_cell.column_name_)
{
cur_cell = *beg;
break;
}
}
if (target->second.end() == beg)
{
if (ret_null)
{
cur_cell.value_.set_null();
}
else
{
cur_cell.value_.set_ext(ObActionFlag::OP_NOP);
}
}
}
cur_cell.table_id_ = OB_INVALID_ID;
cur_cell.column_id_ = OB_INVALID_ID;
err = scanner.add_cell(cur_cell);
}
return err;
}
int MultiGetP::form_get_param(ObGetParam &get_param, const uint32_t min_key_include, const uint32_t max_key_include,
void *&arg)
{
int err = OB_SUCCESS;
arg = NULL;
get_param.reset(true);
char c_name;
ObCellInfo cell;
cell.table_name_ = msolap::target_table_name;
cell.column_name_.assign(&c_name, sizeof(c_name));
int64_t cell_count = get_cell_count;
if (cell_count > max_key_include - min_key_include + 1)
{
cell_count = max_key_include - min_key_include + 1;
}
int64_t random_selector = (max_key_include - min_key_include + 1)/cell_count;
/// bool get_last_cell = false;
uint32_t start_key = min_key_include;
uint32_t start_key_val = 0;
char strkey[32];
for (int64_t cell_idx = 0;
(OB_SUCCESS == err) && (start_key <= max_key_include) && (cell_idx < cell_count);
cell_idx ++,start_key = static_cast<int32_t>(start_key + random_selector))
{
uint32_t cur_key = static_cast<uint32_t>(start_key + random() % random_selector);
c_name = msolap::olap_get_column_name(msolap::min_column_id + random()%(msolap::max_column_id - msolap::min_column_id + 1));
start_key_val = htonl(cur_key);
snprintf(strkey, 32, "%d", start_key_val);
cell.row_key_ = make_rowkey(strkey, &allocator_);
assert(cell.column_name_.ptr()[0] <= 'z' && cell.column_name_.ptr()[0] >= 'a');
if (OB_SUCCESS != (err = get_param.add_cell(cell)))
{
TBSYS_LOG(WARN,"fail to add cell to ObGetParam [err:%d,c_name:%c]", err, c_name);
}
}
if (OB_SUCCESS == err)
{
arg = &get_param;
}
return err;
}
int get_test_case(
BaseClient& client,
const ObServer& server,
const uint64_t table_id,
const int64_t user_id,
const int8_t item_type,
const uint64_t item_id
)
{
ObGetParam input;
const int32_t key_size = 17;
char start_key[key_size] ;
int64_t pos = 0;
encode_i64(start_key, key_size, pos, user_id);
if (item_type <= 1)
encode_i8(start_key, key_size, pos, item_type);
else
encode_i8(start_key, key_size, pos, 0x0);
encode_i64(start_key, key_size, pos, item_id);
hex_dump(start_key, key_size);
ObCellInfo cell;
cell.table_id_ = table_id;
cell.row_key_.assign(start_key, key_size);
cell.column_id_ = 0; //get full row
// cell.value_.set_extend_field(ObActionFlag::OP_READ);
input.add_cell(cell);
ObScanner scanner;
int64_t start = tbsys::CTimeUtil::getTime();
int ret = rpc_cs_get(client.client_, server, input, scanner);
int64_t end = tbsys::CTimeUtil::getTime();
fprintf(stderr,"rpc_cs_get time consume:%ld\n", end - start);
if (OB_SUCCESS != ret)
{
fprintf(stderr,"ret:%d\n", ret);
}
else
{
dump_scanner(scanner);
}
return ret;
}
int ObSKeyInfoKey::build_get_param(ObGetParam& get_param)
{
int ret = OB_SUCCESS;
ObCellInfo cell_info;
ObRowkey rowkey;
ObObj rowkey_obj;
ObVersionRange ver_range;
int64_t pos = 0;
if (NULL == buffer_)
{
buffer_ = reinterpret_cast<char*>(ob_malloc(BUF_SIZE, ObModIds::OB_SKEY_INFO_KEY));
}
ret = serialization::encode_i64(buffer_, BUF_SIZE, pos, skey_version_);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(ERROR, "encode failed pkey_version=%ld", skey_version_);
}
else
{
rowkey_obj.set_varchar(ObString(0, static_cast<int32_t>(pos), buffer_));
rowkey.assign(&rowkey_obj, 1);
get_param.reset();
get_param.set_is_result_cached(true);
cell_info.table_id_ = SKEY_TABLE_ID;
cell_info.row_key_ = rowkey;
cell_info.column_id_ = SKEY_COL_ID;
ret = get_param.add_cell(cell_info);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(ERROR, "Get param add cell failed table_id[%lu]", cell_info.table_id_);
}
else
{
ver_range.start_version_ = 0;
ver_range.end_version_ = INT64_MAX - 1;
ver_range.border_flag_.set_inclusive_start();
ver_range.border_flag_.set_inclusive_end();
get_param.set_version_range(ver_range);
}
}
return ret;
}
int ObMergerPrefetchData::modify_prefetch_param(ObGetParam & param)
{
ObVersionRange range;
int ret = version_.get_version(range.end_version_.version_);
if ((ret != OB_SUCCESS) || (range.end_version_ <= 0))
{
TBSYS_LOG(WARN, "get last version failed:version[%ld], ret[%d]", (int64_t)range.end_version_, ret);
}
else
{
cur_version_ = range.end_version_;
range.border_flag_.set_min_value();
range.border_flag_.set_inclusive_end();
// maybe not need someday
param.set_is_result_cached(true);
param.set_is_read_consistency(true);
param.set_version_range(range);
}
return ret;
}
int ObUserInfoKey:: build_get_param(ObGetParam& get_param)
{
int ret = OB_SUCCESS;
ObCellInfo cell_info;
ObString rowkey;
ObVersionRange ver_range;
if (NULL == buffer_)
{
buffer_ = reinterpret_cast<char*>(ob_malloc(BUF_SIZE));
}
if (NULL != buffer_)
{
memcpy(buffer_, user_name_, length_);
rowkey.assign(buffer_, static_cast<int32_t>(length_));
get_param.set_is_result_cached(true);
cell_info.table_id_ = USER_TABLE_ID;
cell_info.row_key_ = rowkey;
cell_info.column_id_ = USER_COL_ID;
ret = get_param.add_cell(cell_info);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(ERROR, "Get param add cell failed table_id[%lu]", cell_info.table_id_);
}
else
{
ver_range.start_version_ = 0;
ver_range.end_version_ = INT64_MAX - 1;
ver_range.border_flag_.set_inclusive_start();
ver_range.border_flag_.set_inclusive_end();
get_param.set_version_range(ver_range);
}
}
else
{
TBSYS_LOG(ERROR, "malloc key buffer failed");
ret = OB_ERROR;
}
return ret;
}
int GetSingleRowAllColumn::form_get_param(ObGetParam &get_param, const uint32_t min_key_include, const uint32_t max_key_include,
void *&arg)
{
int err = OB_SUCCESS;
arg = NULL;
uint32_t start_key_val = htonl(static_cast<int32_t>(random()%(max_key_include - min_key_include + 1) + min_key_include));
get_param.reset(true);
ObCellInfo cell;
cell.table_name_ = msolap::target_table_name;
cell.column_name_ = ObGetParam::OB_GET_ALL_COLUMN_NAME;
cell.row_key_.assign((char*)&start_key_val,sizeof(start_key_val));
if (OB_SUCCESS != (err = get_param.add_cell(cell)))
{
TBSYS_LOG(WARN,"fail to add cell to ObGetParam [err:%d]", err);
}
if (OB_SUCCESS == err)
{
arg = &get_param;
}
return err;
}
int ObMutatorParamDecoder::add_cell(const ObCellInfo & cell, ObGetParam & get_param)
{
// reset the value at first
ObCellInfo no_value_cell = cell;
no_value_cell.value_.reset();
int ret = get_param.add_cell(no_value_cell);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(WARN, "check add cell to get param failed:ret[%d]", ret);
}
return ret;
}
int get_ups_param(ObGetParam & param, const ObScanner & cs_result)
{
int err = OB_SUCCESS;
ObReadParam &read_param = param;
err = get_ups_read_param(read_param,cs_result);
if (OB_SUCCESS == err)
{
bool is_fullfilled = false;
int64_t fullfilled_cell_num = 0;
err = cs_result.get_is_req_fullfilled(is_fullfilled, fullfilled_cell_num);
if (OB_SUCCESS == err && fullfilled_cell_num < param.get_cell_size())
{
err = param.rollback(param.get_cell_size() - fullfilled_cell_num);
if ((err != OB_SUCCESS) || (param.get_cell_size() != fullfilled_cell_num))
{
TBSYS_LOG(WARN, "check param rollback failed:full_fill[%ld], param_size[%ld]",
fullfilled_cell_num, param.get_cell_size());
}
}
}
return err;
}
int get_test_case_common_hex(
BaseClient& client,
const ObServer& server,
const uint64_t table_id,
const int64_t key_value,
const int64_t key_size
)
{
ObGetParam input;
char start_key[key_size] ;
memset(start_key, 0, key_size);
int64_t pos = 0;
encode_i64(start_key, key_size, pos, key_value);
hex_dump(start_key, static_cast<int32_t>(key_size));
ObCellInfo cell;
cell.table_id_ = table_id;
cell.row_key_.assign((char*)start_key, static_cast<int32_t>(key_size));
cell.column_id_ = 0; //get full row
input.add_cell(cell);
ObScanner scanner;
int64_t start = tbsys::CTimeUtil::getTime();
int ret = rpc_cs_get(client.client_, server, input, scanner);
int64_t end = tbsys::CTimeUtil::getTime();
fprintf(stderr,"rpc_cs_get time consume:%ld\n", end - start);
if (OB_SUCCESS != ret)
{
fprintf(stderr,"ret:%d\n", ret);
}
else
{
dump_scanner(scanner);
}
return ret;
}
int ObGetScanProxy::get_compact_scanner(const ObGetParam& get_param,
ObScanner& orig_scanner,ObScanner& compact_scanner)
{
ObTabletManager::ObGetThreadContext*& get_context = tablet_manager_.get_cur_thread_get_contex();
const ObGetParam::ObRowIndex* row_index = get_param.get_row_index();
int64_t row_num = orig_scanner.get_row_num();
ColumnFilter* cf = GET_TSI_MULT(ColumnFilter,TSI_CS_COLUMNFILTER_1);
ObTablet* tablet = NULL;
int ret = OB_SUCCESS;
ObRowkey rowkey;
if ((NULL == get_context) || (NULL == row_index) || (NULL == cf))
{
TBSYS_LOG(WARN,"get thread context of get failed");
ret = OB_ERROR;
}
else
{
int64_t compactsstable_version = get_context->min_compactsstable_version_;
if (ObVersion::compare(compactsstable_version,orig_scanner.get_data_version()) > 0)
{
compact_scanner.set_data_version(compactsstable_version);
for(int64_t i=0; (i < row_num) && (OB_SUCCESS == ret); ++i)
{
tablet = get_context->tablets_[i];
rowkey = get_param[row_index[i].offset_]->row_key_;
build_get_column_filter(get_param,row_index[i].offset_,row_index[i].size_,*cf);
if ((tablet != NULL) &&
(ret = get_compact_row(*tablet,rowkey,compactsstable_version,cf,compact_scanner)) != OB_SUCCESS)
{
TBSYS_LOG(WARN,"get data from compactsstable failed,ret=%d",ret);
}
}
}
}
return ret;
}
int MockUpdateServer::handle_get_table(ObPacket * ob_packet)
{
int ret = OB_SUCCESS;
ObDataBuffer* data = ob_packet->get_buffer();
if (NULL == data)
{
ret = OB_ERROR;
}
ObGetParam param;
if (OB_SUCCESS == ret)
{
ret = param.deserialize(data->get_data(), data->get_capacity(), data->get_position());
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "%s", "check param failed");
}
}
tbnet::Connection* connection = ob_packet->get_connection();
ThreadSpecificBuffer::Buffer* thread_buffer = response_packet_buffer_.get_buffer();
if (NULL == thread_buffer)
{
ret = OB_ERROR;
}
else
{
thread_buffer->reset();
ObDataBuffer out_buffer(thread_buffer->current(), thread_buffer->remain());
ObResultCode result_msg;
result_msg.result_code_ = ret;
ret = result_msg.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position());
// fake data cell
ObCellInfo cell;
ObScanner scanner;
ObString row_key;
ObString column_name;
char temp[256] = "";
ObString table_name;
char * name = "update_test_table";
table_name.assign(name, strlen(name));
cell.table_name_ = table_name;
for (uint64_t i = 0; i < 10; ++i)
{
snprintf(temp, 256, "update_%lu_get_row_key:%lu", i, i);
row_key.assign(temp, strlen(temp));
cell.row_key_ = row_key;
column_name.assign(temp, strlen(temp));
cell.column_name_ = column_name;
cell.value_.set_int(2234 + i);
scanner.add_cell(cell);
}
int32_t channel_id = ob_packet->getChannelId();
ret = scanner.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position());
//
ret = send_response(OB_SCAN_RESPONSE, 1, out_buffer, connection, channel_id);
}
TBSYS_LOG(INFO, "handle scan root table result:ret[%d]", ret);
return ret;
}
int ObMergerRpcStub::get(
const int64_t timeout, const ObServer & server,
const ObGetParam & get_param, ObScanner & scanner) const
{
int ret = OB_SUCCESS;
ObDataBuffer data_buff;
int64_t start_time = tbsys::CTimeUtil::getTime();
ret = get_rpc_buffer(data_buff);
// step 1. serialize ObGetParam to the data_buff
if (OB_SUCCESS == ret)
{
ret = get_param.serialize(data_buff.get_data(), data_buff.get_capacity(),
data_buff.get_position());
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "serialize get_param failed:ret[%d]", ret);
}
}
// step 2. send request for get data
if (OB_SUCCESS == ret)
{
ret = rpc_frame_->send_request(server, OB_GET_REQUEST, DEFAULT_VERSION,
timeout, data_buff);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(WARN, "send get request to server failed:ret[%d], timeout=%ld",
ret, timeout);
}
}
// step 3. deserialize the response result
int64_t pos = 0;
if (OB_SUCCESS == ret)
{
ObResultCode result_code;
ret = result_code.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "deserialize result failed:pos[%ld], ret[%d]", pos, ret);
}
else
{
ret = result_code.result_code_;
}
}
// step 4. deserialize the scanner
if (OB_SUCCESS == ret)
{
ret = scanner.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "deserialize scanner from buff failed:pos[%ld], ret[%d]", pos, ret);
}
}
const int32_t MAX_SERVER_ADDR_SIZE = 128;
char server_addr[MAX_SERVER_ADDR_SIZE];
bool is_fullfilled = false;
int64_t fullfilled_item_num = 0;
scanner.get_is_req_fullfilled(is_fullfilled,fullfilled_item_num);
server.to_string(server_addr, MAX_SERVER_ADDR_SIZE);
// write debug log
if (OB_SUCCESS == ret)
{
TBSYS_LOG(DEBUG, "get data succ from server:addr[%s]", server_addr);
}
else
{
TBSYS_LOG(WARN, "get data failed from server:addr[%s], ret[%d]", server_addr, ret);
}
int64_t consume_time = tbsys::CTimeUtil::getTime() - start_time;
if ((double)consume_time > (double)timeout * 0.8)
{
TBSYS_LOG(WARN, "slow ups get, ups_addr=%s, timeout=%ld, consume=%ld",
server.to_cstring(), timeout, consume_time);
}
return ret;
}
// maybe not find a merge server
int TaskFactory::get_table_tablet(const char * table_name, const uint64_t table_id, uint64_t & count)
{
int ret = OB_SUCCESS;
ObScanParam scan_param;
uint64_t max_len = 0;
if (NULL == table_name)
{
TBSYS_LOG(ERROR, "check table name failed:table[%s], id[%lu]", table_name, table_id);
ret = OB_ERROR;
}
else
{
ret = init_scan_param(table_name, table_id, max_len, scan_param);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init scan param failed:table[%s], ret[%d]", table_name, ret);
}
}
if (OB_SUCCESS == ret)
{
TaskInfo task;
task.set_table_id(table_id);
task.set_table_name(table_name);
// for the first table tablet
ObString row_key;
char temp_buffer[1];
memset(temp_buffer, 0, sizeof(temp_buffer));
row_key.assign(temp_buffer, sizeof(temp_buffer));
ObGetParam param;
ObScanner scanner;
ObServer server;
ObString start_key;
ObString end_key;
ObCellInfo * cell = NULL;
ObScannerIterator iter;
bool row_change = false;
ObString name;
name.assign(const_cast<char*>(table_name), strlen(table_name));
ObCellInfo temp_cell;
temp_cell.table_id_ = table_id;
temp_cell.column_id_ = 0;
const uint64_t MAX_LEN = 1024;
char last_tablet_rowkey[MAX_LEN] = "";
const int32_t MAX_SERVER_ADDR_SIZE = 128;
char server_addr[MAX_SERVER_ADDR_SIZE];
while ((OB_SUCCESS == ret)
&& (!is_max_rowkey(max_len, row_key.ptr(), row_key.length() - 1)))
{
param.reset();
param.set_is_read_consistency(false);
temp_cell.row_key_ = row_key;
ret = param.add_cell(temp_cell);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "add cell failed:ret[%d]", ret);
break;
}
ret = rpc_->get(root_server_, timeout_, param, scanner);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "get root table for tablet failed:table[%lu], ret[%d]", table_id, ret);
break;
}
else
{
// skip the first row
iter = scanner.begin();
++iter;
while ((iter != scanner.end())
&& (OB_SUCCESS == (ret = iter.get_cell(&cell, &row_change))) && !row_change)
{
if (NULL == cell)
{
TBSYS_LOG(ERROR, "%s", "check cell failed");
ret = OB_INNER_STAT_ERROR;
break;
}
start_key.assign(cell->row_key_.ptr(), cell->row_key_.length());
++iter;
}
}
if (ret == OB_SUCCESS)
{
int64_t ip = 0;
int64_t port = 0;
int64_t version = 0;
TabletLocation list;
for (++iter; iter != scanner.end(); ++iter)
{
ret = iter.get_cell(&cell, &row_change);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret);
break;
}
else if (row_change) // && (iter != last_iter))
{
ret = init_new_task(name, start_key, end_key, scan_param, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret);
break;
}
else
{
ret = insert_new_task(list, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret);
break;
}
++count;
}
list.clear();
start_key = end_key;
end_key.assign(cell->row_key_.ptr(), cell->row_key_.length());
}
else
{
end_key.assign(cell->row_key_.ptr(), cell->row_key_.length());
if ((cell->column_name_.compare("1_ms_port") == 0)
|| (cell->column_name_.compare("2_ms_port") == 0)
|| (cell->column_name_.compare("3_ms_port") == 0))
{
ret = cell->value_.get_int(port);
}
else if ((cell->column_name_.compare("1_ipv4") == 0)
|| (cell->column_name_.compare("2_ipv4") == 0)
|| (cell->column_name_.compare("3_ipv4") == 0))
{
ret = cell->value_.get_int(ip);
}
else if ((cell->column_name_.compare("1_tablet_version") == 0)
|| (cell->column_name_.compare("2_tablet_version") == 0)
|| (cell->column_name_.compare("3_tablet_version") == 0))
{
ret = cell->value_.get_int(version);
if (OB_SUCCESS == ret)
{
if (0 == port || port == 0)
{
TBSYS_LOG(WARN, "%s", "check port or ip failed");
}
else
{
server.set_ipv4_addr(ip, port);
ObTabletLocation addr(version, server);
if (OB_SUCCESS != (ret = list.add(addr)))
{
TBSYS_LOG(ERROR, "add addr failed:server[%d], port[%d], ret[%d]",
server.get_ipv4(), server.get_port(), ret);
}
else
{
server.to_string(server_addr, MAX_SERVER_ADDR_SIZE);
TBSYS_LOG(DEBUG, "add addr succ:server[%s], version:%ld", server_addr, version);
}
}
ip = port = version = 0;
}
}
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret);
break;
}
}
}
// for the last row
if ((OB_SUCCESS == ret) && (start_key != end_key))
{
ret = init_new_task(name, start_key, end_key, scan_param, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret);
}
else
{
ret = insert_new_task(list, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret);
}
++count;
}
if (OB_SUCCESS == ret)
{
// modify last row key for next get root table
uint64_t len = end_key.length();
if (MAX_LEN > len + 1)
{
memcpy(last_tablet_rowkey, end_key.ptr(), len);
last_tablet_rowkey[len] = 0;
row_key.assign(last_tablet_rowkey, len + 1);
}
else
{
TBSYS_LOG(ERROR, "check end key failed:len[%lu]", len);
}
}
}
list.clear();
}
}
}
return ret;
}
bool check_result(const ObGetParam &get_param, ObScanner &ob_scanner,
ObFinalResult & local_result)
{
int err = OB_SUCCESS;
bool res = true;
ObScanner local_scanner;
ObCellInfo *ob_cell = NULL;
ObCellInfo *local_cell = NULL;
err = local_result.get(get_param,local_scanner);
if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN,"fail to get local result");
}
for (int64_t get_idx = 0;
get_idx < get_param.get_cell_size() && OB_SUCCESS == err;
get_idx ++)
{
int ob_err = ob_scanner.next_cell();
int local_err = local_scanner.next_cell();
if (OB_ITER_END == ob_err && OB_ITER_END == local_err)
{
err = OB_SUCCESS;
break;
}
if (OB_SUCCESS != ob_err)
{
err = ob_err;
TBSYS_LOG(WARN,"ob next cell err");
break;
}
if (OB_SUCCESS != local_err)
{
err = local_err;
TBSYS_LOG(WARN,"local next cell err");
break;
}
if (OB_SUCCESS == err)
{
err = ob_scanner.get_cell(&ob_cell);
}
if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN,"fail to get cell from ob result [idx:%lld]", get_idx);
}
if (OB_SUCCESS == err)
{
if (OB_SUCCESS == err)
{
err = local_scanner.get_cell(&local_cell);
}
if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN,"fail to get cell from local result [idx:%lld]", get_idx);
}
}
if (OB_SUCCESS == err)
{
/*
/// check ob result
if (ob_cell->table_name_ != get_param[get_idx]->table_name_)
{
TBSYS_LOG(WARN,"ob result table name error [idx:%lld,ob.table_name_:%.*s,"
"param.table_name_:%.*s]", get_idx,ob_cell->table_name_.length(),
ob_cell->table_name_.ptr(), get_param[get_idx]->table_name_.length(),
get_param[get_idx]->table_name_.ptr());
err = OB_INVALID_ARGUMENT;
}
if (ob_cell->row_key_ != get_param[get_idx]->row_key_)
{
TBSYS_LOG(WARN,"ob result rowkey error [idx:%lld,ob.row_key_:%.*s,"
"param.row_key_:%.*s]", get_idx,ob_cell->row_key_.length(),
ob_cell->row_key_.ptr(), get_param[get_idx]->row_key_.length(),
get_param[get_idx]->row_key_.ptr());
err = OB_INVALID_ARGUMENT;
}
if (ob_cell->column_name_ != get_param[get_idx]->column_name_)
{
TBSYS_LOG(WARN,"ob result column name error [idx:%lld,ob.column_name_:%.*s,"
"param.column_name_:%.*s]", get_idx,ob_cell->column_name_.length(),
ob_cell->column_name_.ptr(), get_param[get_idx]->column_name_.length(),
get_param[get_idx]->column_name_.ptr());
err = OB_INVALID_ARGUMENT;
}
/// check local result
if (local_cell->table_name_ != get_param[get_idx]->table_name_)
{
TBSYS_LOG(WARN,"local result table name error [idx:%lld,ob.table_name_:%.*s,"
"param.table_name_:%.*s]", get_idx,local_cell->table_name_.length(),
local_cell->table_name_.ptr(), get_param[get_idx]->table_name_.length(),
get_param[get_idx]->table_name_.ptr());
err = OB_INVALID_ARGUMENT;
}
if (local_cell->row_key_ != get_param[get_idx]->row_key_)
{
TBSYS_LOG(WARN,"local result rowkey error [idx:%lld,local.row_key_:%.*s,"
"param.row_key_:%.*s]", get_idx,local_cell->row_key_.length(),
local_cell->row_key_.ptr(), get_param[get_idx]->row_key_.length(),
get_param[get_idx]->row_key_.ptr());
err = OB_INVALID_ARGUMENT;
}
if (local_cell->column_name_ != get_param[get_idx]->column_name_)
{
TBSYS_LOG(WARN,"local result column_name error [idx:%lld,local.column_name_:%.*s,"
"param.column_name_:%.*s]", get_idx,local_cell->column_name_.length(),
local_cell->column_name_.ptr(), get_param[get_idx]->column_name_.length(),
get_param[get_idx]->column_name_.ptr());
err = OB_INVALID_ARGUMENT;
}
*/
TBSYS_LOG(DEBUG, "ob.table_name:%.*s,ob.rowkey:%.*s,ob.column_name:%.*s",
ob_cell->table_name_.length(),ob_cell->table_name_.ptr(),
ob_cell->row_key_.length(),ob_cell->row_key_.ptr(),
ob_cell->column_name_.length(),ob_cell->column_name_.ptr()
);
TBSYS_LOG(DEBUG, "local.table_name:%.*s,local.rowkey:%.*s,local.column_name:%.*s",
local_cell->table_name_.length(),local_cell->table_name_.ptr(),
local_cell->row_key_.length(),local_cell->row_key_.ptr(),
local_cell->column_name_.length(),local_cell->column_name_.ptr()
);
if (local_cell->table_name_ != ob_cell->table_name_)
{
TBSYS_LOG(WARN,"table name not coincident [idx:%lld,ob.table_name_:%.*s,"
"local.table_name_:%.*s]", get_idx,ob_cell->table_name_.length(),
ob_cell->table_name_.ptr(), local_cell->table_name_.length(),
local_cell->table_name_.ptr());
err = OB_INVALID_ARGUMENT;
}
if (local_cell->row_key_ != ob_cell->row_key_)
{
TBSYS_LOG(WARN,"rowkey not coincident [idx:%lld,ob.row_key_:%.*s,"
"local.row_key_:%.*s]", get_idx,ob_cell->row_key_.length(),
ob_cell->row_key_.ptr(), local_cell->row_key_.length(),
local_cell->row_key_.ptr());
err = OB_INVALID_ARGUMENT;
}
if (local_cell->column_name_ != ob_cell->column_name_)
{
TBSYS_LOG(WARN,"column name not coincident [idx:%lld,ob.column_name_:%.*s,"
"local.column_name_:%.*s]", get_idx,ob_cell->column_name_.length(),
ob_cell->column_name_.ptr(), local_cell->column_name_.length(),
local_cell->column_name_.ptr());
err = OB_INVALID_ARGUMENT;
}
if (OB_SUCCESS == err)
{
if (ob_cell->value_.get_ext() != local_cell->value_.get_ext())
{
TBSYS_LOG(WARN,"ext info not coincident [idx:%lld,ob.ext:%lld,local.ext:%lld]",
get_idx, ob_cell->value_.get_ext(), local_cell->value_.get_ext());
err = OB_INVALID_ARGUMENT;
}
if (OB_SUCCESS == err && ObExtendType != ob_cell->value_.get_type())
{
int64_t ob_value;
int64_t local_value;
if (ob_cell->value_.get_ext() != local_cell->value_.get_ext())
{
TBSYS_LOG(WARN,"ext not coincident [idx:%lld,ob.ext:%lld,"
"local.ext:%lld]",get_idx, ob_cell->value_.get_ext(),
local_cell->value_.get_ext());
err = OB_INVALID_ARGUMENT;
}
if (ob_cell->value_.get_ext() == 0 && OB_SUCCESS == err)
{
if (ob_cell->value_.get_type() == ObNullType
&& ob_cell->value_.get_type() == local_cell->value_.get_type())
{
/// check pass
}
else
{
if (OB_SUCCESS == err)
{
err = local_cell->value_.get_int(local_value);
if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN,"fail to get int from local, ext:%lld,type:%d",
local_cell->value_.get_ext(),local_cell->value_.get_type());
}
else
{
TBSYS_LOG(DEBUG, "local value:%lld", local_value);
}
}
if (OB_SUCCESS == err)
{
err = ob_cell->value_.get_int(ob_value);
if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN,"fail to get int from ob, ext:%lld,type:%d",
ob_cell->value_.get_ext(),ob_cell->value_.get_type());
}
else
{
TBSYS_LOG(DEBUG, "ob value:%lld", ob_value);
}
}
if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN,"fail to get int value [idx:%lld]", get_idx);
err = OB_INVALID_ARGUMENT;
}
else
{
if (ob_value != local_value)
{
TBSYS_LOG(WARN,"value not coincident [idx:%lld,ob.value:%lld,"
"local.value:%lld]",get_idx, ob_value, local_value);
err = OB_INVALID_ARGUMENT;
}
}
}
}
}
}
}
}
if (OB_SUCCESS == err && ob_scanner.next_cell() != OB_ITER_END)
{
TBSYS_LOG(WARN,"ob return more result than expected");
}
if (OB_SUCCESS == err && local_scanner.next_cell() != OB_ITER_END)
{
TBSYS_LOG(WARN,"local return more result than expected");
}
if (OB_SUCCESS != err)
{
res = false;
TBSYS_LOG(DEBUG, "check fail");
}
else
{
res = true;
TBSYS_LOG(DEBUG, "check pass");
}
return res;
}
int MockChunkServer::handle_get_table(ObPacket * ob_packet)
{
int ret = OB_SUCCESS;
#if 0
ObDataBuffer* data = ob_packet->get_buffer();
if (NULL == data)
{
ret = OB_ERROR;
}
ObGetParam param;
if (OB_SUCCESS == ret)
{
ret = param.deserialize(data->get_data(), data->get_capacity(), data->get_position());
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "%s", "check param failed");
}
}
tbnet::Connection* connection = ob_packet->get_connection();
ThreadSpecificBuffer::Buffer* thread_buffer = response_packet_buffer_.get_buffer();
if (NULL == thread_buffer)
{
ret = OB_ERROR;
}
else
{
thread_buffer->reset();
ObDataBuffer out_buffer(thread_buffer->current(), thread_buffer->remain());
ObResultCode result_msg;
result_msg.result_code_ = ret;
ret = result_msg.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position());
// fake data cell
ObCellInfo cell;
ObNewScanner scanner;
ObRow row;
ObObj obj_a, obj_b, obj_d;
ObObj str_c;
ObString var_str;
var_str.assign("hello", 5);
row_desc_.add_column_desc(test::ObFakeTable::TABLE_ID, OB_APP_MIN_COLUMN_ID + 0);
row_desc_.add_column_desc(test::ObFakeTable::TABLE_ID, OB_APP_MIN_COLUMN_ID + 1);
row_desc_.add_column_desc(test::ObFakeTable::TABLE_ID, OB_APP_MIN_COLUMN_ID + 2);
row_desc_.add_column_desc(test::ObFakeTable::TABLE_ID, OB_APP_MIN_COLUMN_ID + 3);
obj_a.set_int(19);
obj_b.set_int(2);
str_c.set_varchar(var_str);
obj_d.set_int(3);
row_.set_row_desc(row_desc_);
row_.set_cell(test::ObFakeTable::TABLE_ID, OB_APP_MIN_COLUMN_ID + 0, obj_a);
row_.set_cell(test::ObFakeTable::TABLE_ID, OB_APP_MIN_COLUMN_ID + 1, obj_b);
row_.set_cell(test::ObFakeTable::TABLE_ID, OB_APP_MIN_COLUMN_ID + 2, str_c);
row_.set_cell(test::ObFakeTable::TABLE_ID, OB_APP_MIN_COLUMN_ID + 3, obj_d);
ObString row_key;
ObString column_name;
char temp[256] = "";
cell.table_id_ = 101;
for (uint64_t i = 0; i < 10; ++i)
{
const ObRowStore::StoredRow *stored_row = NULL;
ASSERT_EQ(OB_SUCCESS, store_.add_row(row_, stored_row));
ASSERT_TRUE(NULL != stored_row);
/*
snprintf(temp, 256, "chunk_%lu_get_row_key:%lu", i, i);
row_key.assign(temp, static_cast<int32_t>(strlen(temp)));
printf("server:%.*s\n", row_key.length(), row_key.ptr());
cell.row_key_ = row_key;
cell.column_id_ = i + 1;
cell.value_.set_int(2234 + i);
scanner.add_cell(cell);
*/
}
scanner.set_is_req_fullfilled(true, 1);
int32_t channel_id = ob_packet->getChannelId();
ret = scanner.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position());
//
ret = send_response(OB_GET_RESPONSE, 1, out_buffer, connection, channel_id);
}
#endif
TBSYS_LOG(INFO, "handle get table result:ret[%d]", ret);
return ret;
}
int ObClientServerStub::cs_get(const ObGetParam& get_param, ObScanner& scanner)
{
int ret = OB_SUCCESS;
const int64_t timeout = 1000000; // send_request timeout millionseconds
ObDataBuffer data_buff;
ret = get_frame_buffer(data_buff);
// step 1. serialize timestamp to data_buff
if (OB_SUCCESS == ret)
{
ret = get_param.serialize(data_buff.get_data(),
data_buff.get_capacity(), data_buff.get_position());
if (OB_SUCCESS != ret)
{
TBSYS_LOG(ERROR, "serialize get_param failed:ret[%d].", ret);
}
}
// step 2. send request for fetch new schema
if (OB_SUCCESS == ret)
{
uint32_t pos = random() % merge_server_list_.size();
ObServer &cur_server = merge_server_list_.at(pos);
ret = rpc_frame_->send_request(cur_server,
OB_GET_REQUEST, DEFAULT_VERSION, timeout, data_buff);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "send request to remote server for get failed"
" ret[%d].", ret);
}
}
// step 3. deserialize the response code
int64_t pos = 0;
if (OB_SUCCESS == ret)
{
ObResultCode result_code;
ret = result_code.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(ERROR, "deserialize result_code failed:pos[%ld], ret[%d].", pos, ret);
}
else
{
ret = result_code.result_code_;
}
}
// step 4. deserialize the table schema
if (OB_SUCCESS == ret)
{
ret = scanner.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(ERROR, "deserialize scanner from buff failed"
"pos[%ld], ret[%d].", pos, ret);
}
}
return ret;
}
int rpc_cs_get(ObClientManager& cp,
const ObServer& cs,
const ObGetParam& get_param,
ObScanner& scanner)
{
int ret = OB_SUCCESS;
int64_t start = 0, end = 0;
const int32_t BUFFER_SIZE = 2*1024*1024;
char* param_buffer = new char[BUFFER_SIZE];
ObDataBuffer ob_inout_buffer;
ObResultCode rc;
int64_t return_start_pos = 0;
if (NULL == param_buffer)
{
goto exit;
}
ob_inout_buffer.set_data(param_buffer, BUFFER_SIZE);
ret = get_param.serialize(ob_inout_buffer.get_data(),
ob_inout_buffer.get_capacity(), ob_inout_buffer.get_position());
if (OB_SUCCESS != ret)
{
fprintf(stderr,"serialize get_param into buffer failed\n");
goto exit;
}
// send request;
start = tbsys::CTimeUtil::getTime();
ret = cp.send_request(cs, OB_GET_REQUEST, 1, 2000*2000, ob_inout_buffer);
end = tbsys::CTimeUtil::getTime();
fprintf(stderr,"time consume:%ld\n", end - start);
if (OB_SUCCESS != ret)
{
fprintf(stderr,"rpc failed\n");
goto exit;
}
ret = rc.deserialize(ob_inout_buffer.get_data(),
ob_inout_buffer.get_position(), return_start_pos);
if (OB_SUCCESS != ret)
{
fprintf(stderr,"deserialize obscanner failed\n");
goto exit;
}
fprintf(stderr,"return rc code:%d, msg:%s\n", rc.result_code_, rc.message_.ptr());
if (OB_SUCCESS != rc.result_code_)
{
goto exit;
}
fprintf(stderr,"return_start_pos:%ld, %ld\n", return_start_pos, ob_inout_buffer.get_position());
// deserialize output
ret = scanner.deserialize(ob_inout_buffer.get_data(),
ob_inout_buffer.get_position(), return_start_pos);
if (OB_SUCCESS != ret)
{
fprintf(stderr,"deserialize obscanner failed\n");
goto exit;
}
fprintf(stderr,"return_start_pos:%ld, %ld\n", return_start_pos, ob_inout_buffer.get_position());
exit:
if (param_buffer) delete []param_buffer;
return ret;
}
int ObGetRequestEvent::setup_new_request(const bool retry, const ObGetParam & get_param)
{
int ret = OB_SUCCESS;
ObMergerRpcEvent * event = NULL;
// step 1. create new rpc event and add to the waiting queue
if (false == check_inner_stat())
{
ret = OB_INNER_STAT_ERROR;
TBSYS_LOG(WARN, "check inner stat failed");
}
else
{
ret = ObMergerRequestEvent::create(&event);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "add new rpc event failed:request[%lu], ret[%d]", get_request_id(), ret);
}
}
// TODO check retry with other cs
UNUSED(retry);
ObMergerTabletLocationList list;
// step 3. select the right cs for request
if ((OB_SUCCESS == ret)) // && (false == retry))
{
ret = get_cache_proxy()->get_tablet_location(get_param[0]->table_id_, get_param[0]->row_key_, list);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(WARN, "get tablet location failed:client[%lu], event[%lu], request[%lu], ret[%d]",
event->get_client_id(), event->get_event_id(), get_request_id(), ret);
}
}
// step 4. send reqeust for get
if (OB_SUCCESS == ret)
{
// TODO access list[0]
event->set_server(list[0].server_.chunkserver_);
ret = get_rpc()->get(get_timeout(), list[0].server_.chunkserver_, get_param, *event);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "check async rpc stub failed:client[%lu], event[%lu], request[%lu], ret[%d]",
event->get_client_id(), event->get_event_id(), get_request_id(), ret);
}
else
{
TBSYS_LOG(DEBUG, "send get param to server succ:client[%lu], event[%lu], request[%lu], "
"get_cell[%ld]", event->get_client_id(), event->get_event_id(), get_request_id(),
get_param.get_cell_size());
}
}
/// if not send succ
if ((event != NULL) && (ret != OB_SUCCESS))
{
uint64_t client_id = event->get_client_id();
uint64_t event_id = event->get_event_id();
int err = ObMergerRequestEvent::destroy(event);
if (err != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "destroy the event failed when rpc send failed:client[%lu], event[%lu], "
"request[%lu], ret[%d], err[%d]", client_id, event_id, get_request_id(), ret, err);
}
else
{
TBSYS_LOG(INFO, "destroy directly succ:client[%lu], event[%lu], request[%lu], ret[%d]",
client_id, event_id, get_request_id(), ret);
}
}
return ret;
}
int ObServerRpc::get(const ObServer& remote_server,
const ObGetParam& get_param,
ObScanner& scanner,
const int64_t timeout)
{
int ret = OB_SUCCESS;
int64_t pos = 0;
ObResultCode result_code;
ObDataBuffer data_buff;
if (remote_server.get_ipv4() == 0 || remote_server.get_port() == 0
|| timeout <= 0)
{
TBSYS_LOG(WARN, "invalid param, ip=%d, port=%d, timeout=%ld",
remote_server.get_ipv4(), remote_server.get_port(),
timeout);
ret = OB_ERROR;
}
else if (get_param.get_cell_size() <= 0 || get_param.get_row_size() <= 0)
{
TBSYS_LOG(WARN, "invalid param, cell_size=%ld, row_size=%ld",
get_param.get_cell_size(), get_param.get_row_size());
ret = OB_ERROR;
}
else if (NULL == rpc_frame_)
{
TBSYS_LOG(WARN, "server rpc doesn't init.");
ret = OB_ERROR;
}
if (OB_SUCCESS == ret)
{
ret = get_frame_buffer(data_buff);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "get frame buffer failed, ret=%d.", ret);
}
}
// step 1. serialize get param to data_buff
if (OB_SUCCESS == ret)
{
ret = get_param.serialize(data_buff.get_data(),
data_buff.get_capacity(),
data_buff.get_position());
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "serialize get_param failed, ret=%d.", ret);
}
}
// step 2. send request for get
if (OB_SUCCESS == ret)
{
ret = rpc_frame_->send_request(remote_server,
OB_GET_REQUEST, DEFAULT_VERSION,
timeout, data_buff);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "send request to remote server for get failed, "
"ret=%d.", ret);
}
}
// step 3. deserialize the response code
if (OB_SUCCESS == ret)
{
ret = result_code.deserialize(data_buff.get_data(),
data_buff.get_position(), pos);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "deserialize result_code failed,pos=%ld, ret=%d.",
pos, ret);
}
else
{
ret = result_code.result_code_;
if (OB_SUCCESS != ret && OB_DATA_NOT_SERVE != ret)
{
TBSYS_LOG(WARN, "get response from remote server failed, "
"timeout=%ld, ret=%d.", timeout, ret);
}
}
}
// step 4. deserialize the scanner
if (OB_SUCCESS == ret)
{
ret = scanner.deserialize(data_buff.get_data(),
data_buff.get_position(), pos);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "deserialize scanner from buff failed, "
"pos=%ld, ret=%d.", pos, ret);
}
}
return ret;
}
int MutatorBuilder::build_get_param(ObGetParam &get_param, PageArena<char> &allocer,
const int64_t table_start_version, const bool using_id,
const int64_t row_num, const int64_t cell_num_per_row)
{
int ret = OB_SUCCESS;
get_param.reset();
UNUSED(table_start_version);
static int64_t row_info_array[OB_MAX_TABLE_NUMBER];
static int64_t suffix_num_array[OB_MAX_TABLE_NUMBER];
static int64_t counter = 0;
static const int64_t ROW_INFO_CACHE_FLUSH_PERIOD = 100;
//static const int64_t ROW_INFO_CACHE_FLUSH_PERIOD = 1000000000;
if (0 == (counter++ % ROW_INFO_CACHE_FLUSH_PERIOD))
{
memset(row_info_array, 0, sizeof(row_info_array));
memset(suffix_num_array, 0, sizeof(suffix_num_array));
}
for (int64_t i = 0; i < row_num; i++)
{
int64_t rand = 0;
lrand48_r(&table_rand_seed_, &rand);
int64_t table_pos = range_rand(0, table_num_ - 1, rand);
const ObSchema &cur_schema = schema_mgr_.begin()[table_pos];
if (0 == row_info_array[table_pos]
&& 0 == (row_info_array[table_pos] = get_cur_rowkey_info_(cur_schema, using_id, rb_array_[table_pos]->get_prefix_start())))
{
assert(OB_SUCCESS == ret);
ret = OB_ERROR;
break;
}
if (0 == suffix_num_array[table_pos]
&& 0 == (suffix_num_array[table_pos] = get_suffix_num_(cur_schema, using_id, rb_array_[table_pos]->get_prefix_start())))
{
assert(OB_SUCCESS == ret);
ret = OB_ERROR;
break;
}
ObCellInfo ci;
ci.row_key_ = rb_array_[table_pos]->get_random_rowkey(row_info_array[table_pos], suffix_num_array[table_pos], allocer);
if (using_id)
{
ci.table_id_ = cur_schema.get_table_id();
}
else
{
ci.table_name_.assign_ptr(const_cast<char*>(cur_schema.get_table_name()), static_cast<int32_t>(strlen(cur_schema.get_table_name())));
}
if (using_id)
{
ci.column_id_ = SEED_COLUMN_ID;
ci.column_name_.assign_ptr(NULL, 0);
}
else
{
ci.column_id_ = OB_INVALID_ID;
ci.column_name_.assign_ptr((char*)SEED_COLUMN_NAME, static_cast<int32_t>(strlen(SEED_COLUMN_NAME)));
}
get_param.add_cell(ci);
if (using_id)
{
ci.column_id_ = CELL_NUM_COLUMN_ID;
ci.column_name_.assign_ptr(NULL, 0);
}
else
{
ci.column_id_ = OB_INVALID_ID;
ci.column_name_.assign_ptr((char*)CELL_NUM_COLUMN_NAME, static_cast<int32_t>(strlen(CELL_NUM_COLUMN_NAME)));
}
get_param.add_cell(ci);
lrand48_r(&table_rand_seed_, &rand);
int64_t cell_num = range_rand(1, cell_num_per_row, rand);
int64_t column_num = cur_schema.column_end() - cur_schema.column_begin();
for (int64_t j = 0; j < cell_num; j++)
{
lrand48_r(&table_rand_seed_, &rand);
int64_t column_pos = range_rand(0, column_num - 1, rand);
//int64_t column_pos = 0;
const ObColumnSchema &column_schema = cur_schema.column_begin()[column_pos];
if (using_id)
{
ci.column_id_ = column_schema.get_id();
ci.column_name_.assign_ptr(NULL, 0);
}
else
{
ci.column_id_ = OB_INVALID_ID;
ci.column_name_.assign_ptr(const_cast<char*>(column_schema.get_name()), static_cast<int32_t>(strlen(column_schema.get_name())));
}
get_param.add_cell(ci);
}
}
ObVersionRange version_range;
version_range.start_version_ = table_start_version_;
version_range.border_flag_.set_max_value();
version_range.border_flag_.set_inclusive_start();
get_param.set_version_range(version_range);
char* is_read_consistency_cfg = getenv("is_read_consistency");
bool is_read_consistency = (NULL != is_read_consistency_cfg && 0 == strcmp(is_read_consistency_cfg, "true"));
get_param.set_is_read_consistency(is_read_consistency);
assert(get_param.get_cell_size() > 0);
return ret;
}
// maybe not find a merge server
int TaskFactory::get_table_tablet(const char * table_name, const uint64_t table_id, uint64_t & count)
{
int ret = OB_SUCCESS;
ObScanParam scan_param;
uint64_t max_len = 0;
if (NULL == table_name)
{
TBSYS_LOG(ERROR, "check table name failed:table[%s], id[%lu]", table_name, table_id);
ret = OB_ERROR;
}
else
{
ret = init_scan_param(table_name, table_id, max_len, scan_param);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init scan param failed:table[%s], ret[%d]", table_name, ret);
}
}
if (OB_SUCCESS == ret)
{
// for the first table tablet
ObRowkey row_key;
ObObj temp_buffer[common::OB_MAX_ROWKEY_COLUMN_NUMBER];
row_key.assign(temp_buffer, common::OB_MAX_ROWKEY_COLUMN_NUMBER);
ObGetParam param;
ObScanner scanner;
ObServer server;
ObRowkey start_key;
ObRowkey end_key;
ObCellInfo * cell = NULL;
ObScannerIterator iter;
bool row_change = false;
ObString name;
name.assign(const_cast<char*>(table_name), (int32_t)strlen(table_name));
ObCellInfo temp_cell;
temp_cell.table_id_ = table_id;
temp_cell.column_id_ = 0;
common::ModuleArena allocator;
while ((OB_SUCCESS == ret) && (!row_key.is_max_row()))
{
param.reset();
param.set_is_read_consistency(false);
temp_cell.row_key_ = row_key;
ret = param.add_cell(temp_cell);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "add cell failed:ret[%d]", ret);
break;
}
ret = rpc_->get(root_server_, timeout_, param, scanner);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "get root table for tablet failed:table[%lu], ret[%d]", table_id, ret);
break;
}
else
{
// skip the first row
iter = scanner.begin();
++iter;
while ((iter != scanner.end())
&& (OB_SUCCESS == (ret = iter.get_cell(&cell, &row_change))) && !row_change)
{
if (NULL == cell)
{
TBSYS_LOG(ERROR, "%s", "check cell failed");
ret = OB_INNER_STAT_ERROR;
break;
}
ret = cell->row_key_.deep_copy(start_key, allocator);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(WARN, "deep copy the rowkey failed:ret[%d]", ret);
}
++iter;
}
}
// scanner iter end
if (ret == OB_SUCCESS)
{
int64_t ip = 0;
int64_t port = 0;
int64_t version = 0;
TabletLocation list;
for (++iter; iter != scanner.end(); ++iter)
{
ret = iter.get_cell(&cell, &row_change);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret);
break;
}
else if (row_change) // && (iter != last_iter))
{
TaskInfo task;
task.set_table_id(table_id);
task.set_table_name(table_name);
ret = init_new_task(name, start_key, end_key, scan_param, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret);
break;
}
else
{
ret = insert_new_task(list, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret);
break;
}
++count;
}
list.clear();
start_key = end_key;
}
else
{
cell->row_key_.deep_copy(end_key, allocator);
if ((cell->column_name_.compare("1_ms_port") == 0)
|| (cell->column_name_.compare("2_ms_port") == 0)
|| (cell->column_name_.compare("3_ms_port") == 0))
{
ret = cell->value_.get_int(port);
}
else if ((cell->column_name_.compare("1_ipv4") == 0)
|| (cell->column_name_.compare("2_ipv4") == 0)
|| (cell->column_name_.compare("3_ipv4") == 0))
{
ret = cell->value_.get_int(ip);
}
else if ((cell->column_name_.compare("1_tablet_version") == 0)
|| (cell->column_name_.compare("2_tablet_version") == 0)
|| (cell->column_name_.compare("3_tablet_version") == 0))
{
ret = cell->value_.get_int(version);
if (OB_SUCCESS == ret)
{
if (0 == port || port == 0)
{
TBSYS_LOG(WARN, "%s", "check port or ip failed");
}
else
{
server.set_ipv4_addr(int32_t(ip), int32_t(port));
ObTabletLocation addr(version, server);
if (OB_SUCCESS != (ret = list.add(addr)))
{
TBSYS_LOG(ERROR, "add addr failed:server[%d], port[%d], ret[%d]",
server.get_ipv4(), server.get_port(), ret);
}
else
{
TBSYS_LOG(DEBUG, "add addr succ:server[%s], version:%ld", server.to_cstring(), version);
}
}
ip = port = version = 0;
}
}
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret);
break;
}
}
}
// for the last row
if ((OB_SUCCESS == ret) && (start_key != end_key))
{
TaskInfo task;
task.set_table_id(table_id);
task.set_table_name(table_name);
ret = init_new_task(name, start_key, end_key, scan_param, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret);
}
else
{
ret = insert_new_task(list, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret);
}
++count;
}
if (OB_SUCCESS == ret)
{
// copy all the end key objs to row_key
for (int64_t i = 0; i < end_key.length(); ++i)
{
const_cast<ObObj *> (row_key.ptr())[i] = end_key.ptr()[i];
row_key.assign(const_cast<ObObj *> (row_key.ptr()), end_key.length());
}
// modify last row key for next get root table
if (!end_key.is_max_row())
{
const_cast<ObObj *>(row_key.ptr())[end_key.length()].set_max_value();
row_key.assign(const_cast<ObObj *> (row_key.ptr()), end_key.length() + 1);
}
}
}
list.clear();
}
}
}
return ret;
}
