bool check_result(const ObGetParam &get_param, ObScanner &ob_scanner,
ObFinalResult & local_result)
{
int err = OB_SUCCESS;
bool res = true;
ObScanner local_scanner;
ObCellInfo *ob_cell = NULL;
ObCellInfo *local_cell = NULL;
err = local_result.get(get_param,local_scanner);
if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN,"fail to get local result");
}
for (int64_t get_idx = 0;
get_idx < get_param.get_cell_size() && OB_SUCCESS == err;
get_idx ++)
{
int ob_err = ob_scanner.next_cell();
int local_err = local_scanner.next_cell();
if (OB_ITER_END == ob_err && OB_ITER_END == local_err)
{
err = OB_SUCCESS;
break;
}
if (OB_SUCCESS != ob_err)
{
err = ob_err;
TBSYS_LOG(WARN,"ob next cell err");
break;
}
if (OB_SUCCESS != local_err)
{
err = local_err;
TBSYS_LOG(WARN,"local next cell err");
break;
}
if (OB_SUCCESS == err)
{
err = ob_scanner.get_cell(&ob_cell);
}
if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN,"fail to get cell from ob result [idx:%lld]", get_idx);
}
if (OB_SUCCESS == err)
{
if (OB_SUCCESS == err)
{
err = local_scanner.get_cell(&local_cell);
}
if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN,"fail to get cell from local result [idx:%lld]", get_idx);
}
}
if (OB_SUCCESS == err)
{
/*
/// check ob result
if (ob_cell->table_name_ != get_param[get_idx]->table_name_)
{
TBSYS_LOG(WARN,"ob result table name error [idx:%lld,ob.table_name_:%.*s,"
"param.table_name_:%.*s]", get_idx,ob_cell->table_name_.length(),
ob_cell->table_name_.ptr(), get_param[get_idx]->table_name_.length(),
get_param[get_idx]->table_name_.ptr());
err = OB_INVALID_ARGUMENT;
}
if (ob_cell->row_key_ != get_param[get_idx]->row_key_)
{
TBSYS_LOG(WARN,"ob result rowkey error [idx:%lld,ob.row_key_:%.*s,"
"param.row_key_:%.*s]", get_idx,ob_cell->row_key_.length(),
ob_cell->row_key_.ptr(), get_param[get_idx]->row_key_.length(),
get_param[get_idx]->row_key_.ptr());
err = OB_INVALID_ARGUMENT;
}
if (ob_cell->column_name_ != get_param[get_idx]->column_name_)
{
TBSYS_LOG(WARN,"ob result column name error [idx:%lld,ob.column_name_:%.*s,"
"param.column_name_:%.*s]", get_idx,ob_cell->column_name_.length(),
ob_cell->column_name_.ptr(), get_param[get_idx]->column_name_.length(),
get_param[get_idx]->column_name_.ptr());
err = OB_INVALID_ARGUMENT;
}
/// check local result
if (local_cell->table_name_ != get_param[get_idx]->table_name_)
{
TBSYS_LOG(WARN,"local result table name error [idx:%lld,ob.table_name_:%.*s,"
"param.table_name_:%.*s]", get_idx,local_cell->table_name_.length(),
local_cell->table_name_.ptr(), get_param[get_idx]->table_name_.length(),
get_param[get_idx]->table_name_.ptr());
err = OB_INVALID_ARGUMENT;
}
if (local_cell->row_key_ != get_param[get_idx]->row_key_)
{
TBSYS_LOG(WARN,"local result rowkey error [idx:%lld,local.row_key_:%.*s,"
"param.row_key_:%.*s]", get_idx,local_cell->row_key_.length(),
local_cell->row_key_.ptr(), get_param[get_idx]->row_key_.length(),
get_param[get_idx]->row_key_.ptr());
err = OB_INVALID_ARGUMENT;
}
if (local_cell->column_name_ != get_param[get_idx]->column_name_)
{
TBSYS_LOG(WARN,"local result column_name error [idx:%lld,local.column_name_:%.*s,"
"param.column_name_:%.*s]", get_idx,local_cell->column_name_.length(),
local_cell->column_name_.ptr(), get_param[get_idx]->column_name_.length(),
get_param[get_idx]->column_name_.ptr());
err = OB_INVALID_ARGUMENT;
}
*/
TBSYS_LOG(DEBUG, "ob.table_name:%.*s,ob.rowkey:%.*s,ob.column_name:%.*s",
ob_cell->table_name_.length(),ob_cell->table_name_.ptr(),
ob_cell->row_key_.length(),ob_cell->row_key_.ptr(),
ob_cell->column_name_.length(),ob_cell->column_name_.ptr()
);
TBSYS_LOG(DEBUG, "local.table_name:%.*s,local.rowkey:%.*s,local.column_name:%.*s",
local_cell->table_name_.length(),local_cell->table_name_.ptr(),
local_cell->row_key_.length(),local_cell->row_key_.ptr(),
local_cell->column_name_.length(),local_cell->column_name_.ptr()
);
if (local_cell->table_name_ != ob_cell->table_name_)
{
TBSYS_LOG(WARN,"table name not coincident [idx:%lld,ob.table_name_:%.*s,"
"local.table_name_:%.*s]", get_idx,ob_cell->table_name_.length(),
ob_cell->table_name_.ptr(), local_cell->table_name_.length(),
local_cell->table_name_.ptr());
err = OB_INVALID_ARGUMENT;
}
if (local_cell->row_key_ != ob_cell->row_key_)
{
TBSYS_LOG(WARN,"rowkey not coincident [idx:%lld,ob.row_key_:%.*s,"
"local.row_key_:%.*s]", get_idx,ob_cell->row_key_.length(),
ob_cell->row_key_.ptr(), local_cell->row_key_.length(),
local_cell->row_key_.ptr());
err = OB_INVALID_ARGUMENT;
}
if (local_cell->column_name_ != ob_cell->column_name_)
{
TBSYS_LOG(WARN,"column name not coincident [idx:%lld,ob.column_name_:%.*s,"
"local.column_name_:%.*s]", get_idx,ob_cell->column_name_.length(),
ob_cell->column_name_.ptr(), local_cell->column_name_.length(),
local_cell->column_name_.ptr());
err = OB_INVALID_ARGUMENT;
}
if (OB_SUCCESS == err)
{
if (ob_cell->value_.get_ext() != local_cell->value_.get_ext())
{
TBSYS_LOG(WARN,"ext info not coincident [idx:%lld,ob.ext:%lld,local.ext:%lld]",
get_idx, ob_cell->value_.get_ext(), local_cell->value_.get_ext());
err = OB_INVALID_ARGUMENT;
}
if (OB_SUCCESS == err && ObExtendType != ob_cell->value_.get_type())
{
int64_t ob_value;
int64_t local_value;
if (ob_cell->value_.get_ext() != local_cell->value_.get_ext())
{
TBSYS_LOG(WARN,"ext not coincident [idx:%lld,ob.ext:%lld,"
"local.ext:%lld]",get_idx, ob_cell->value_.get_ext(),
local_cell->value_.get_ext());
err = OB_INVALID_ARGUMENT;
}
if (ob_cell->value_.get_ext() == 0 && OB_SUCCESS == err)
{
if (ob_cell->value_.get_type() == ObNullType
&& ob_cell->value_.get_type() == local_cell->value_.get_type())
{
/// check pass
}
else
{
if (OB_SUCCESS == err)
{
err = local_cell->value_.get_int(local_value);
if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN,"fail to get int from local, ext:%lld,type:%d",
local_cell->value_.get_ext(),local_cell->value_.get_type());
}
else
{
TBSYS_LOG(DEBUG, "local value:%lld", local_value);
}
}
if (OB_SUCCESS == err)
{
err = ob_cell->value_.get_int(ob_value);
if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN,"fail to get int from ob, ext:%lld,type:%d",
ob_cell->value_.get_ext(),ob_cell->value_.get_type());
}
else
{
TBSYS_LOG(DEBUG, "ob value:%lld", ob_value);
}
}
if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN,"fail to get int value [idx:%lld]", get_idx);
err = OB_INVALID_ARGUMENT;
}
else
{
if (ob_value != local_value)
{
TBSYS_LOG(WARN,"value not coincident [idx:%lld,ob.value:%lld,"
"local.value:%lld]",get_idx, ob_value, local_value);
err = OB_INVALID_ARGUMENT;
}
}
}
}
}
}
}
}
if (OB_SUCCESS == err && ob_scanner.next_cell() != OB_ITER_END)
{
TBSYS_LOG(WARN,"ob return more result than expected");
}
if (OB_SUCCESS == err && local_scanner.next_cell() != OB_ITER_END)
{
TBSYS_LOG(WARN,"local return more result than expected");
}
if (OB_SUCCESS != err)
{
res = false;
TBSYS_LOG(DEBUG, "check fail");
}
else
{
res = true;
TBSYS_LOG(DEBUG, "check pass");
}
return res;
}
// maybe not find a merge server
int TaskFactory::get_table_tablet(const char * table_name, const uint64_t table_id, uint64_t & count)
{
int ret = OB_SUCCESS;
ObScanParam scan_param;
uint64_t max_len = 0;
if (NULL == table_name)
{
TBSYS_LOG(ERROR, "check table name failed:table[%s], id[%lu]", table_name, table_id);
ret = OB_ERROR;
}
else
{
ret = init_scan_param(table_name, table_id, max_len, scan_param);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init scan param failed:table[%s], ret[%d]", table_name, ret);
}
}
if (OB_SUCCESS == ret)
{
TaskInfo task;
task.set_table_id(table_id);
task.set_table_name(table_name);
// for the first table tablet
ObString row_key;
char temp_buffer[1];
memset(temp_buffer, 0, sizeof(temp_buffer));
row_key.assign(temp_buffer, sizeof(temp_buffer));
ObGetParam param;
ObScanner scanner;
ObServer server;
ObString start_key;
ObString end_key;
ObCellInfo * cell = NULL;
ObScannerIterator iter;
bool row_change = false;
ObString name;
name.assign(const_cast<char*>(table_name), strlen(table_name));
ObCellInfo temp_cell;
temp_cell.table_id_ = table_id;
temp_cell.column_id_ = 0;
const uint64_t MAX_LEN = 1024;
char last_tablet_rowkey[MAX_LEN] = "";
const int32_t MAX_SERVER_ADDR_SIZE = 128;
char server_addr[MAX_SERVER_ADDR_SIZE];
while ((OB_SUCCESS == ret)
&& (!is_max_rowkey(max_len, row_key.ptr(), row_key.length() - 1)))
{
param.reset();
param.set_is_read_consistency(false);
temp_cell.row_key_ = row_key;
ret = param.add_cell(temp_cell);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "add cell failed:ret[%d]", ret);
break;
}
ret = rpc_->get(root_server_, timeout_, param, scanner);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "get root table for tablet failed:table[%lu], ret[%d]", table_id, ret);
break;
}
else
{
// skip the first row
iter = scanner.begin();
++iter;
while ((iter != scanner.end())
&& (OB_SUCCESS == (ret = iter.get_cell(&cell, &row_change))) && !row_change)
{
if (NULL == cell)
{
TBSYS_LOG(ERROR, "%s", "check cell failed");
ret = OB_INNER_STAT_ERROR;
break;
}
start_key.assign(cell->row_key_.ptr(), cell->row_key_.length());
++iter;
}
}
if (ret == OB_SUCCESS)
{
int64_t ip = 0;
int64_t port = 0;
int64_t version = 0;
TabletLocation list;
for (++iter; iter != scanner.end(); ++iter)
{
ret = iter.get_cell(&cell, &row_change);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret);
break;
}
else if (row_change) // && (iter != last_iter))
{
ret = init_new_task(name, start_key, end_key, scan_param, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret);
break;
}
else
{
ret = insert_new_task(list, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret);
break;
}
++count;
}
list.clear();
start_key = end_key;
end_key.assign(cell->row_key_.ptr(), cell->row_key_.length());
}
else
{
end_key.assign(cell->row_key_.ptr(), cell->row_key_.length());
if ((cell->column_name_.compare("1_ms_port") == 0)
|| (cell->column_name_.compare("2_ms_port") == 0)
|| (cell->column_name_.compare("3_ms_port") == 0))
{
ret = cell->value_.get_int(port);
}
else if ((cell->column_name_.compare("1_ipv4") == 0)
|| (cell->column_name_.compare("2_ipv4") == 0)
|| (cell->column_name_.compare("3_ipv4") == 0))
{
ret = cell->value_.get_int(ip);
}
else if ((cell->column_name_.compare("1_tablet_version") == 0)
|| (cell->column_name_.compare("2_tablet_version") == 0)
|| (cell->column_name_.compare("3_tablet_version") == 0))
{
ret = cell->value_.get_int(version);
if (OB_SUCCESS == ret)
{
if (0 == port || port == 0)
{
TBSYS_LOG(WARN, "%s", "check port or ip failed");
}
else
{
server.set_ipv4_addr(ip, port);
ObTabletLocation addr(version, server);
if (OB_SUCCESS != (ret = list.add(addr)))
{
TBSYS_LOG(ERROR, "add addr failed:server[%d], port[%d], ret[%d]",
server.get_ipv4(), server.get_port(), ret);
}
else
{
server.to_string(server_addr, MAX_SERVER_ADDR_SIZE);
TBSYS_LOG(DEBUG, "add addr succ:server[%s], version:%ld", server_addr, version);
}
}
ip = port = version = 0;
}
}
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret);
break;
}
}
}
// for the last row
if ((OB_SUCCESS == ret) && (start_key != end_key))
{
ret = init_new_task(name, start_key, end_key, scan_param, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret);
}
else
{
ret = insert_new_task(list, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret);
}
++count;
}
if (OB_SUCCESS == ret)
{
// modify last row key for next get root table
uint64_t len = end_key.length();
if (MAX_LEN > len + 1)
{
memcpy(last_tablet_rowkey, end_key.ptr(), len);
last_tablet_rowkey[len] = 0;
row_key.assign(last_tablet_rowkey, len + 1);
}
else
{
TBSYS_LOG(ERROR, "check end key failed:len[%lu]", len);
}
}
}
list.clear();
}
}
}
return ret;
}
int ObFinalResult::scan(const ObScanParam &scan_param, ObScanner &scanner)
{
UNUSED(scanner);
int err = OB_SUCCESS;
char max_key_buf[512];
memset(max_key_buf,0xff, sizeof(max_key_buf));
ObString max_key;
max_key.assign(max_key_buf,sizeof(max_key_buf));
ObRowInfo start;
ObRowInfo end;
map<ObRowInfo, vector<ObCellInfo> >::iterator start_it;
map<ObRowInfo, vector<ObCellInfo> >::iterator end_it;
ObCellInfo cur_cell;
start.table_name_ = scan_param.get_table_name();
end.table_name_ = scan_param.get_table_name();
if (!scan_param.get_range()->border_flag_.is_min_value())
{
start.row_key_ = scan_param.get_range()->start_key_;
}
if (!scan_param.get_range()->border_flag_.is_max_value())
{
end.row_key_ = scan_param.get_range()->end_key_;
}
else
{
end.row_key_ = max_key;
}
if (scan_param.get_range()->border_flag_.inclusive_start())
{
start_it = row_infos_->lower_bound(start);
}
else
{
start_it = row_infos_->upper_bound(start);
}
if (scan_param.get_range()->border_flag_.inclusive_end())
{
end_it = row_infos_->upper_bound(end);
}
else
{
end_it = row_infos_->lower_bound(end);
}
for (;start_it != end_it && OB_SUCCESS == err; start_it ++)
{
for (int64_t scan_idx = 0;
scan_idx < scan_param.get_column_name_size() && OB_SUCCESS == err;
scan_idx ++)
{
vector<ObCellInfo>::iterator cell_it = start_it->second.begin();
cur_cell.table_name_ = scan_param.get_table_name();
cur_cell.row_key_ = start_it->first.row_key_;
cur_cell.column_name_ = scan_param.get_column_name()[scan_idx];
cur_cell.value_.set_null();
for (;cell_it != start_it->second.end(); cell_it ++)
{
if (cell_it->column_name_ == scan_param.get_column_name()[scan_idx])
{
break;
}
}
if (cell_it != start_it->second.end())
{
cur_cell.value_ = cell_it->value_;
}
err = scanner.add_cell(cur_cell);
}
}
return err;
}
int TaskOutputFile::convert_result(const ObScanner &result, ObDataBuffer &buffer) const
{
bool first_line = true;
int ret = OB_SUCCESS;
int len = 0;
ObCellInfo *cell = NULL;
ObScannerIterator itr = result.begin();
TableConf::ColumnIterator col_itr;
if (conf_ != NULL)
{
col_itr= conf_->column_begin();
}
else
{
TBSYS_LOG(WARN, "please specify table conf");
return OB_ERROR;
}
while (itr != result.end())
{
bool row_changed = false;
ret = itr.get_cell(&cell, &row_changed);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "get cell failed ret[%d]", ret);
break;
}
//append end of line for the last rowkey
if (first_line)
{
row_changed = false;
first_line = false;
}
else
{
if (row_changed == true)
{
if (ret == OB_SUCCESS)
{
ret = append_end_rec(buffer, rec_delima_);
col_itr = conf_->column_begin();
}
}
else
{
ret = append_delima(buffer, delima_);
}
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "append delima or end or record failed");
break;
}
}
if (split_rowkey_ && conf_)
{
ret = handle_rk_null(buffer, cell, col_itr, false);
}
if (col_itr == conf_->column_end())
{
ret = OB_ERROR;
TBSYS_LOG(ERROR, "task_worker conf is diff with master conf, please check it");
break;
}
if (cell == NULL || cell->column_name_.compare(*col_itr))
{
ret = OB_ERROR;
TBSYS_LOG(ERROR, "column name mismatch ,cell column name = %s, expected name = %s",
std:: string(cell->column_name_.ptr(), cell->column_name_.length()).c_str(), *col_itr);
break;
}
if (ret == OB_SUCCESS) /* normal column */
{
if (conf_->is_revise_column(cell->column_name_))
{
ret = conf_->revise_column(*cell, buffer);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "revise column failed, column_id=%ld", cell->column_id_);
break;
}
}
else
{
len = serialize_cell(cell, buffer);
if (len < 0)
{
TBSYS_LOG(ERROR, "serialize cell failed");
ret = OB_ERROR;
break;
}
}
}
if (col_itr != conf_->column_end())
{
col_itr++; /* move to next column */
}
if (ret == OB_SUCCESS && conf_ != NULL)
{
ret = handle_rk_null(buffer, cell, col_itr, true);
if (ret != OB_SUCCESS)
{
break;
}
}
itr++;
}
if (ret == OB_SUCCESS)
{
if (col_itr != conf_->column_end())
{
TBSYS_LOG(ERROR, "error happens column does not all proceeded");
}
if (ret == OB_SUCCESS)
{
ret = append_end_rec(buffer, rec_delima_);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "last row, append end rec failed");
}
}
}
return ret;
}
TEST(TestObScanner, add_cell)
{
ObScanner fem;
fem.set_is_req_fullfilled(true, 1011);
fem.set_data_version(170);
char buf[BUFSIZ];
int64_t pospos = 0;
fem.serialize(buf, BUFSIZ, pospos);
FILE *fd;
fd = fopen("tmptest", "w+");
fwrite(buf, 1, pospos, fd);
fclose(fd);
ObScanner os;
ObScanner ost;
/*** added by wushi ***/
int64_t fullfilled_item = 0;
int64_t tmp = 0;
bool is_fullfilled = true;
bool tmp_bool = false;
/// initialize
ASSERT_EQ(os.get_is_req_fullfilled(tmp_bool, fullfilled_item),0);
EXPECT_FALSE(tmp_bool);
ASSERT_EQ(fullfilled_item,0);
/// set fullfilled
is_fullfilled = true;
fullfilled_item = 1000;
os.set_is_req_fullfilled(is_fullfilled,fullfilled_item);
ASSERT_EQ(os.get_is_req_fullfilled(tmp_bool, tmp),0);
ASSERT_EQ(tmp_bool,is_fullfilled);
ASSERT_EQ(tmp,fullfilled_item);
/*** added by wushi ***/
ObCellInfo oci;
ObObj rowkey_obj;
rowkey_obj.set_int(4);
oci.table_name_.assign((char*)"table1", 6);
oci.row_key_.assign(&rowkey_obj, 1);
oci.column_name_.assign((char*)"column1", 7);
oci.value_.set_int(0xff);
ost.set_mem_size_limit(20);
ASSERT_EQ(OB_SIZE_OVERFLOW, ost.add_cell(oci));
os.set_mem_size_limit(1024 * 1024 * 2);
ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));
oci.column_name_.assign((char*)"column2", 7);
oci.value_.set_int(0xfe);
ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));
ObScannerIterator iter = os.begin();
ObCellInfo tci;
ASSERT_EQ(OB_SUCCESS, iter.get_cell(tci));
ASSERT_EQ(string("table1"), string(tci.table_name_.ptr(), tci.table_name_.length()));
ASSERT_TRUE(rowkey_obj == tci.row_key_.get_obj_ptr()[0]);
ASSERT_EQ(string("column1"), string(tci.column_name_.ptr(), tci.column_name_.length()));
ASSERT_EQ(OB_SUCCESS, tci.value_.get_int(tmp));
ASSERT_EQ(0xff, tmp);
iter++;
ASSERT_EQ(OB_SUCCESS, iter.get_cell(tci));
ASSERT_EQ(string("table1"), string(tci.table_name_.ptr(), tci.table_name_.length()));
ASSERT_TRUE(rowkey_obj == tci.row_key_.get_obj_ptr()[0]);
ASSERT_EQ(string("column2"), string(tci.column_name_.ptr(), tci.column_name_.length()));
ASSERT_EQ(OB_SUCCESS, tci.value_.get_int(tmp));
ASSERT_EQ(0xfe, tmp);
iter++;
ASSERT_EQ(true, os.end() == iter);
//ObCellInfo oci;
char row_key_buffer[1024];
oci.table_name_.assign((char*)"table1", 6);
sprintf(row_key_buffer, "row1");
ObString obstr(0, 4, row_key_buffer);
rowkey_obj.set_varchar(obstr);
oci.row_key_.assign(&rowkey_obj, 1);
oci.column_name_.assign((char*)"column1", 7);
oci.value_.set_int(0xff);
ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));
oci.table_name_.assign((char*)"table1", 6);
sprintf(row_key_buffer, "row1");
obstr.assign_ptr(row_key_buffer, static_cast<int32_t>(strlen(row_key_buffer)));
rowkey_obj.set_varchar(obstr);
oci.row_key_.assign(&rowkey_obj, 1);
oci.column_name_.assign((char*)"column2", 7);
oci.value_.set_int(0xee);
os.set_mem_size_limit(20);
ASSERT_EQ(OB_SIZE_OVERFLOW, os.add_cell(oci));
os.set_mem_size_limit(1024 * 1024 * 2);
ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));
oci.table_name_.assign((char*)"table1", 6);
sprintf(row_key_buffer, "row2");
obstr.assign_ptr(row_key_buffer, static_cast<int32_t>(strlen(row_key_buffer)));
rowkey_obj.set_varchar(obstr);
oci.row_key_.assign(&rowkey_obj, 1);
oci.column_name_.assign((char*)"column1", 7);
oci.value_.set_int(0xdd);
ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));
oci.table_name_.assign((char*)"table2", 6);
sprintf(row_key_buffer, "row2");
obstr.assign_ptr(row_key_buffer, static_cast<int32_t>(strlen(row_key_buffer)));
rowkey_obj.set_varchar(obstr);
oci.row_key_.assign(&rowkey_obj, 1);
oci.column_name_.assign((char*)"column1", 7);
oci.value_.set_int(0xcc);
ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));
oci.table_name_.assign((char*)"table3", 6);
sprintf(row_key_buffer, "row2");
obstr.assign_ptr(row_key_buffer, static_cast<int32_t>(strlen(row_key_buffer)));
rowkey_obj.set_varchar(obstr);
oci.row_key_.assign(&rowkey_obj, 1);
oci.column_name_.assign((char*)"column1", 7);
oci.value_.set_int(0xbb);
ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));
char buffer[2048];
int64_t pos = 0;
ASSERT_EQ(OB_SUCCESS, os.serialize(buffer, 1024, pos));
ASSERT_EQ(pos, os.get_serialize_size());
fd = fopen("./test.data.before_rollback", "w+");
fwrite(buffer, 1, 1024, fd);
fclose(fd);
for (iter = os.begin(); iter != os.end(); iter++)
{
ObCellInfo ci;
ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci));
print_cell_info(ci);
}
fprintf(stdout, "==============================\n");
os.reset_iter();
while (OB_SUCCESS == os.next_cell())
{
ObCellInfo *ci = NULL;
bool is_row_changed;
ASSERT_EQ(OB_SUCCESS, os.get_cell(&ci, &is_row_changed));
print_cell_info(*ci);
}
os.reset_iter();
fprintf(stdout, "==============================\n");
ASSERT_EQ(OB_SUCCESS, os.rollback());
fprintf(stdout, "size=%ld\n", os.get_serialize_size());
pos = 0;
ASSERT_EQ(OB_SUCCESS, os.serialize(buffer, 1024, pos));
ASSERT_EQ(pos, os.get_serialize_size());
fd = fopen("./test.data.after_rollback", "w+");
fwrite(buffer, 1, 1024, fd);
fclose(fd);
for (iter = os.begin(); iter != os.end(); iter++)
{
ObCellInfo ci;
ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci));
print_cell_info(ci);
ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci));
print_cell_info(ci);
}
fprintf(stdout, "==============================\n");
os.reset_iter();
while (OB_SUCCESS == os.next_cell())
{
ObCellInfo *ci = NULL;
bool is_row_changed;
ASSERT_EQ(OB_SUCCESS, os.get_cell(&ci, &is_row_changed));
print_cell_info(*ci);
ASSERT_EQ(OB_SUCCESS, os.get_cell(&ci, &is_row_changed));
print_cell_info(*ci);
}
os.reset_iter();
fprintf(stdout, "==============================\n");
pos = 0;
ASSERT_EQ(OB_SUCCESS, os.deserialize(buffer, os.get_serialize_size(), pos));
ASSERT_EQ(pos, os.get_serialize_size());
for (iter = os.begin(); iter != os.end(); iter++)
{
ObCellInfo ci;
ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci));
print_cell_info(ci);
ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci));
print_cell_info(ci);
}
fprintf(stdout, "==============================\n");
while (OB_SUCCESS == os.next_cell())
{
ObCellInfo *ci = NULL;
bool is_row_changed;
ASSERT_EQ(OB_SUCCESS, os.get_cell(&ci, &is_row_changed));
print_cell_info(*ci);
}
os.reset_iter();
/// set fullfilled
// ASSERT_EQ(os.get_is_req_fullfilled(tmp_bool, tmp),0);
// ASSERT_EQ(tmp_bool,is_fullfilled);
// ASSERT_EQ(tmp,fullfilled_item);
{
ObObj start_key_obj, end_key_obj;
start_key_obj.set_int(11111);
end_key_obj.set_int(22222);
ObNewRange range;
range.border_flag_.set_inclusive_start();
range.start_key_.assign(&start_key_obj, 1);
range.end_key_.assign(&end_key_obj, 1);
ObNewRange range2;
range2.border_flag_.set_inclusive_start();
range2.start_key_.assign(&start_key_obj, 1);
range2.end_key_.assign(&end_key_obj, 1);
int64_t pos = 0;
int64_t data_len = 0;
is_fullfilled = true;
fullfilled_item = 1212;
ObScanner os1;
ObScanner os2;
ASSERT_EQ(os1.set_is_req_fullfilled(is_fullfilled, fullfilled_item),0);
os1.set_data_version(100);
ASSERT_EQ(os1.set_range(range),0);
range.border_flag_.set_inclusive_end();
char buf[1024];
ASSERT_EQ(os1.serialize(buf,sizeof(buf),pos),0);
data_len = pos;
pos = 0;
ASSERT_EQ(os2.deserialize(buf, data_len, pos),0);
memset(buf, 0x00, 1024);
ASSERT_EQ(os2.get_is_req_fullfilled(tmp_bool, tmp),0);
EXPECT_EQ(is_fullfilled, tmp_bool);
ASSERT_EQ(fullfilled_item, tmp);
ASSERT_EQ(os1.get_data_version(),100);
ObNewRange range3;
ASSERT_EQ(OB_SUCCESS, os2.get_range(range3));
ASSERT_EQ(range2.border_flag_.get_data(), range3.border_flag_.get_data());
ASSERT_EQ(range3.start_key_, range2.start_key_);
ASSERT_EQ(range3.end_key_, range2.end_key_);
const int times = 1000000;
ObScanner os3;
ObCellInfo cinull;
cinull.table_name_.assign((char*)"hello", 5);
cinull.value_.set_ext(ObActionFlag::OP_INSERT);
ASSERT_EQ(OB_ERROR, os3.add_cell(cinull));
cinull.value_.set_ext(ObActionFlag::OP_DEL_ROW);
ASSERT_EQ(OB_SUCCESS, os3.add_cell(cinull));
cinull.value_.set_ext(ObActionFlag::OP_ROW_DOES_NOT_EXIST);
ASSERT_EQ(OB_SUCCESS, os3.add_cell(cinull));
int i = 0;
for (; i < times; i++)
{
if (OB_SUCCESS != os3.add_cell(oci))
{
break;
}
}
const int64_t buf_len = 1 << 22;
char* buff = (char*)ob_malloc(buf_len, ObModIds::TEST);
int64_t ppos = 0;
ASSERT_NE(OB_SUCCESS, os3.serialize(buff, 10, ppos));
ASSERT_EQ(OB_SUCCESS, os3.serialize(buff, buf_len, ppos));
int64_t npos = 0;
ObScanner os4;
ASSERT_EQ(OB_ERROR, os4.deserialize(buff, 10, npos));
ASSERT_EQ(OB_SUCCESS, os4.deserialize(buff, ppos, npos));
ASSERT_EQ(npos, ppos);
int64_t otmp;
oci.value_.get_int(otmp);
ObCellInfo *oo;
int t = 0;
ASSERT_EQ(OB_SUCCESS, os4.next_cell());
ASSERT_EQ(OB_SUCCESS, os4.get_cell(&oo));
ASSERT_EQ(0, oo->row_key_.length());
ASSERT_EQ(string(cinull.table_name_.ptr(), cinull.table_name_.length()), string(oo->table_name_.ptr(), oo->table_name_.length()));
ASSERT_EQ(OB_SUCCESS, oo->value_.get_ext(tmp));
ASSERT_EQ(+ObActionFlag::OP_DEL_ROW, tmp);
ASSERT_EQ(OB_SUCCESS, os4.next_cell());
ASSERT_EQ(OB_SUCCESS, os4.get_cell(&oo));
ASSERT_EQ(string(cinull.table_name_.ptr(), cinull.table_name_.length()), string(oo->table_name_.ptr(), oo->table_name_.length()));
ASSERT_EQ(OB_SUCCESS, oo->value_.get_ext(tmp));
ASSERT_EQ(+ObActionFlag::OP_ROW_DOES_NOT_EXIST, tmp);
while (OB_SUCCESS == os4.next_cell())
{
ASSERT_EQ(OB_SUCCESS, os4.get_cell(&oo));
t++;
ASSERT_EQ(string(oci.table_name_.ptr(), oci.table_name_.length()), string(oo->table_name_.ptr(), oo->table_name_.length()));
ASSERT_EQ(oci.row_key_, oo->row_key_);
ASSERT_EQ(string(oci.column_name_.ptr(), oci.column_name_.length()), string(oo->column_name_.ptr(), oo->column_name_.length()));
ASSERT_EQ(OB_SUCCESS, oo->value_.get_int(tmp));
ASSERT_EQ(otmp, tmp);
}
ASSERT_EQ(t, i);
ob_free(buff);
}
ObCellInfo st;
ObObj obj;
obj.set_int(10);
st.table_name_.assign((char*)"table1", 6);
st.row_key_.assign(&obj, 1);
st.column_name_.assign((char*)"column1", 7);
st.value_.set_int(0xff);
ObScanner ss;
ss.add_cell(st);
st.table_name_.assign((char*)"table2", 6);
st.row_key_.assign(&obj, 1);
st.column_name_.assign((char*)"column1", 7);
st.value_.set_int(0xff);
ss.add_cell(st);
ObCellInfo *sto;
bool is_row_changed;
ASSERT_EQ(OB_SUCCESS, ss.next_cell());
ASSERT_EQ(OB_SUCCESS, ss.get_cell(&sto, &is_row_changed));
ASSERT_EQ(true, is_row_changed);
ASSERT_EQ(OB_SUCCESS, ss.next_cell());
ASSERT_EQ(OB_SUCCESS, ss.get_cell(&sto, &is_row_changed));
ASSERT_EQ(true, is_row_changed);
ASSERT_EQ(OB_ITER_END, ss.next_cell());
}
int ObRootServerRpcStub::get_tablet_info(const common::ObSchemaManagerV2& schema,
const uint64_t table_id, const ObRange& range, ObTabletLocation location [],int32_t& size)
{
int ret = OB_SUCCESS;
int32_t index = 0;
const ObChunkServer& chunk_server = ObChunkServerMain::get_instance()->get_chunk_server();
const int64_t timeout = chunk_server.get_param().get_network_time_out();
if (OB_INVALID_ID == table_id || size <= 0)
{
TBSYS_LOG(ERROR,"invalid table id");
ret = OB_ERROR;
}
ObScanParam param;
ObScanner scanner;
ObString table_name;
const ObTableSchema *table = schema.get_table_schema(table_id);
if (NULL == table)
{
TBSYS_LOG(ERROR,"can not find table %lu",table_id);
ret = OB_ERROR;
}
else
{
table_name.assign_ptr(const_cast<char *>(table->get_table_name()),strlen(table->get_table_name()));
}
if (OB_SUCCESS == ret)
{
param.set(OB_INVALID_ID,table_name,range); //use table name
param.set_is_read_consistency(false);
}
if ((OB_SUCCESS == ret) && ((ret = scan(root_server_,timeout,param,scanner)) != OB_SUCCESS) )
{
TBSYS_LOG(ERROR,"get tablet from rootserver failed:[%d]",ret);
}
ObServer server;
char tmp_buf[32];
ObString start_key;
ObString end_key;
ObCellInfo * cell = NULL;
ObScannerIterator iter;
bool row_change = false;
if (OB_SUCCESS == ret)
{
int64_t ip = 0;
int64_t port = 0;
int64_t version = 0;
iter = scanner.begin();
ret = iter.get_cell(&cell, &row_change);
row_change = false;
while((OB_SUCCESS == ret) && index < size)
{
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret);
}
else if (row_change)
{
TBSYS_LOG(DEBUG,"row changed,ignore");
hex_dump(cell->row_key_.ptr(),cell->row_key_.length(),false,TBSYS_LOG_LEVEL_INFO);
break; //just get one row
}
else if (cell != NULL)
{
end_key.assign(cell->row_key_.ptr(), cell->row_key_.length());
if ((cell->column_name_.compare("1_port") == 0)
|| (cell->column_name_.compare("2_port") == 0)
|| (cell->column_name_.compare("3_port") == 0))
{
ret = cell->value_.get_int(port);
TBSYS_LOG(DEBUG,"port is %ld",port);
}
else if ((cell->column_name_.compare("1_ipv4") == 0)
|| (cell->column_name_.compare("2_ipv4") == 0)
|| (cell->column_name_.compare("3_ipv4") == 0))
{
ret = cell->value_.get_int(ip);
TBSYS_LOG(DEBUG,"ip is %ld",ip);
}
else if (cell->column_name_.compare("1_tablet_version") == 0 ||
cell->column_name_.compare("2_tablet_version") == 0 ||
cell->column_name_.compare("3_tablet_version") == 0)
{
ret = cell->value_.get_int(version);
hex_dump(cell->row_key_.ptr(),cell->row_key_.length(),false,TBSYS_LOG_LEVEL_INFO);
TBSYS_LOG(DEBUG,"tablet_version is %d",version);
}
if (OB_SUCCESS == ret)
{
if (0 != port && 0 != ip && 0 != version)
{
server.set_ipv4_addr(ip, port);
server.to_string(tmp_buf,sizeof(tmp_buf));
TBSYS_LOG(INFO,"add tablet s:%s,%ld",tmp_buf,version);
ObTabletLocation addr(version, server);
location[index++] = addr;
ip = port = version = 0;
}
}
else
{
TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret);
}
if (++iter == scanner.end())
break;
ret = iter.get_cell(&cell, &row_change);
}
else
{
//impossible
}
}
}
if (OB_SUCCESS == ret)
{
size = index;
TBSYS_LOG(INFO,"get %d tablets from rootserver",size);
}
return ret;
}
int ObScannerLoader::get_decoded_scanner(ObScanner ¶m)
{
ObNewRange range;
ObString column;
ObString table_name;
char value[4096];
if (config_loaded_)
{
TBSYS_LOG(INFO, "creating scanner");
param.reset();
range.reset();
range.table_id_ = table_id_;
TBSYS_LOG(DEBUG, "range_start_=%s", to_cstring(range_start_));
TBSYS_LOG(DEBUG, "range_end_=%s", to_cstring(range_end_));
range.start_key_ = range_start_;
if (range_start_inclusive_)
range.border_flag_.set_inclusive_start();
else
range.border_flag_.unset_inclusive_start();
if (range_start_min_)
range.start_key_.set_min_row();
range.end_key_ = range_end_;
if (range_end_inclusive_)
range.border_flag_.set_inclusive_end();
else
range.border_flag_.unset_inclusive_end();
if (range_end_max_)
range.end_key_.set_max_row();
param.set_range(range);
//param.set_is_req_fullfilled(is_fullfilled_, actual_fullfilled_item_num_);
param.set_is_req_fullfilled(is_fullfilled_, fullfilled_item_num_);
param.set_data_version(data_version_);
FILE *fp = NULL;
if ((fp = fopen(data_file_name_, "rb")) == NULL) {
TBSYS_LOG(ERROR, "Fail to open %s", data_file_name_);
}
else
{
while(fgets(value, 4096, fp))
{
/* only digit and string accepted. if not dig, then string */
int start = 0;
int end = 0;
int cnt = 0;
char buf[4096];
int table_id = 0;
int column_cnt = 0;
ObRowkey row_key;
while(true)
{
while(isspace(value[start]))
{
start++;
}
end = start;
while(!isspace(value[end]) && value[end] != '\0')
{
end++;
}
if (start != end)
{
memset(buf, 0, 4096);
cnt = end - start;
strncpy(buf, &value[start], cnt);
buf[cnt] = '\0';
if (column_cnt == 0)
{
table_id = atoi(buf);
}
else if(column_cnt == 1)
{
// TODO build rowkey
//strcpy(row_key_buf, buf);
//row_key.assign(row_key_buf, cnt);
}
else
{
build_cell_info(table_id, row_key, buf, param);
}
column_cnt++;
start = end;
}
if (value[end] == '\0')
{
break;
}
}
}
fclose(fp);
}
}
return OB_SUCCESS;
}
int MockChunkServer::handle_mock_get(ObPacket * ob_packet)
{
int ret = OB_SUCCESS;
ObDataBuffer* data = ob_packet->get_buffer();
if (NULL == data)
{
ret = OB_ERROR;
}
ObGetParam get_param;
if (OB_SUCCESS == ret)
{
ret = get_param.deserialize(data->get_data(), data->get_capacity(), data->get_position());
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "%s", "check param failed");
}
}
tbnet::Connection* connection = ob_packet->get_connection();
ThreadSpecificBuffer::Buffer* thread_buffer = response_packet_buffer_.get_buffer();
if (NULL == thread_buffer)
{
ret = OB_ERROR;
}
else
{
thread_buffer->reset();
ObDataBuffer out_buffer(thread_buffer->current(), thread_buffer->remain());
ObResultCode result_msg;
result_msg.result_code_ = ret;
ret = result_msg.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position());
// fake cell
ObCellInfo cell;
ObScanner scanner;
ObRowkey row_key;
ObString column_name;
for (int32_t i = 0; i < get_param.get_cell_size(); i ++)
{
cell.table_id_ = get_param[i]->table_id_;
cell.column_id_ = get_param[i]->column_id_;
if (mock::join_table_id == cell.table_id_)
{
if (mock::join_column1_id == cell.column_id_)
{
row_key = make_rowkey(mock::join_rowkey, &allocator_);
cell.column_id_ = mock::join_column1_id;
cell.row_key_ = row_key;
cell.value_.set_int(mock::join_column1_cs_value);
ret = scanner.add_cell(cell);
}
else if (mock::join_column2_id == cell.column_id_)
{
row_key = make_rowkey(mock::join_rowkey, &allocator_);
cell.column_id_ = mock::join_column2_id;
cell.row_key_ = row_key;
cell.value_.set_int(mock::join_column2_cs_value);
ret = scanner.add_cell(cell);
}
else
{
TBSYS_LOG(ERROR, "unepxected column id [tableid:%lu,columnid:%lu]", cell.table_id_,
cell.column_id_);
ret = OB_ERR_UNEXPECTED;
}
}
else if (mock::table_id == cell.table_id_)
{
if (mock::column1_id == cell.column_id_)
{
row_key = make_rowkey(mock::rowkey, &allocator_);
cell.column_id_ = mock::column1_id;
cell.row_key_ = row_key;
cell.value_.set_int(mock::column1_cs_value);
ret = scanner.add_cell(cell);
}
else if (mock::column2_id == cell.column_id_)
{
row_key = make_rowkey(mock::rowkey, &allocator_);
cell.column_id_ = mock::column2_id;
cell.row_key_ = row_key;
cell.value_.set_int(mock::column2_cs_value);
ret = scanner.add_cell(cell);
}
else
{
TBSYS_LOG(ERROR, "unepxected column id [tableid:%lu,columnid:%lu]", cell.table_id_,
cell.column_id_);
ret = OB_ERR_UNEXPECTED;
}
}
else
{
TBSYS_LOG(ERROR, "unexpected table id [tableid:%lu]", cell.table_id_);
ret = OB_ERR_UNEXPECTED;
}
}
if (OB_SUCCESS == ret)
{
//scanner.set_timestamp(mock::schema_timestamp);
ret = scanner.set_is_req_fullfilled(true,2);
}
int32_t channel_id = ob_packet->getChannelId();
ret = scanner.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position());
ret = send_response(OB_GET_RESPONSE, 1, out_buffer, connection, channel_id);
}
TBSYS_LOG(INFO, "handle get table result:ret[%d]", ret);
return ret;
}
int MockChunkServer::handle_mock_scan(ObPacket * ob_packet)
{
int ret = OB_SUCCESS;
ObDataBuffer* data = ob_packet->get_buffer();
if (NULL == data)
{
ret = OB_ERROR;
}
ObScanParam scan_param;
if (OB_SUCCESS == ret)
{
ret = scan_param.deserialize(data->get_data(), data->get_capacity(), data->get_position());
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "%s", "check scan_param failed");
}
}
tbnet::Connection* connection = ob_packet->get_connection();
ThreadSpecificBuffer::Buffer* thread_buffer = response_packet_buffer_.get_buffer();
if (NULL == thread_buffer)
{
ret = OB_ERROR;
}
else
{
thread_buffer->reset();
ObDataBuffer out_buffer(thread_buffer->current(), thread_buffer->remain());
ObResultCode result_msg;
result_msg.result_code_ = ret;
ret = result_msg.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position());
// fake cell
ObCellInfo cell;
ObScanner scanner;
ObRowkey row_key;
ObString column_name;
cell.table_id_ = scan_param.get_table_id();
if (mock::join_table_id == cell.table_id_)
{
for (int32_t i = 0; i < scan_param.get_column_id_size(); i++)
{
if (mock::join_column1_id == scan_param.get_column_id()[i])
{
row_key = make_rowkey(mock::join_rowkey, &allocator_);
cell.column_id_ = mock::join_column1_id;
cell.row_key_ = row_key;
cell.value_.set_int(mock::join_column1_cs_value);
ret = scanner.add_cell(cell);
}
else if (mock::join_column2_id == scan_param.get_column_id()[i])
{
row_key = make_rowkey(mock::join_rowkey, &allocator_);
cell.column_id_ = mock::join_column2_id;
cell.row_key_ = row_key;
cell.value_.set_int(mock::join_column2_cs_value);
ret = scanner.add_cell(cell);
}
else
{
TBSYS_LOG(ERROR, "unepxected column id [tableid:%lu,columnid:%lu]", cell.table_id_,
scan_param.get_column_id()[i]);
ret = OB_ERR_UNEXPECTED;
}
}
}
else if (mock::table_id == cell.table_id_)
{
for (int32_t i = 0; i < scan_param.get_column_id_size(); i++)
{
if (mock::column1_id == scan_param.get_column_id()[i])
{
row_key = make_rowkey(mock::rowkey, &allocator_);
cell.column_id_ = mock::column1_id;
cell.row_key_ = row_key;
cell.value_.set_int(mock::column1_cs_value);
ret = scanner.add_cell(cell);
}
else if (mock::column2_id == scan_param.get_column_id()[i])
{
row_key = make_rowkey(mock::rowkey, &allocator_);
cell.column_id_ = mock::column2_id;
cell.row_key_ = row_key;
cell.value_.set_int(mock::column2_cs_value);
ret = scanner.add_cell(cell);
}
else
{
TBSYS_LOG(ERROR, "unepxected column id [tableid:%lu,columnid:%lu]", cell.table_id_,
scan_param.get_column_id()[i]);
ret = OB_ERR_UNEXPECTED;
}
}
}
else
{
TBSYS_LOG(ERROR, "unexpected table id [tableid:%lu]", cell.table_id_);
ret = OB_ERR_UNEXPECTED;
}
if (OB_SUCCESS == ret)
{
//scanner.set_timestamp(mock::schema_timestamp);
ret = scanner.set_is_req_fullfilled(true,1);
}
int64_t pos = 0;
char range_buf[512];
ObString range_str;
if (OB_SUCCESS == ret)
{
ObRange range;
range.border_flag_.set_min_value();
range.border_flag_.set_max_value();
ret = range.serialize(range_buf,sizeof(range_buf),pos);
if (OB_SUCCESS == ret)
{
range_str.assign(range_buf,static_cast<int32_t>(pos));
//ret = scanner.set_ext_info(range_str);
}
pos = 0;
TBSYS_LOG(INFO, "pos:%ld,ret:%d",pos,
range.deserialize(range_str.ptr(),range_str.length(),pos));
}
int32_t channel_id = ob_packet->getChannelId();
if (OB_SUCCESS == ret)
{
ret = scanner.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position());
}
if (OB_SUCCESS == ret)
{
ret = send_response(OB_SCAN_RESPONSE, 1, out_buffer, connection, channel_id);
}
}
TBSYS_LOG(INFO, "handle scan table result:ret[%d]", ret);
return ret;
}
int ObMergerSortedOperator::add_sharding_result(ObScanner & sharding_res, const ObNewRange &query_range,
bool &is_finish)
{
int err = OB_SUCCESS;
if ((OB_SUCCESS == err) && (NULL == scan_param_))
{
TBSYS_LOG(WARN,"operator was not initialized yet [scan_param_:%p]", scan_param_);
err = OB_INVALID_ARGUMENT;
}
if ((OB_SUCCESS == err) && (sharding_result_count_ >= MAX_SHARDING_RESULT_COUNT))
{
TBSYS_LOG(WARN,"array is full [MAX_SHARDING_RESULT_COUNT:%ld,sharding_result_count_:%ld]",
MAX_SHARDING_RESULT_COUNT, sharding_result_count_);
err = OB_ARRAY_OUT_OF_RANGE;
}
/// @todo (wushi [email protected]) check fullfill item number and size property
bool is_fullfilled = false;
int64_t fullfilled_item_num = 0;
ObRowkey last_row_key;
ObNewRange cs_tablet_range;
if ((OB_SUCCESS == err) && (OB_SUCCESS != (err = sharding_res.get_is_req_fullfilled(is_fullfilled,fullfilled_item_num))))
{
TBSYS_LOG(WARN,"fail to get fullfilled info from sharding result [err:%d]", err);
}
if (OB_SUCCESS == err)
{
if (OB_SUCCESS == (err = sharding_res.get_last_row_key(last_row_key)))
{
}
else if (OB_ENTRY_NOT_EXIST == err)
{
err = OB_SUCCESS;
}
else
{
TBSYS_LOG(WARN,"fail to get last rowkey from sharding result [err:%d]", err);
}
}
if (OB_SUCCESS == err)
{
if (is_fullfilled)
{
if ((OB_SUCCESS == err) && (OB_SUCCESS != (err = sharding_res.get_range(cs_tablet_range))))
{
TBSYS_LOG(WARN,"fail to get tablet range from sharding result [err:%d]", err);
}
if (OB_SUCCESS == err)
{
if (ScanFlag::FORWARD == scan_param_->get_scan_direction())
{
last_row_key = cs_tablet_range.end_key_;
}
else
{
last_row_key = cs_tablet_range.start_key_;
}
}
}
if (OB_SUCCESS == err)
{
fullfilled_item_num = sharding_res.get_row_num();
sharding_result_arr_[sharding_result_count_].init(sharding_res,query_range,*scan_param_, last_row_key, fullfilled_item_num);
sharding_result_count_ ++;
}
}
is_finish = false;
if (OB_SUCCESS == err)
{
sort(is_finish, &sharding_res);
}
TBSYS_LOG(DEBUG, "add sharding result. is_finish=%d, err=%d", is_finish, err);
return err;
}
bool GetSingleRowAllColumn::check_result(const uint32_t min_key_include, const uint32_t max_key_include, ObScanner &result,
void *arg)
{
bool res = true;
int err = OB_SUCCESS;
UNUSED(min_key_include);
UNUSED(max_key_include);
ObGetParam *get_param = reinterpret_cast<ObGetParam*>(arg);
uint32_t big_endian_rowkey_val = *(uint32_t*)(((*get_param)[0])->row_key_.ptr());
uint32_t rowkey_val = ntohl(big_endian_rowkey_val);
ObCellInfo target_cell;
target_cell.row_key_.assign((char*)&big_endian_rowkey_val, sizeof(big_endian_rowkey_val));
target_cell.table_name_ = msolap::target_table_name;
if (result.get_cell_num() != msolap::max_column_id - msolap::min_column_id + 1)
{
TBSYS_LOG(WARN,"result cell number error [got:%ld,expcted:%ld]", result.get_cell_num(),
msolap::max_column_id - msolap::min_column_id + 1);
err = OB_ERR_UNEXPECTED;
}
int64_t got_cell_num = 0;
ObCellInfo *cur_cell = NULL;
int64_t intval = 0;
for (got_cell_num = 0; (got_cell_num < result.get_cell_num()) && (OB_SUCCESS == err); got_cell_num++)
{
if (OB_SUCCESS != (err = result.next_cell()))
{
TBSYS_LOG(WARN,"fail to get next cell from result [err:%d]", err);
}
if ((OB_SUCCESS == err) && (OB_SUCCESS != (err = result.get_cell(&cur_cell))))
{
TBSYS_LOG(WARN,"fail to get next cell from result [err:%d]", err);
}
if ((OB_SUCCESS == err)
&& ((cur_cell->table_name_ != msolap::target_table_name)
|| (cur_cell->row_key_ != target_cell.row_key_)
|| (cur_cell->column_name_.length() != 1)
|| (OB_SUCCESS != cur_cell->value_.get_int(intval))))
{
TBSYS_LOG(WARN,"cell content error");
err = OB_ERR_UNEXPECTED;
}
if ((OB_SUCCESS == err) && ((cur_cell->column_name_.ptr()[0] < 'a') || (cur_cell->column_name_.ptr()[0] > 'z')))
{
TBSYS_LOG(WARN,"cell content error");
err = OB_ERR_UNEXPECTED;
}
if (OB_SUCCESS == err)
{
int64_t expect_intval = 0;
char c_name = cur_cell->column_name_.ptr()[0];
switch (c_name)
{
case 'a':
case 'b':
case 'c':
case 'd':
expect_intval = ((unsigned char*)&big_endian_rowkey_val)[c_name - 'a'];
break;
default:
expect_intval = rowkey_val;
}
if (expect_intval != intval)
{
TBSYS_LOG(WARN,"intvalue error [column_name:%c,expect_val:%ld,got_val:%ld,rowkey:%u]", c_name,
expect_intval, intval, rowkey_val);
err = OB_ERR_UNEXPECTED;
}
}
}
if ((OB_SUCCESS == err) && (got_cell_num != msolap::max_column_id - msolap::min_column_id + 1))
{
TBSYS_LOG(WARN,"fail to get enough cells from result");
err = OB_ERR_UNEXPECTED;
}
if ((OB_SUCCESS == err) && (OB_ITER_END != result.next_cell()))
{
TBSYS_LOG(WARN,"fail to get enough cells from result");
err = OB_ERR_UNEXPECTED;
}
if (OB_SUCCESS != err)
{
res = false;
}
return res;
}
TEST_F(TestRpcStub, test_scan_servers)
{
ObMergerRpcStub stub;
ThreadSpecificBuffer buffer;
ObPacketFactory factory;
tbnet::Transport transport;
tbnet::DefaultPacketStreamer streamer;
streamer.setPacketFactory(&factory);
transport.start();
ObClientManager client_manager;
EXPECT_TRUE(OB_SUCCESS == client_manager.initialize(&transport, &streamer));
EXPECT_TRUE(OB_SUCCESS == stub.init(&buffer, &client_manager));
ObMergerTabletLocationList list;
ObServer chunk_server;
chunk_server.set_ipv4_addr(addr, MockChunkServer::CHUNK_SERVER_PORT);
ObTabletLocation addr;
//addr.tablet_id_ = 100;
addr.chunkserver_ = chunk_server;
list.add(addr);
list.add(addr);
list.add(addr);
// start root server
MockChunkServer server;
MockServerRunner test_chunk_server(server);
tbsys::CThread chunk_server_thread;
chunk_server_thread.start(&test_chunk_server, NULL);
sleep(2);
ObScanParam param;
ObCellInfo cell;
ObRange range;
ObString name;
ObScanner scanner;
param.set(1, name, range);
EXPECT_TRUE(OB_SUCCESS == stub.scan(timeout, chunk_server, param, scanner));
EXPECT_TRUE(!scanner.is_empty());
uint64_t count = 0;
ObScannerIterator iter;
for (iter = scanner.begin(); iter != scanner.end(); ++iter)
{
EXPECT_TRUE(OB_SUCCESS == iter.get_cell(cell));
//EXPECT_TRUE(cell.column_id_ == count);
printf("client:%.*s\n", cell.row_key_.length(), cell.row_key_.ptr());
++count;
}
// return 10 cells
EXPECT_TRUE(count == 10);
ObMergerTabletLocation succ_addr;
bool update = false;
EXPECT_TRUE(OB_SUCCESS == stub.scan(timeout, list, param, succ_addr, scanner, update));
EXPECT_TRUE(!scanner.is_empty());
EXPECT_TRUE(update == false);
for (iter = scanner.begin(); iter != scanner.end(); ++iter)
{
EXPECT_TRUE(OB_SUCCESS == iter.get_cell(cell));
printf("client:%.*s\n", cell.row_key_.length(), cell.row_key_.ptr());
}
transport.stop();
server.stop();
sleep(10);
}
int get_next_param(
const ObScanParam &org_scan_param, const ObScanner &prev_scan_result,
ObScanParam *scan_param, ObMemBuf &range_buffer)
{
int err = OB_SUCCESS;
const ObReadParam &org_read_param = org_scan_param;
ObReadParam *read_param = scan_param;
ObNewRange tablet_range;
const ObNewRange *org_scan_range = NULL;
ObNewRange cur_range;
if(NULL != scan_param)
{
scan_param->reset();
*read_param = org_read_param;
}
bool request_fullfilled = false;
int64_t fullfilled_row_num = 0;
err = prev_scan_result.get_is_req_fullfilled(request_fullfilled,fullfilled_row_num);
if (OB_SUCCESS == err)
{
org_scan_range = org_scan_param.get_range();
if (NULL == org_scan_range)
{
TBSYS_LOG(WARN, "unexpected error, can't get range from org scan param");
err = OB_INVALID_ARGUMENT;
}
}
ObRowkey last_row_key;
if (OB_SUCCESS == err)
{
if (request_fullfilled)
{
err = prev_scan_result.get_range(tablet_range);
if (OB_SUCCESS == err)
{
if (true == is_finish_scan(org_scan_param, tablet_range))
{
err = OB_ITER_END;
}
else if (ScanFlag::FORWARD == org_scan_param.get_scan_direction())
{
last_row_key = tablet_range.end_key_;
}
else
{
last_row_key = tablet_range.start_key_;
}
}
}
else
{
err = prev_scan_result.get_last_row_key(last_row_key);
if (OB_ENTRY_NOT_EXIST == err)
{
/// first time, remain unchanged
}
else if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN, "unexpected error, scanner should contain at least one cell");
}
}
}
if (OB_SUCCESS == err && NULL != scan_param)
{
cur_range =*org_scan_range;
// forward
if (ScanFlag::FORWARD == org_scan_param.get_scan_direction())
{
cur_range.start_key_ = last_row_key;
cur_range.border_flag_.unset_inclusive_start();
}
else
{
cur_range.end_key_ = last_row_key;
if (request_fullfilled)
{
// tablet range start key
cur_range.border_flag_.set_inclusive_end();
}
else
{
// the smallest rowkey
cur_range.border_flag_.unset_inclusive_end();
}
}
}
// first time remain unchanged
if (OB_ENTRY_NOT_EXIST == err)
{
cur_range =*org_scan_range;
err = OB_SUCCESS;
}
if (OB_SUCCESS == err && NULL != scan_param)
{
err = allocate_range_buffer(cur_range, range_buffer);
}
if (OB_SUCCESS == err && NULL != scan_param)
{
scan_param->set(org_scan_param.get_table_id(), org_scan_param.get_table_name(), cur_range);
for (int32_t cell_idx = 0;
cell_idx < org_scan_param.get_column_id_size() && OB_SUCCESS == err;
cell_idx ++)
{
err = scan_param->add_column(org_scan_param.get_column_id()[cell_idx]);
}
if(OB_SUCCESS == err)
{
scan_param->set_scan_size(GET_SCAN_SIZE(org_scan_param.get_scan_size()));
scan_param->set_scan_flag(org_scan_param.get_scan_flag());
}
}
return err;
}
TEST_F(TestGet, test_get_one_row)
{
int err = 0;
CommonSchemaManagerWrapper schema_mgr;
tbsys::CConfig config;
bool ret_val = schema_mgr.parse_from_file("scan_schema.ini", config);
ASSERT_TRUE(ret_val);
ObUpsTableMgr& mgr = ObUpdateServerMain::get_instance()->get_update_server().get_table_mgr();
err = mgr.init();
ASSERT_EQ(0, err);
err = mgr.set_schemas(schema_mgr);
ASSERT_EQ(0, err);
TestUpsTableMgrHelper test_helper(mgr);
TableMgr& table_mgr = test_helper.get_table_mgr();
table_mgr.sstable_scan_finished(10);
TableItem* active_memtable_item = table_mgr.get_active_memtable();
MemTable& active_memtable = active_memtable_item->get_memtable();
// construct multi-row mutator
static const int64_t ROW_NUM = 1;
static const int64_t COL_NUM = 10;
ObCellInfo cell_infos[ROW_NUM][COL_NUM];
std::bitset<COL_NUM> check_flags[ROW_NUM];
char row_key_strs[ROW_NUM][50];
uint64_t table_id = 10;
// init cell infos
for (int64_t i = 0; i < ROW_NUM; ++i)
{
sprintf(row_key_strs[i], "row_key_%08ld", i);
for (int64_t j = 0; j < COL_NUM; ++j)
{
cell_infos[i][j].table_id_ = table_id;
cell_infos[i][j].row_key_ = make_rowkey(row_key_strs[i], &allocator_);
cell_infos[i][j].column_id_ = j + 10;
cell_infos[i][j].value_.set_int(1000 + i * COL_NUM + j);
}
}
ObUpsMutator ups_mutator;
ObMutator &mutator = ups_mutator.get_mutator();
// add cell to array
for (int64_t i = 0; i < ROW_NUM; ++i)
{
for (int64_t j = 0; j < COL_NUM; ++j)
{
ObMutatorCellInfo mutator_cell;
mutator_cell.cell_info = cell_infos[i][j];
mutator_cell.op_type.set_ext(ObActionFlag::OP_UPDATE);
err = mutator.add_cell(mutator_cell);
EXPECT_EQ(0, err);
}
}
prepare_mutator(mutator);
// write row to active memtable
uint64_t trans_descriptor = 0;
err = active_memtable.start_transaction(WRITE_TRANSACTION, trans_descriptor);
ASSERT_EQ(0, err);
err = active_memtable.start_mutation(trans_descriptor);
ASSERT_EQ(0, err);
err = active_memtable.set(trans_descriptor, ups_mutator);
EXPECT_EQ(0, err);
err = active_memtable.end_mutation(trans_descriptor, false);
ASSERT_EQ(0, err);
err = active_memtable.end_transaction(trans_descriptor, false);
ASSERT_EQ(0, err);
/*
ObString text;
text.assign("/tmp", strlen("/tmp"));
active_memtable.dump2text(text);
*/
for (int64_t i = 0; i < ROW_NUM; ++i)
{
ObScanner scanner;
ObGetParam get_param;
//get_param.set_timestamp(version);
ObVersionRange version_range;
//version_range.start_version_ = version;
//version_range.end_version_ = version;
version_range.start_version_ = 2;
//version_range.end_version_ = 2;
version_range.border_flag_.set_inclusive_start();
//version_range.border_flag_.set_inclusive_end();
version_range.border_flag_.set_max_value();
get_param.set_version_range(version_range);
for (int64_t j = 0; j < COL_NUM; ++j)
{
get_param.add_cell(cell_infos[i][j]);
}
int64_t count = 0;
err = mgr.get(get_param, scanner, tbsys::CTimeUtil::getTime(), 2000000000 * 1000L * 1000L);
EXPECT_EQ(0, err);
// check result
count = 0;
while (OB_SUCCESS == scanner.next_cell())
{
ObCellInfo* p_cell = NULL;
scanner.get_cell(&p_cell);
ASSERT_TRUE(p_cell != NULL);
ObCellInfo expected = cell_infos[count / COL_NUM + i][p_cell->column_id_ - 10];
check_cell(expected, *p_cell);
check_flags[count / COL_NUM].set(p_cell->column_id_ - 10);
++count;
}
EXPECT_EQ(COL_NUM, count);
for (int64_t i = 0; i < ROW_NUM; i++)
{
EXPECT_EQ(COL_NUM, (int64_t)check_flags[i].size());
}
}
}
bool check_result(const ObScanParam &scan_param, ObScanner &ob_scanner,
ObFinalResult & local_result)
{
int err = OB_SUCCESS;
bool res = true;
ObScanner local_scanner;
ObCellInfo *ob_cell = NULL;
ObCellInfo *local_cell = NULL;
err = local_result.scan(scan_param,local_scanner);
if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN,"fail to scan local result");
}
int64_t row_number = 0;
bool ob_end = false;
ObString rowkey;
while (OB_SUCCESS == err)
{
for (int64_t scan_idx = 0; scan_idx < scan_param.get_column_name_size(); scan_idx ++)
{
if (OB_SUCCESS == err)
{
err = ob_scanner.next_cell();
}
if (OB_ITER_END == err && scan_idx == 0)
{
ob_end = true;
err = OB_SUCCESS;
}
else if (OB_SUCCESS == err)
{
err = ob_scanner.get_cell(&ob_cell);
}
if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN,"fail to get cell from ob result [row_idx:%lld,idx:%lld]",
row_number, scan_idx);
}
if (OB_SUCCESS == err)
{
err = local_scanner.next_cell();
if (ob_end && OB_ITER_END != err)
{
TBSYS_LOG(WARN,"local result return more cells than ob result");
err = OB_INVALID_ARGUMENT;
}
if ( !ob_end && OB_ITER_END == err)
{
TBSYS_LOG(WARN,"ob result return more cells than local result");
err = OB_INVALID_ARGUMENT;
}
if (ob_end && OB_ITER_END == err)
{
err = OB_ITER_END;
break;
}
if (OB_SUCCESS == err)
{
err = local_scanner.get_cell(&local_cell);
}
if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN,"fail to get cell from local result [row_idx:%lld,idx:%lld]",
row_number, scan_idx);
}
}
if (OB_SUCCESS == err)
{
if (0 == scan_idx)
{
rowkey = ob_cell->row_key_;
}
/// check ob result
if (ob_cell->table_name_ != scan_param.get_table_name())
{
TBSYS_LOG(WARN,"ob result table name error [row_idx:%lld,idx:%lld,ob.table_name_:%.*s,"
"param.table_name_:%.*s]", row_number, scan_idx,ob_cell->table_name_.length(),
ob_cell->table_name_.ptr(), scan_param.get_table_name().length(),
scan_param.get_table_name().ptr());
err = OB_INVALID_ARGUMENT;
}
if (ob_cell->column_name_ != scan_param.get_column_name()[scan_idx])
{
TBSYS_LOG(WARN,"ob result rowkey error [row_idx:%lld,idx:%lld,ob.column_name_:%.*s,"
"param.column_name_:%.*s]", row_number, scan_idx,ob_cell->column_name_.length(),
ob_cell->column_name_.ptr(), scan_param.get_column_name()[scan_idx].length(),
scan_param.get_column_name()[scan_idx].ptr());
err = OB_INVALID_ARGUMENT;
}
/// check local result
if (local_cell->table_name_ != scan_param.get_table_name())
{
TBSYS_LOG(WARN,"ob result table name error [row_idx:%lld,idx:%lld,ob.table_name_:%.*s,"
"param.table_name_:%.*s]", row_number, scan_idx,local_cell->table_name_.length(),
local_cell->table_name_.ptr(), scan_param.get_table_name().length(),
scan_param.get_table_name().ptr());
err = OB_INVALID_ARGUMENT;
}
if (local_cell->column_name_ != scan_param.get_column_name()[scan_idx])
{
TBSYS_LOG(WARN,"ob result rowkey error [row_idx:%lld,idx:%lld,ob.column_name_:%.*s,"
"param.column_name_:%.*s]", row_number,scan_idx,local_cell->column_name_.length(),
local_cell->column_name_.ptr(), scan_param.get_column_name()[scan_idx].length(),
scan_param.get_column_name()[scan_idx].ptr());
err = OB_INVALID_ARGUMENT;
}
if (local_cell->row_key_ != ob_cell->row_key_)
{
TBSYS_LOG(WARN,"rowkey error [row_idx:%lld,idx:%lld,ob.row_key_:%.*s,"
"local.row_key_:%.*s]", row_number, scan_idx,ob_cell->row_key_.length(),
ob_cell->row_key_.ptr(), local_cell->row_key_.length(),
local_cell->row_key_.ptr());
err = OB_INVALID_ARGUMENT;
}
if (local_cell->row_key_ != rowkey)
{
TBSYS_LOG(WARN,"rowkey error [row_idx:%lld,idx:%lld,ob.row_key_:%.*s,"
"row_first_cell.row_key_:%.*s]", row_number,scan_idx,local_cell->row_key_.length(),
local_cell->row_key_.ptr(), rowkey.length(), rowkey.ptr());
err = OB_INVALID_ARGUMENT;
}
if (OB_SUCCESS == err)
{
if (ob_cell->value_.get_ext() != local_cell->value_.get_ext())
{
TBSYS_LOG(WARN,"ext info not coincident [row_idx:%lld,idx:%lld,"
"ob.ext:%lld,local.ext:%lld]",
row_number, scan_idx, ob_cell->value_.get_ext(), local_cell->value_.get_ext());
err = OB_INVALID_ARGUMENT;
}
if (ob_cell->value_.get_type() == ObNullType
&& ob_cell->value_.get_type() == local_cell->value_.get_type())
{
/// check pass
}
else
{
int64_t ob_value = 0;
int64_t local_value = 0;
if (OB_SUCCESS == err)
{
err = ob_cell->value_.get_int(ob_value);
}
if (OB_SUCCESS == err)
{
err = local_cell->value_.get_int(local_value);
}
if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN,"fail to get int value [row_idx:%lld,idx:%lld]",row_number, scan_idx);
err = OB_INVALID_ARGUMENT;
}
else
{
if (ob_value != local_value)
{
TBSYS_LOG(WARN,"value not coincident [row_idx:%lld,idx:%lld,ob.value:%lld,"
"local.value:%lld]",row_number, scan_idx, ob_value, local_value);
err = OB_INVALID_ARGUMENT;
}
}
}
}
}
}
if (OB_ITER_END == err)
{
err = OB_SUCCESS;
break;
}
row_number ++;
}
if (OB_SUCCESS != err)
{
res = false;
TBSYS_LOG(DEBUG, "check fail");
}
else
{
res = true;
TBSYS_LOG(DEBUG, "check pass");
}
return res;
}
int MockChunkServer::handle_scan_table(ObPacket * ob_packet)
{
int ret = OB_SUCCESS;
ObDataBuffer* data = ob_packet->get_buffer();
if (NULL == data)
{
ret = OB_ERROR;
}
ObScanParam param;
if (OB_SUCCESS == ret)
{
ret = param.deserialize(data->get_data(), data->get_capacity(), data->get_position());
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "%s", "check param failed");
}
}
tbnet::Connection* connection = ob_packet->get_connection();
ThreadSpecificBuffer::Buffer* thread_buffer = response_packet_buffer_.get_buffer();
if (NULL == thread_buffer)
{
ret = OB_ERROR;
}
else
{
thread_buffer->reset();
ObDataBuffer out_buffer(thread_buffer->current(), thread_buffer->remain());
ObResultCode result_msg;
result_msg.result_code_ = ret;
ret = result_msg.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position());
// fake cell
ObCellInfo cell;
ObScanner scanner;
ObRowkey row_key;
ObString column_name;
char temp[256] = "";
#if 1
cell.table_id_ = 101;
for (uint64_t i = 0; i < 10; ++i)
{
snprintf(temp, 256, "chunk_%lu_scan_row_key:%lu", i, i);
row_key = make_rowkey(temp, &allocator_);
cell.row_key_ = row_key;
cell.column_id_ = i + 1;
cell.value_.set_int(2234 + i);
scanner.add_cell(cell);
}
#else
cell.table_id_ = 123;
for (uint64_t i = 100; i < 200; ++i)
{
snprintf(temp, 256, "row_%lu", i);
row_key = make_rowkey(temp, &allocator_);
cell.row_key_ = row_key;
cell.column_id_ = 101;
cell.value_.set_int(2234 + i);
scanner.add_cell(cell);
}
#endif
/* begin add by xiaochu */
//Scanner Range must be set other wise the ms client will report error
ObNewRange range;
/*
/// This will cause rowkey mismatch
//char *start= "chunk_0_scan_row_key:0";
//char *end = "chunk_9_scan_row_key:9";
*/
char *start= (char*)"row_100";
char *end = (char*)"row_200";
range.start_key_ = make_rowkey(start, &allocator_);;
range.end_key_ = make_rowkey(end, &allocator_);
range.table_id_ = 103;
scanner.set_range(range);
scanner.set_is_req_fullfilled(true, 10);
/* end add by xiaochu */
int32_t channel_id = ob_packet->getChannelId();
ret = scanner.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position());
ObScannerIterator iter;
for (iter = scanner.begin(); iter != scanner.end(); ++iter)
{
iter.get_cell(cell);
printf("server_temp:%s\n", to_cstring(cell.row_key_));
}//
ret = send_response(OB_SCAN_RESPONSE, 1, out_buffer, connection, channel_id);
}
TBSYS_LOG(INFO, "handle scan table result:ret[%d]", ret);
return ret;
}
int MockRootServer::handle_get_root(ObPacket * ob_packet)
{
int ret = OB_SUCCESS;
ObDataBuffer* data = ob_packet->get_buffer();
if (NULL == data)
{
ret = OB_ERROR;
}
tbnet::Connection* connection = ob_packet->get_connection();
ThreadSpecificBuffer::Buffer* thread_buffer = response_packet_buffer_.get_buffer();
if (NULL == thread_buffer)
{
ret = OB_ERROR;
}
else
{
thread_buffer->reset();
ObDataBuffer out_buffer(thread_buffer->current(), thread_buffer->remain());
ObResultCode result_msg;
result_msg.result_code_ = ret;
ret = result_msg.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position());
// fake location cell
ObScanner scanner;
char table[128] = "";
char rowkey[128] = "";
char temp[128] = "";
ObCellInfo cell;
// the first row
sprintf(table, "root_table");
cell.table_name_.assign(table, strlen(table));
sprintf(rowkey, "row_100");
cell.row_key_.assign(rowkey, strlen(rowkey));
sprintf(temp, "2_ipv4");
cell.column_name_.assign(temp, strlen(temp));
cell.value_.set_int(16777343);
scanner.add_cell(cell);
sprintf(temp, "2_port");
cell.column_name_.assign(temp, strlen(temp));
cell.value_.set_int(10240);
scanner.add_cell(cell);
// the second row
sprintf(rowkey, "row_200");
cell.row_key_.assign(rowkey, strlen(rowkey));
for (int i = 1; i <= 3; ++i)
{
sprintf(temp, "%d_ipv4", i);
cell.column_name_.assign(temp, strlen(temp));
cell.value_.set_int(16777343);
scanner.add_cell(cell);
sprintf(temp, "%d_port", i);
cell.column_name_.assign(temp, strlen(temp));
cell.value_.set_int(12341);
scanner.add_cell(cell);
}
// the third row
sprintf(rowkey, "row_999");
cell.row_key_.assign(rowkey, strlen(rowkey));
for (int i = 1; i <= 3; ++i)
{
sprintf(temp, "%d_ipv4", i);
cell.column_name_.assign(temp, strlen(temp));
cell.value_.set_int(16777343);
scanner.add_cell(cell);
sprintf(temp, "%d_port", i);
cell.column_name_.assign(temp, strlen(temp));
cell.value_.set_int(12341);
scanner.add_cell(cell);
}
int32_t channel_id = ob_packet->getChannelId();
ret = scanner.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position());
ret = send_response(OB_REPORT_TABLETS_RESPONSE, 1, out_buffer, connection, channel_id);
TBSYS_LOG(INFO, "send get root response:ret[%d]", ret);
}
TBSYS_LOG(INFO, "handle get root table result:ret[%d]", ret);
return ret;
}
int ObMergerRpcStub::scan(
const int64_t timeout, const ObServer & server,
const ObScanParam & scan_param, ObScanner & scanner) const
{
int ret = OB_SUCCESS;
ObDataBuffer data_buff;
int64_t start_time = tbsys::CTimeUtil::getTime();
ret = get_rpc_buffer(data_buff);
// step 1. serialize ObScanParam to the data_buff
if (OB_SUCCESS == ret)
{
ret = scan_param.serialize(data_buff.get_data(), data_buff.get_capacity(),
data_buff.get_position());
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "serialize scan param failed:ret[%d]", ret);
}
}
// step 2. send request for scan data
if (OB_SUCCESS == ret)
{
ret = rpc_frame_->send_request(server, OB_SCAN_REQUEST, DEFAULT_VERSION,
timeout, data_buff);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(WARN, "send scan request to server failed:ret[%d], timeout=%ld",
ret, timeout);
}
}
// step 3. deserialize the response result
int64_t pos = 0;
if (OB_SUCCESS == ret)
{
ObResultCode result_code;
ret = result_code.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "deserialize result failed:pos[%ld], ret[%d]", pos, ret);
}
else
{
ret = result_code.result_code_;
}
}
// step 4. deserialize the scanner
if (OB_SUCCESS == ret)
{
ret = scanner.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "deserialize scanner from buff failed:pos[%ld], ret[%d]", pos, ret);
}
}
// write debug log
const int32_t MAX_SERVER_ADDR_SIZE = 128;
char server_addr[MAX_SERVER_ADDR_SIZE];
server.to_string(server_addr, MAX_SERVER_ADDR_SIZE);
bool is_fullfilled = false;
int64_t fullfilled_item_num = 0;
scanner.get_is_req_fullfilled(is_fullfilled,fullfilled_item_num);
if (OB_SUCCESS == ret)
{
TBSYS_LOG(DEBUG, "scan data succ from server:addr[%s]", server_addr);
}
else
{
TBSYS_LOG(WARN, "scan data failed from server:addr[%s], "
"version_range=%s, ret[%d]",
server_addr, range2str(scan_param.get_version_range()), ret);
}
int64_t consume_time = tbsys::CTimeUtil::getTime() - start_time;
if ((double)consume_time > (double)timeout * 0.8)
{
TBSYS_LOG(WARN, "slow ups scan, ups_addr=%s, timeout=%ld, consume=%ld",
server.to_cstring(), timeout, consume_time);
}
return ret;
}
void scan_check_all(MutatorBuilder &mb, MockClient &client, const int64_t table_start_version, const bool using_id, const bool check)
{
for (int64_t i = 0; i < mb.get_schema_num(); ++i)
{
ObScanParam scan_param;
prepare_scan_param(scan_param, mb.get_schema(i), table_start_version, using_id);
int ret = OB_SUCCESS;
bool is_fullfilled = false;
int64_t fullfilled_item_num = 0;
RowChecker rc;
int64_t row_counter = 0;
ObScanner scanner;
while (!is_fullfilled
&& OB_SUCCESS == ret)
{
int64_t timeu = tbsys::CTimeUtil::getTime();
ret = client.ups_scan(scan_param, scanner, TIMEOUT_MS);
TBSYS_LOG(INFO, "scan ret=%d timeu=%ld", ret, tbsys::CTimeUtil::getTime() - timeu);
if (check
&& OB_SUCCESS == ret)
{
while (OB_SUCCESS == scanner.next_cell())
{
ObCellInfo *ci = NULL;
bool is_row_changed = false;
if (OB_SUCCESS == scanner.get_cell(&ci, &is_row_changed))
{
if (!using_id)
{
trans_name2id(*ci, mb.get_schema(i));
}
if (is_row_changed && 0 != rc.cell_num())
{
std::string row_key_str(rc.get_cur_rowkey().ptr(), 0, rc.get_cur_rowkey().length());
//fprintf(stderr, "[%.*s] ", rc.get_cur_rowkey().length(), rc.get_cur_rowkey().ptr());
bool get_row_bret = get_check_row(mb.get_schema(i), rc.get_cur_rowkey(), mb.get_cellinfo_builder(i), client, table_start_version, using_id);
//fprintf(stderr, "[get_row check_ret=%d] ", bret);
bool cell_info_bret = rc.check_row(mb.get_cellinfo_builder(i), mb.get_schema(i));
//fprintf(stderr, "[cell_info check_ret=%d]\n", bret);
TBSYS_LOG(INFO, "[%s] [get_row check_ret=%d] [cell_info check_ret=%d]", row_key_str.c_str(), get_row_bret, cell_info_bret);
}
if (is_row_changed
&& 0 != rc.rowkey_num()
&& rc.is_prefix_changed(ci->row_key_))
{
std::string row_key_str(rc.get_last_rowkey().ptr(), 0, rc.get_last_rowkey().length());
//fprintf(stderr, "[%.*s] ", rc.get_last_rowkey().length(), rc.get_last_rowkey().ptr());
bool bret = rc.check_rowkey(mb.get_rowkey_builder(i));
//fprintf(stderr, "row_key check_ret=%d\n", bret);
TBSYS_LOG(INFO, "[%s] [row_key check_ret=%d]", row_key_str.c_str(), bret);
}
rc.add_cell(ci);
if (is_row_changed)
{
rc.add_rowkey(ci->row_key_);
row_counter++;
}
}
}
if (0 != rc.cell_num())
{
std::string row_key_str(rc.get_cur_rowkey().ptr(), 0, rc.get_cur_rowkey().length());
//fprintf(stderr, "[%.*s] ", rc.get_cur_rowkey().length(), rc.get_cur_rowkey().ptr());
bool get_row_bret = get_check_row(mb.get_schema(i), rc.get_cur_rowkey(), mb.get_cellinfo_builder(i), client, table_start_version, using_id);
//fprintf(stderr, "[get_row check_ret=%d] ", bret);
bool cell_info_bret = rc.check_row(mb.get_cellinfo_builder(i), mb.get_schema(i));
//fprintf(stderr, "[cell_info check_ret=%d]\n", bret);
TBSYS_LOG(INFO, "[%s] [get_row check_ret=%d] [cell_info check_ret=%d]", row_key_str.c_str(), get_row_bret, cell_info_bret);
}
}
scanner.get_is_req_fullfilled(is_fullfilled, fullfilled_item_num);
ObRange *range = const_cast<ObRange*>(scan_param.get_range());
scanner.get_last_row_key(range->start_key_);
range->border_flag_.unset_min_value();
range->border_flag_.unset_inclusive_start();
}
if (check
&& 0 != rc.rowkey_num())
{
std::string row_key_str(rc.get_last_rowkey().ptr(), 0, rc.get_last_rowkey().length());
//fprintf(stderr, "[%.*s] ", rc.get_last_rowkey().length(), rc.get_last_rowkey().ptr());
bool bret = rc.check_rowkey(mb.get_rowkey_builder(i));
//fprintf(stderr, "row_key check_ret=%d\n", bret);
TBSYS_LOG(INFO, "[%s] [row_key check_ret=%d]", row_key_str.c_str(), bret);
}
TBSYS_LOG(INFO, "table_id=%lu row_counter=%ld", mb.get_schema(i).get_table_id(), row_counter);
}
}
int ObFakeSqlUpsRpcProxy2::gen_new_scanner(uint64_t table_id, int64_t start_rowkey, int64_t end_rowkey, ObBorderFlag border_flag, ObNewScanner &new_scanner, bool is_fullfilled)
{
int ret = OB_SUCCESS;
char rowkey_buf[100];
ObString rowkey_str;
ObObj rowkey_obj;
ObRowkey rowkey;
ObScanner scanner;
ObNewRange range;
range.table_id_ = table_id;
range.border_flag_ = border_flag;
gen_new_range(start_rowkey, end_rowkey, range_buf_, range);
if(OB_SUCCESS != (ret = scanner.set_range(range)))
{
TBSYS_LOG(WARN, "scanner set range fail:ret[%d]", ret);
}
ObUpsRow ups_row;
ObRowDesc row_desc;
ObObj value;
if(OB_SUCCESS == ret)
{
for(uint64_t i = 0; OB_SUCCESS == ret && i<COLUMN_NUMS; i++)
{
if(OB_SUCCESS != (ret = row_desc.add_column_desc(TABLE_ID, i+OB_APP_MIN_COLUMN_ID)))
{
TBSYS_LOG(WARN, "add column desc fail:ret[%d]", ret);
}
}
ups_row.set_row_desc(row_desc);
}
int64_t start = border_flag.inclusive_start() ? start_rowkey : start_rowkey + 1;
int64_t end = border_flag.inclusive_end() ? end_rowkey : end_rowkey - 1;
ObCellInfo cell_info;
if(OB_SUCCESS == ret)
{
for(int64_t i=start; OB_SUCCESS == ret && i<=end; i++)
{
sprintf(rowkey_buf, "rowkey_%05ld", i);
rowkey_str.assign_ptr(rowkey_buf, (int32_t)strlen(rowkey_buf));
rowkey_obj.set_varchar(rowkey_str);
rowkey.assign(&rowkey_obj, 1);
for(int64_t j=0; OB_SUCCESS == ret && j<COLUMN_NUMS; j++)
{
cell_info.table_id_ = TABLE_ID;
cell_info.row_key_ = rowkey;
cell_info.column_id_ = j+OB_APP_MIN_COLUMN_ID;
cell_info.value_.set_int(i * 1000 + j);
if(OB_SUCCESS != (ret = scanner.add_cell(cell_info)))
{
TBSYS_LOG(WARN, "scanner add cell fail:ret[%d]", ret);
}
}
}
}
int64_t fullfilled_cell_num = (end - start + 1) * COLUMN_NUMS;
if(OB_SUCCESS == ret)
{
if(OB_SUCCESS != (ret = scanner.set_is_req_fullfilled(is_fullfilled, fullfilled_cell_num)))
{
TBSYS_LOG(WARN, "set fullfilled fail:ret[%d]", ret);
}
else if(OB_SUCCESS != (ret = ObNewScannerHelper::convert(scanner, &row_desc, new_scanner)))
{
TBSYS_LOG(WARN, "convert scanner to new scanner fail:ret[%d]", ret);
}
}
return ret;
}