void total_scan(const char *fname, PageArena<char> &allocer, MockClient &client, const char *version_range)
{
ObScanner scanner;
ObScanParam scan_param;
read_scan_param(fname, SCAN_PARAM_SECTION, allocer, scan_param);
scan_param.set_version_range(str2range(version_range));
scan_param.set_is_read_consistency(false);
int64_t total_fullfilled_num = 0;
int64_t total_row_counter = 0;
int64_t total_timeu = 0;
while (true)
{
int64_t timeu = tbsys::CTimeUtil::getTime();
int err = client.ups_scan(scan_param, scanner, timeout);
timeu = tbsys::CTimeUtil::getTime() - timeu;
if (OB_SUCCESS != err)
{
fprintf(stdout, "[%s] err=%d\n", __FUNCTION__, err);
break;
}
else
{
int64_t row_counter = 0;
while (OB_SUCCESS == scanner.next_cell())
{
ObCellInfo *ci = NULL;
bool is_row_changed = false;
scanner.get_cell(&ci, &is_row_changed);
//fprintf(stdout, "%s\n", updateserver::print_cellinfo(ci, "CLI_SCAN"));
if (is_row_changed)
{
row_counter++;
}
}
bool is_fullfilled = false;
int64_t fullfilled_num = 0;
ObRowkey last_rk;
scanner.get_last_row_key(last_rk);
scanner.get_is_req_fullfilled(is_fullfilled, fullfilled_num);
fprintf(stdout, "[SINGLE_SCAN] is_fullfilled=%s fullfilled_num=%ld row_counter=%ld timeu=%ld last_row_key=[%s]\n",
STR_BOOL(is_fullfilled), fullfilled_num, row_counter, timeu, to_cstring(last_rk));
total_fullfilled_num += fullfilled_num;
total_row_counter += row_counter;
total_timeu += timeu;
if (is_fullfilled)
{
break;
}
else
{
const_cast<ObNewRange*>(scan_param.get_range())->start_key_ = last_rk;
const_cast<ObNewRange*>(scan_param.get_range())->border_flag_.unset_min_value();
const_cast<ObNewRange*>(scan_param.get_range())->border_flag_.unset_inclusive_start();
}
}
}
fprintf(stdout, "[TOTAL_SCAN] total_fullfilled_num=%ld total_row_counter=%ld total_timeu=%ld\n",
total_fullfilled_num, total_row_counter, total_timeu);
}
int get_ups_param(ObGetParam & param, const ObScanner & cs_result)
{
int err = OB_SUCCESS;
ObReadParam &read_param = param;
err = get_ups_read_param(read_param,cs_result);
if (OB_SUCCESS == err)
{
bool is_fullfilled = false;
int64_t fullfilled_cell_num = 0;
err = cs_result.get_is_req_fullfilled(is_fullfilled, fullfilled_cell_num);
if (OB_SUCCESS == err && fullfilled_cell_num < param.get_cell_size())
{
err = param.rollback(param.get_cell_size() - fullfilled_cell_num);
if ((err != OB_SUCCESS) || (param.get_cell_size() != fullfilled_cell_num))
{
TBSYS_LOG(WARN, "check param rollback failed:full_fill[%ld], param_size[%ld]",
fullfilled_cell_num, param.get_cell_size());
}
}
}
return err;
}
void print_scanner(ObScanner &scanner)
{
int64_t row_counter = 0;
while (OB_SUCCESS == scanner.next_cell())
{
ObCellInfo *ci = NULL;
bool is_row_changed = false;
scanner.get_cell(&ci, &is_row_changed);
fprintf(stdout, "%s\n", common::print_cellinfo(ci, "CLI_SCAN"));
if (is_row_changed)
{
row_counter++;
}
}
bool is_fullfilled = false;
int64_t fullfilled_num = 0;
ObRowkey last_rk;
scanner.get_last_row_key(last_rk);
scanner.get_is_req_fullfilled(is_fullfilled, fullfilled_num);
fprintf(stdout, "is_fullfilled=%s fullfilled_num=%ld row_counter=%ld last_row_key=[%s]\n",
STR_BOOL(is_fullfilled), fullfilled_num, row_counter, to_cstring(last_rk));
}
int param_get(const char *fname, MockClient &client)
{
int err = OB_SUCCESS;
int fd = open(fname, O_RDONLY);
if (-1 == fd)
{
fprintf(stdout, "open [%s] fail errno=%u\n", fname, errno);
err = OB_ERROR;
}
else
{
struct stat st;
fstat(fd, &st);
char *buf = (char*)malloc(st.st_size);
if (NULL == buf)
{
fprintf(stdout, "malloc buf fail size=%ld\n", st.st_size);
err = OB_ERROR;
}
else
{
ssize_t __attribute__((unused)) ret = read(fd, buf, st.st_size);
int64_t pos = 0;
ObGetParam get_param;
if (OB_SUCCESS != get_param.deserialize(buf, st.st_size, pos))
{
fprintf(stdout, "deserialize get_param fail\n");
err = OB_ERROR;
}
else
{
get_param.set_is_read_consistency(false);
ObScanner scanner;
int err = client.ups_get(get_param, scanner, timeout);
fprintf(stdout, "[%s] err=%d\n", __FUNCTION__, err);
if (OB_SUCCESS == err)
{
int64_t row_counter = 0;
while (OB_SUCCESS == scanner.next_cell())
{
ObCellInfo *ci = NULL;
bool is_row_changed = false;
scanner.get_cell(&ci, &is_row_changed);
fprintf(stdout, "%s\n", common::print_cellinfo(ci, "CLI_GET"));
if (is_row_changed)
{
row_counter++;
}
}
bool is_fullfilled = false;
int64_t fullfilled_num = 0;
ObRowkey last_rk;
scanner.get_last_row_key(last_rk);
scanner.get_is_req_fullfilled(is_fullfilled, fullfilled_num);
fprintf(stdout, "is_fullfilled=%s fullfilled_num=%ld row_counter=%ld last_row_key=[%s]\n",
STR_BOOL(is_fullfilled), fullfilled_num, row_counter, to_cstring(last_rk));
}
}
free(buf);
}
close(fd);
}
return err;
}
TEST(TestObScanner, add_cell)
{
ObScanner fem;
fem.set_is_req_fullfilled(true, 1011);
fem.set_data_version(170);
char buf[BUFSIZ];
int64_t pospos = 0;
fem.serialize(buf, BUFSIZ, pospos);
FILE *fd;
fd = fopen("tmptest", "w+");
fwrite(buf, 1, pospos, fd);
fclose(fd);
ObScanner os;
ObScanner ost;
/*** added by wushi ***/
int64_t fullfilled_item = 0;
int64_t tmp = 0;
bool is_fullfilled = true;
bool tmp_bool = false;
/// initialize
ASSERT_EQ(os.get_is_req_fullfilled(tmp_bool, fullfilled_item),0);
EXPECT_FALSE(tmp_bool);
ASSERT_EQ(fullfilled_item,0);
/// set fullfilled
is_fullfilled = true;
fullfilled_item = 1000;
os.set_is_req_fullfilled(is_fullfilled,fullfilled_item);
ASSERT_EQ(os.get_is_req_fullfilled(tmp_bool, tmp),0);
ASSERT_EQ(tmp_bool,is_fullfilled);
ASSERT_EQ(tmp,fullfilled_item);
/*** added by wushi ***/
ObCellInfo oci;
oci.table_name_.assign((char*)"table1", 6);
oci.row_key_.assign((char*)"row1", 4);
oci.column_name_.assign((char*)"column1", 7);
oci.value_.set_int(0xff);
ost.set_mem_size_limit(20);
ASSERT_EQ(OB_SIZE_OVERFLOW, ost.add_cell(oci));
os.set_mem_size_limit(1024 * 1024 * 2);
ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));
oci.column_name_.assign((char*)"column2", 7);
oci.value_.set_int(0xfe);
ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));
ObScannerIterator iter = os.begin();
ObCellInfo tci;
ASSERT_EQ(OB_SUCCESS, iter.get_cell(tci));
ASSERT_EQ(string("table1"), string(tci.table_name_.ptr(), tci.table_name_.length()));
ASSERT_EQ(string("row1"), string(tci.row_key_.ptr(), tci.row_key_.length()));
ASSERT_EQ(string("column1"), string(tci.column_name_.ptr(), tci.column_name_.length()));
ASSERT_EQ(OB_SUCCESS, tci.value_.get_int(tmp));
ASSERT_EQ(0xff, tmp);
iter++;
ASSERT_EQ(OB_SUCCESS, iter.get_cell(tci));
ASSERT_EQ(string("table1"), string(tci.table_name_.ptr(), tci.table_name_.length()));
ASSERT_EQ(string("row1"), string(tci.row_key_.ptr(), tci.row_key_.length()));
ASSERT_EQ(string("column2"), string(tci.column_name_.ptr(), tci.column_name_.length()));
ASSERT_EQ(OB_SUCCESS, tci.value_.get_int(tmp));
ASSERT_EQ(0xfe, tmp);
iter++;
ASSERT_EQ(true, os.end() == iter);
//ObCellInfo oci;
char row_key_buffer[1024];
oci.table_name_.assign((char*)"table1", 6);
sprintf(row_key_buffer, "row1");
oci.row_key_.assign(row_key_buffer, 4);
oci.column_name_.assign((char*)"column1", 7);
oci.value_.set_int(0xff);
ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));
oci.table_name_.assign((char*)"table1", 6);
sprintf(row_key_buffer, "row1");
oci.row_key_.assign(row_key_buffer, 4);
oci.column_name_.assign((char*)"column2", 7);
oci.value_.set_int(0xee);
os.set_mem_size_limit(20);
ASSERT_EQ(OB_SIZE_OVERFLOW, os.add_cell(oci));
os.set_mem_size_limit(1024 * 1024 * 2);
ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));
oci.table_name_.assign((char*)"table1", 6);
sprintf(row_key_buffer, "row2");
oci.row_key_.assign(row_key_buffer, 4);
oci.column_name_.assign((char*)"column1", 7);
oci.value_.set_int(0xdd);
ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));
oci.table_name_.assign((char*)"table2", 6);
sprintf(row_key_buffer, "row2");
oci.row_key_.assign(row_key_buffer, 4);
oci.column_name_.assign((char*)"column1", 7);
oci.value_.set_int(0xcc);
ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));
oci.table_name_.assign((char*)"table3", 6);
sprintf(row_key_buffer, "row2");
oci.row_key_.assign(row_key_buffer, 4);
oci.column_name_.assign((char*)"column1", 7);
oci.value_.set_int(0xbb);
ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));
fprintf(stdout, "size=%ld\n", os.get_serialize_size());
char buffer[2048];
int64_t pos = 0;
ASSERT_EQ(OB_SUCCESS, os.serialize(buffer, 1024, pos));
ASSERT_EQ(pos, os.get_serialize_size());
fd = fopen("./test.data.before_rollback", "w+");
fwrite(buffer, 1, 1024, fd);
fclose(fd);
for (iter = os.begin(); iter != os.end(); iter++)
{
ObCellInfo ci;
ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci));
fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s]\n",
ci.table_name_.length(), ci.table_name_.ptr(),
ci.row_key_.length(), ci.row_key_.ptr(),
ci.column_name_.length(), ci.column_name_.ptr());
}
fprintf(stdout, "==============================\n");
while (OB_SUCCESS == os.next_cell())
{
ObCellInfo *ci = NULL;
bool is_row_changed;
ASSERT_EQ(OB_SUCCESS, os.get_cell(&ci, &is_row_changed));
fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s] "
"table_id=[%lu] column_id=[%lu] row_changed=[%d]\n",
ci->table_name_.length(), ci->table_name_.ptr(),
ci->row_key_.length(), ci->row_key_.ptr(),
ci->column_name_.length(),ci->column_name_.ptr(),
ci->table_id_, ci->column_id_, is_row_changed);
}
os.reset_iter();
fprintf(stdout, "==============================\n");
ASSERT_EQ(OB_SUCCESS, os.rollback());
fprintf(stdout, "size=%ld\n", os.get_serialize_size());
pos = 0;
ASSERT_EQ(OB_SUCCESS, os.serialize(buffer, 1024, pos));
ASSERT_EQ(pos, os.get_serialize_size());
fd = fopen("./test.data.after_rollback", "w+");
fwrite(buffer, 1, 1024, fd);
fclose(fd);
for (iter = os.begin(); iter != os.end(); iter++)
{
ObCellInfo ci;
ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci));
fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s]\n",
ci.table_name_.length(), ci.table_name_.ptr(),
ci.row_key_.length(), ci.row_key_.ptr(),
ci.column_name_.length(), ci.column_name_.ptr());
ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci));
fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s]\n",
ci.table_name_.length(), ci.table_name_.ptr(),
ci.row_key_.length(), ci.row_key_.ptr(),
ci.column_name_.length(), ci.column_name_.ptr());
}
fprintf(stdout, "==============================\n");
while (OB_SUCCESS == os.next_cell())
{
ObCellInfo *ci = NULL;
bool is_row_changed;
ASSERT_EQ(OB_SUCCESS, os.get_cell(&ci, &is_row_changed));
fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s] "
"table_id=[%lu] column_id=[%lu] row_changed=[%d]\n",
ci->table_name_.length(), ci->table_name_.ptr(),
ci->row_key_.length(), ci->row_key_.ptr(),
ci->column_name_.length(),ci->column_name_.ptr(),
ci->table_id_, ci->column_id_, is_row_changed);
ASSERT_EQ(OB_SUCCESS, os.get_cell(&ci, &is_row_changed));
fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s] "
"table_id=[%lu] column_id=[%lu] row_changed=[%d]\n",
ci->table_name_.length(), ci->table_name_.ptr(),
ci->row_key_.length(), ci->row_key_.ptr(),
ci->column_name_.length(),ci->column_name_.ptr(),
ci->table_id_, ci->column_id_, is_row_changed);
}
os.reset_iter();
fprintf(stdout, "==============================\n");
pos = 0;
ASSERT_EQ(OB_SUCCESS, os.deserialize(buffer, os.get_serialize_size(), pos));
ASSERT_EQ(pos, os.get_serialize_size());
for (iter = os.begin(); iter != os.end(); iter++)
{
ObCellInfo ci;
ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci));
fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s]\n",
ci.table_name_.length(), ci.table_name_.ptr(),
ci.row_key_.length(), ci.row_key_.ptr(),
ci.column_name_.length(), ci.column_name_.ptr());
ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci));
fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s]\n",
ci.table_name_.length(), ci.table_name_.ptr(),
ci.row_key_.length(), ci.row_key_.ptr(),
ci.column_name_.length(), ci.column_name_.ptr());
}
fprintf(stdout, "==============================\n");
while (OB_SUCCESS == os.next_cell())
{
ObCellInfo *ci = NULL;
bool is_row_changed;
ASSERT_EQ(OB_SUCCESS, os.get_cell(&ci, &is_row_changed));
fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s] "
"table_id=[%lu] column_id=[%lu] row_changed=[%d]\n",
ci->table_name_.length(), ci->table_name_.ptr(),
ci->row_key_.length(), ci->row_key_.ptr(),
ci->column_name_.length(),ci->column_name_.ptr(),
ci->table_id_, ci->column_id_, is_row_changed);
}
os.reset_iter();
/// set fullfilled
// ASSERT_EQ(os.get_is_req_fullfilled(tmp_bool, tmp),0);
// ASSERT_EQ(tmp_bool,is_fullfilled);
// ASSERT_EQ(tmp,fullfilled_item);
{
ObRange range;
range.border_flag_.set_inclusive_start();
range.start_key_.assign((char*)"11111", 5);
range.end_key_.assign((char*)"22222", 5);
ObRange range2;
range2.border_flag_.set_inclusive_start();
range2.start_key_.assign((char*)"11111", 5);
range2.end_key_.assign((char*)"22222", 5);
int64_t pos = 0;
int64_t data_len = 0;
is_fullfilled = true;
fullfilled_item = 1212;
ObScanner os1;
ObScanner os2;
ASSERT_EQ(os1.set_is_req_fullfilled(is_fullfilled, fullfilled_item),0);
os1.set_data_version(100);
ASSERT_EQ(os1.set_range(range),0);
range.border_flag_.set_inclusive_end();
char buf[1024];
ASSERT_EQ(os1.serialize(buf,sizeof(buf),pos),0);
data_len = pos;
pos = 0;
ASSERT_EQ(os2.deserialize(buf, data_len, pos),0);
memset(buf, 0x00, 1024);
ASSERT_EQ(os2.get_is_req_fullfilled(tmp_bool, tmp),0);
EXPECT_EQ(is_fullfilled, tmp_bool);
ASSERT_EQ(fullfilled_item, tmp);
ASSERT_EQ(os1.get_data_version(),100);
ObRange range3;
ASSERT_EQ(OB_SUCCESS, os2.get_range(range3));
ASSERT_EQ(range2.border_flag_.get_data(), range3.border_flag_.get_data());
ASSERT_EQ(string(range3.start_key_.ptr(), range3.start_key_.length()),
string(range2.start_key_.ptr(), range2.start_key_.length()));
ASSERT_EQ(string(range3.end_key_.ptr(), range3.end_key_.length()),
string(range2.end_key_.ptr(), range2.end_key_.length()));
const int times = 1000000;
ObScanner os3;
ObCellInfo cinull;
cinull.table_name_.assign((char*)"hello", 5);
cinull.value_.set_ext(ObActionFlag::OP_INSERT);
ASSERT_EQ(OB_ERROR, os3.add_cell(cinull));
cinull.value_.set_ext(ObActionFlag::OP_DEL_ROW);
ASSERT_EQ(OB_SUCCESS, os3.add_cell(cinull));
cinull.value_.set_ext(ObActionFlag::OP_ROW_DOES_NOT_EXIST);
ASSERT_EQ(OB_SUCCESS, os3.add_cell(cinull));
int i = 0;
for (; i < times; i++)
{
if (OB_SUCCESS != os3.add_cell(oci))
{
break;
}
}
const int64_t buf_len = 1 << 22;
char* buff = (char*)ob_malloc(buf_len);
int64_t ppos = 0;
int64_t len = os3.get_serialize_size();
ASSERT_EQ(OB_BUF_NOT_ENOUGH, os3.serialize(buff, len - 4, ppos));
ASSERT_EQ(OB_SUCCESS, os3.serialize(buff, buf_len, ppos));
int64_t npos = 0;
ObScanner os4;
ASSERT_EQ(OB_ERROR, os4.deserialize(buff, 10, npos));
ASSERT_EQ(OB_SUCCESS, os4.deserialize(buff, ppos, npos));
ASSERT_EQ(npos, ppos);
int64_t otmp;
oci.value_.get_int(otmp);
ObCellInfo *oo;
int t = 0;
ASSERT_EQ(OB_SUCCESS, os4.next_cell());
ASSERT_EQ(OB_SUCCESS, os4.get_cell(&oo));
ASSERT_EQ(0, oo->row_key_.length());
ASSERT_EQ(string(cinull.table_name_.ptr(), cinull.table_name_.length()), string(oo->table_name_.ptr(), oo->table_name_.length()));
ASSERT_EQ(OB_SUCCESS, oo->value_.get_ext(tmp));
ASSERT_EQ(+ObActionFlag::OP_DEL_ROW, tmp);
ASSERT_EQ(OB_SUCCESS, os4.next_cell());
ASSERT_EQ(OB_SUCCESS, os4.get_cell(&oo));
ASSERT_EQ(string(cinull.table_name_.ptr(), cinull.table_name_.length()), string(oo->table_name_.ptr(), oo->table_name_.length()));
ASSERT_EQ(OB_SUCCESS, oo->value_.get_ext(tmp));
ASSERT_EQ(+ObActionFlag::OP_ROW_DOES_NOT_EXIST, tmp);
while (OB_SUCCESS == os4.next_cell())
{
ASSERT_EQ(OB_SUCCESS, os4.get_cell(&oo));
t++;
ASSERT_EQ(string(oci.table_name_.ptr(), oci.table_name_.length()), string(oo->table_name_.ptr(), oo->table_name_.length()));
ASSERT_EQ(string(oci.row_key_.ptr(), oci.row_key_.length()), string(oo->row_key_.ptr(), oo->row_key_.length()));
ASSERT_EQ(string(oci.column_name_.ptr(), oci.column_name_.length()), string(oo->column_name_.ptr(), oo->column_name_.length()));
ASSERT_EQ(OB_SUCCESS, oo->value_.get_int(tmp));
ASSERT_EQ(otmp, tmp);
}
ASSERT_EQ(t, i);
ob_free(buff);
}
ObCellInfo st;
st.table_name_.assign((char*)"table1", 6);
st.row_key_.assign((char*)"row1", 4);
st.column_name_.assign((char*)"column1", 7);
st.value_.set_int(0xff);
ObScanner ss;
ss.add_cell(st);
st.table_name_.assign((char*)"table2", 6);
st.row_key_.assign((char*)"row1", 4);
st.column_name_.assign((char*)"column1", 7);
st.value_.set_int(0xff);
ss.add_cell(st);
ObCellInfo *sto;
bool is_row_changed;
ASSERT_EQ(OB_SUCCESS, ss.next_cell());
ASSERT_EQ(OB_SUCCESS, ss.get_cell(&sto, &is_row_changed));
ASSERT_EQ(true, is_row_changed);
ASSERT_EQ(OB_SUCCESS, ss.next_cell());
ASSERT_EQ(OB_SUCCESS, ss.get_cell(&sto, &is_row_changed));
ASSERT_EQ(true, is_row_changed);
ASSERT_EQ(OB_ITER_END, ss.next_cell());
}
void scan_check_all(MutatorBuilder &mb, MockClient &client, const int64_t table_start_version, const bool using_id, const bool check)
{
for (int64_t i = 0; i < mb.get_schema_num(); ++i)
{
ObScanParam scan_param;
prepare_scan_param(scan_param, mb.get_schema(i), table_start_version, using_id);
int ret = OB_SUCCESS;
bool is_fullfilled = false;
int64_t fullfilled_item_num = 0;
RowChecker rc;
int64_t row_counter = 0;
ObScanner scanner;
while (!is_fullfilled
&& OB_SUCCESS == ret)
{
int64_t timeu = tbsys::CTimeUtil::getTime();
ret = client.ups_scan(scan_param, scanner, TIMEOUT_MS);
TBSYS_LOG(INFO, "scan ret=%d timeu=%ld", ret, tbsys::CTimeUtil::getTime() - timeu);
if (check
&& OB_SUCCESS == ret)
{
while (OB_SUCCESS == scanner.next_cell())
{
ObCellInfo *ci = NULL;
bool is_row_changed = false;
if (OB_SUCCESS == scanner.get_cell(&ci, &is_row_changed))
{
if (!using_id)
{
trans_name2id(*ci, mb.get_schema(i));
}
if (is_row_changed && 0 != rc.cell_num())
{
std::string row_key_str(rc.get_cur_rowkey().ptr(), 0, rc.get_cur_rowkey().length());
//fprintf(stderr, "[%.*s] ", rc.get_cur_rowkey().length(), rc.get_cur_rowkey().ptr());
bool get_row_bret = get_check_row(mb.get_schema(i), rc.get_cur_rowkey(), mb.get_cellinfo_builder(i), client, table_start_version, using_id);
//fprintf(stderr, "[get_row check_ret=%d] ", bret);
bool cell_info_bret = rc.check_row(mb.get_cellinfo_builder(i), mb.get_schema(i));
//fprintf(stderr, "[cell_info check_ret=%d]\n", bret);
TBSYS_LOG(INFO, "[%s] [get_row check_ret=%d] [cell_info check_ret=%d]", row_key_str.c_str(), get_row_bret, cell_info_bret);
}
if (is_row_changed
&& 0 != rc.rowkey_num()
&& rc.is_prefix_changed(ci->row_key_))
{
std::string row_key_str(rc.get_last_rowkey().ptr(), 0, rc.get_last_rowkey().length());
//fprintf(stderr, "[%.*s] ", rc.get_last_rowkey().length(), rc.get_last_rowkey().ptr());
bool bret = rc.check_rowkey(mb.get_rowkey_builder(i));
//fprintf(stderr, "row_key check_ret=%d\n", bret);
TBSYS_LOG(INFO, "[%s] [row_key check_ret=%d]", row_key_str.c_str(), bret);
}
rc.add_cell(ci);
if (is_row_changed)
{
rc.add_rowkey(ci->row_key_);
row_counter++;
}
}
}
if (0 != rc.cell_num())
{
std::string row_key_str(rc.get_cur_rowkey().ptr(), 0, rc.get_cur_rowkey().length());
//fprintf(stderr, "[%.*s] ", rc.get_cur_rowkey().length(), rc.get_cur_rowkey().ptr());
bool get_row_bret = get_check_row(mb.get_schema(i), rc.get_cur_rowkey(), mb.get_cellinfo_builder(i), client, table_start_version, using_id);
//fprintf(stderr, "[get_row check_ret=%d] ", bret);
bool cell_info_bret = rc.check_row(mb.get_cellinfo_builder(i), mb.get_schema(i));
//fprintf(stderr, "[cell_info check_ret=%d]\n", bret);
TBSYS_LOG(INFO, "[%s] [get_row check_ret=%d] [cell_info check_ret=%d]", row_key_str.c_str(), get_row_bret, cell_info_bret);
}
}
scanner.get_is_req_fullfilled(is_fullfilled, fullfilled_item_num);
ObRange *range = const_cast<ObRange*>(scan_param.get_range());
scanner.get_last_row_key(range->start_key_);
range->border_flag_.unset_min_value();
range->border_flag_.unset_inclusive_start();
}
if (check
&& 0 != rc.rowkey_num())
{
std::string row_key_str(rc.get_last_rowkey().ptr(), 0, rc.get_last_rowkey().length());
//fprintf(stderr, "[%.*s] ", rc.get_last_rowkey().length(), rc.get_last_rowkey().ptr());
bool bret = rc.check_rowkey(mb.get_rowkey_builder(i));
//fprintf(stderr, "row_key check_ret=%d\n", bret);
TBSYS_LOG(INFO, "[%s] [row_key check_ret=%d]", row_key_str.c_str(), bret);
}
TBSYS_LOG(INFO, "table_id=%lu row_counter=%ld", mb.get_schema(i).get_table_id(), row_counter);
}
}
int ObMergerRpcStub::scan(
const int64_t timeout, const ObServer & server,
const ObScanParam & scan_param, ObScanner & scanner) const
{
int ret = OB_SUCCESS;
ObDataBuffer data_buff;
int64_t start_time = tbsys::CTimeUtil::getTime();
ret = get_rpc_buffer(data_buff);
// step 1. serialize ObScanParam to the data_buff
if (OB_SUCCESS == ret)
{
ret = scan_param.serialize(data_buff.get_data(), data_buff.get_capacity(),
data_buff.get_position());
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "serialize scan param failed:ret[%d]", ret);
}
}
// step 2. send request for scan data
if (OB_SUCCESS == ret)
{
ret = rpc_frame_->send_request(server, OB_SCAN_REQUEST, DEFAULT_VERSION,
timeout, data_buff);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(WARN, "send scan request to server failed:ret[%d], timeout=%ld",
ret, timeout);
}
}
// step 3. deserialize the response result
int64_t pos = 0;
if (OB_SUCCESS == ret)
{
ObResultCode result_code;
ret = result_code.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "deserialize result failed:pos[%ld], ret[%d]", pos, ret);
}
else
{
ret = result_code.result_code_;
}
}
// step 4. deserialize the scanner
if (OB_SUCCESS == ret)
{
ret = scanner.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "deserialize scanner from buff failed:pos[%ld], ret[%d]", pos, ret);
}
}
// write debug log
const int32_t MAX_SERVER_ADDR_SIZE = 128;
char server_addr[MAX_SERVER_ADDR_SIZE];
server.to_string(server_addr, MAX_SERVER_ADDR_SIZE);
bool is_fullfilled = false;
int64_t fullfilled_item_num = 0;
scanner.get_is_req_fullfilled(is_fullfilled,fullfilled_item_num);
if (OB_SUCCESS == ret)
{
TBSYS_LOG(DEBUG, "scan data succ from server:addr[%s]", server_addr);
}
else
{
TBSYS_LOG(WARN, "scan data failed from server:addr[%s], "
"version_range=%s, ret[%d]",
server_addr, range2str(scan_param.get_version_range()), ret);
}
int64_t consume_time = tbsys::CTimeUtil::getTime() - start_time;
if ((double)consume_time > (double)timeout * 0.8)
{
TBSYS_LOG(WARN, "slow ups scan, ups_addr=%s, timeout=%ld, consume=%ld",
server.to_cstring(), timeout, consume_time);
}
return ret;
}
int ObMergerSortedOperator::add_sharding_result(ObScanner & sharding_res, const ObNewRange &query_range,
bool &is_finish)
{
int err = OB_SUCCESS;
if ((OB_SUCCESS == err) && (NULL == scan_param_))
{
TBSYS_LOG(WARN,"operator was not initialized yet [scan_param_:%p]", scan_param_);
err = OB_INVALID_ARGUMENT;
}
if ((OB_SUCCESS == err) && (sharding_result_count_ >= MAX_SHARDING_RESULT_COUNT))
{
TBSYS_LOG(WARN,"array is full [MAX_SHARDING_RESULT_COUNT:%ld,sharding_result_count_:%ld]",
MAX_SHARDING_RESULT_COUNT, sharding_result_count_);
err = OB_ARRAY_OUT_OF_RANGE;
}
/// @todo (wushi [email protected]) check fullfill item number and size property
bool is_fullfilled = false;
int64_t fullfilled_item_num = 0;
ObRowkey last_row_key;
ObNewRange cs_tablet_range;
if ((OB_SUCCESS == err) && (OB_SUCCESS != (err = sharding_res.get_is_req_fullfilled(is_fullfilled,fullfilled_item_num))))
{
TBSYS_LOG(WARN,"fail to get fullfilled info from sharding result [err:%d]", err);
}
if (OB_SUCCESS == err)
{
if (OB_SUCCESS == (err = sharding_res.get_last_row_key(last_row_key)))
{
}
else if (OB_ENTRY_NOT_EXIST == err)
{
err = OB_SUCCESS;
}
else
{
TBSYS_LOG(WARN,"fail to get last rowkey from sharding result [err:%d]", err);
}
}
if (OB_SUCCESS == err)
{
if (is_fullfilled)
{
if ((OB_SUCCESS == err) && (OB_SUCCESS != (err = sharding_res.get_range(cs_tablet_range))))
{
TBSYS_LOG(WARN,"fail to get tablet range from sharding result [err:%d]", err);
}
if (OB_SUCCESS == err)
{
if (ScanFlag::FORWARD == scan_param_->get_scan_direction())
{
last_row_key = cs_tablet_range.end_key_;
}
else
{
last_row_key = cs_tablet_range.start_key_;
}
}
}
if (OB_SUCCESS == err)
{
fullfilled_item_num = sharding_res.get_row_num();
sharding_result_arr_[sharding_result_count_].init(sharding_res,query_range,*scan_param_, last_row_key, fullfilled_item_num);
sharding_result_count_ ++;
}
}
is_finish = false;
if (OB_SUCCESS == err)
{
sort(is_finish, &sharding_res);
}
TBSYS_LOG(DEBUG, "add sharding result. is_finish=%d, err=%d", is_finish, err);
return err;
}
int get_next_param(
const ObScanParam &org_scan_param, const ObScanner &prev_scan_result,
ObScanParam *scan_param, ObMemBuf &range_buffer)
{
int err = OB_SUCCESS;
const ObReadParam &org_read_param = org_scan_param;
ObReadParam *read_param = scan_param;
ObNewRange tablet_range;
const ObNewRange *org_scan_range = NULL;
ObNewRange cur_range;
if(NULL != scan_param)
{
scan_param->reset();
*read_param = org_read_param;
}
bool request_fullfilled = false;
int64_t fullfilled_row_num = 0;
err = prev_scan_result.get_is_req_fullfilled(request_fullfilled,fullfilled_row_num);
if (OB_SUCCESS == err)
{
org_scan_range = org_scan_param.get_range();
if (NULL == org_scan_range)
{
TBSYS_LOG(WARN, "unexpected error, can't get range from org scan param");
err = OB_INVALID_ARGUMENT;
}
}
ObRowkey last_row_key;
if (OB_SUCCESS == err)
{
if (request_fullfilled)
{
err = prev_scan_result.get_range(tablet_range);
if (OB_SUCCESS == err)
{
if (true == is_finish_scan(org_scan_param, tablet_range))
{
err = OB_ITER_END;
}
else if (ScanFlag::FORWARD == org_scan_param.get_scan_direction())
{
last_row_key = tablet_range.end_key_;
}
else
{
last_row_key = tablet_range.start_key_;
}
}
}
else
{
err = prev_scan_result.get_last_row_key(last_row_key);
if (OB_ENTRY_NOT_EXIST == err)
{
/// first time, remain unchanged
}
else if (OB_SUCCESS != err)
{
TBSYS_LOG(WARN, "unexpected error, scanner should contain at least one cell");
}
}
}
if (OB_SUCCESS == err && NULL != scan_param)
{
cur_range =*org_scan_range;
// forward
if (ScanFlag::FORWARD == org_scan_param.get_scan_direction())
{
cur_range.start_key_ = last_row_key;
cur_range.border_flag_.unset_inclusive_start();
}
else
{
cur_range.end_key_ = last_row_key;
if (request_fullfilled)
{
// tablet range start key
cur_range.border_flag_.set_inclusive_end();
}
else
{
// the smallest rowkey
cur_range.border_flag_.unset_inclusive_end();
}
}
}
// first time remain unchanged
if (OB_ENTRY_NOT_EXIST == err)
{
cur_range =*org_scan_range;
err = OB_SUCCESS;
}
if (OB_SUCCESS == err && NULL != scan_param)
{
err = allocate_range_buffer(cur_range, range_buffer);
}
if (OB_SUCCESS == err && NULL != scan_param)
{
scan_param->set(org_scan_param.get_table_id(), org_scan_param.get_table_name(), cur_range);
for (int32_t cell_idx = 0;
cell_idx < org_scan_param.get_column_id_size() && OB_SUCCESS == err;
cell_idx ++)
{
err = scan_param->add_column(org_scan_param.get_column_id()[cell_idx]);
}
if(OB_SUCCESS == err)
{
scan_param->set_scan_size(GET_SCAN_SIZE(org_scan_param.get_scan_size()));
scan_param->set_scan_flag(org_scan_param.get_scan_flag());
}
}
return err;
}
