void multi_update_perf(const int64_t times, const int64_t rn, const int64_t cn)
{
ObRowCompaction rc;
MockIterator iter;
ObScanner scanner;
const int64_t buf_len = 2 * 1024 * 1024;
char *buffer = new char[buf_len];
int64_t pos = 0;
build_iterator(iter, rn, cn);
int64_t timeu = tbsys::CTimeUtil::getTime();
for (int64_t i = 0; i < times; i++)
{
scanner.reset();
iter.reset_iter();
rc.set_iterator(&iter);
pos = 0;
build_scanner(rc, scanner, buffer, buf_len, pos);
}
fprintf(stderr, "row_compaction times=%ld row=%ld cell=%ld/row timeu=%ld size=%ld\n",
times, rn, cn, tbsys::CTimeUtil::getTime() - timeu, pos);
FILE *fd = fopen("./rc.data", "w");
fwrite(buffer, 1, pos, fd);
fclose(fd);
timeu = tbsys::CTimeUtil::getTime();
for (int64_t i = 0; i < times; i++)
{
scanner.reset();
iter.reset_iter();
pos = 0;
build_scanner(iter, scanner, buffer, buf_len, pos);
}
fprintf(stderr, "no_compaction times=%ld row=%ld cell=%ld/row timeu=%ld size=%ld\n",
times, rn, cn, tbsys::CTimeUtil::getTime() - timeu, pos);
fd = fopen("./nrc.data", "w");
fwrite(buffer, 1, pos, fd);
fclose(fd);
delete buffer;
}
TEST(TestObScanner, get_row)
{
ObScanner scanner;
const int table_num = 10;
const int row_num = 20;
const int column_num = 10;
buffer table_name[table_num];
buffer row_key[row_num];
buffer column_name[column_num];
for (int i = 0; i < table_num; i++)
{
table_name[i].assigne("T").appende(i);
}
for (int i = 0; i < row_num; i++)
{
row_key[i].assigne("R").appende(i);
}
for (int i = 0; i < column_num; i++)
{
column_name[i].assigne("C").appende(i);
}
for (int i = 0; i < table_num; i++)
{
for (int j = 0; j < row_num; j++)
{
for (int k = 0; k < column_num; k++)
{
ObCellInfo cell;
cell.table_name_ = table_name[i].get_obstring();
cell.row_key_ = row_key[j].get_obstring();
cell.column_name_ = column_name[k].get_obstring();
cell.value_.set_int(1);
scanner.add_cell(cell);
}
}
}
ObScanner::RowIterator it = scanner.row_begin();
ASSERT_EQ(it == scanner.row_begin(), true);
for (int i = 0; i < table_num; i++)
{
for (int j = 0; j < row_num; j++)
{
ObCellInfo *row;
int64_t num = 0;
ASSERT_EQ(OB_SUCCESS, scanner.next_row());
ASSERT_EQ(OB_SUCCESS, scanner.get_row(&row, &num));
ASSERT_EQ(column_num, num);
}
}
ASSERT_EQ(OB_ITER_END, scanner.next_row());
buffer buf(1 << 21);
scanner.serialize(buf.ptre(), buf.capacity(), buf.length());
printf("%ld\n", buf.length());
ObScanner dscanner;
int64_t pos = 0;
dscanner.deserialize(buf.ptre(), buf.length(), pos);
for (int i = 0; i < table_num; i++)
{
for (int j = 0; j < row_num; j++)
{
ObCellInfo *row;
int64_t num = 0;
ASSERT_EQ(OB_SUCCESS, dscanner.next_row());
ASSERT_EQ(OB_SUCCESS, dscanner.get_row(&row, &num));
ASSERT_EQ(column_num, num);
}
}
ASSERT_EQ(OB_ITER_END, dscanner.next_row());
scanner.clear();
scanner.set_mem_size_limit(256);
int j = 0, k = 0;
int err = OB_SUCCESS;
for (; j < row_num; j++)
{
for (k = 0; k < column_num; k++)
{
ObCellInfo cell;
cell.table_name_ = table_name[0].get_obstring();
cell.row_key_ = row_key[j].get_obstring();
cell.column_name_ = column_name[k].get_obstring();
cell.value_.set_int(1);
if (OB_SUCCESS != (err = scanner.add_cell(cell)))
{
break;
}
}
if (OB_SUCCESS != err)
{
break;
}
}
if (OB_SUCCESS != err)
{
scanner.rollback();
}
for (int i = 0; i < j; i++)
{
ObCellInfo *row;
int64_t num = 0;
ASSERT_EQ(OB_SUCCESS, scanner.next_row());
ASSERT_EQ(OB_SUCCESS, scanner.get_row(&row, &num));
ASSERT_EQ(column_num, num);
}
ASSERT_EQ(OB_ITER_END, scanner.next_row());
buf.length() = 0;
scanner.serialize(buf.ptre(), buf.capacity(), buf.length());
printf("%ld\n", buf.length());
dscanner.reset();
pos = 0;
dscanner.deserialize(buf.ptre(), buf.length(), pos);
printf("row_num=%ld\n", dscanner.get_row_num());
for (int i = 0; i < j; i++)
{
ObCellInfo *row;
int64_t num = 0;
ASSERT_EQ(OB_SUCCESS, dscanner.next_row());
ASSERT_EQ(OB_SUCCESS, dscanner.get_row(&row, &num));
ASSERT_EQ(column_num, num);
}
ASSERT_EQ(OB_ITER_END, dscanner.next_row());
}
int ObMetaTable3::MyIterator::scan_beginning(ConstIterator &meta_it, const KeyRange &search_range, ObScanner& scanner)
{
int ret = OB_SUCCESS;
scanner.reset();
ObString null_string;
ObScanParam *scan_param = aggr_.scan_param_allocator_.alloc();
ObScanner *my_location = aggr_.scanner_allocator_.alloc();
if (NULL == scan_param)
{
TBSYS_LOG(ERROR, "failed to alloc scan param");
ret = OB_ALLOCATE_MEMORY_FAILED;
}
else if (NULL == my_location)
{
TBSYS_LOG(ERROR, "failed to alloc scanner");
ret = OB_ALLOCATE_MEMORY_FAILED;
}
else
{
scan_param->reset();
my_location->reset();
scan_param->set(my_tid_, null_string, search_range);
if (OB_SUCCESS != (ret = scan_param->set_limit_info(0, SCAN_LIMIT_COUNT)))
{
TBSYS_LOG(WARN, "failed to set limit info, err=%d", ret);
}
// scan all columns
else if (OB_SUCCESS != (ret = add_scan_columns(*scan_param)))
{
TBSYS_LOG(WARN, "failed to add scan columsn, err=%d", ret);
}
// find the locations of the tablet which the startkey is in
else if (OB_SUCCESS != (ret = get_location(meta_it, search_range.start_key_,
search_range.border_flag_.inclusive_start(), *my_location)))
{
TBSYS_LOG(WARN, "failed to get location, err=%d", ret);
}
else if (OB_SUCCESS != (ret = scan_param->set_location_info(*my_location)))
{
TBSYS_LOG(WARN, "failed to add location, err=%d", ret);
}
else if (OB_SUCCESS != (ret = aggr_.scan(*scan_param, scanner)))
{
TBSYS_LOG(WARN, "scan error, err=%d range=%s", ret, to_cstring(search_range));
}
else
{
scanner.reset_row_iter();
read_scanner_row_count_ = 0;
TBSYS_LOG(DEBUG, "scan meta table succ, range=%s", to_cstring(search_range));
}
}
if (NULL != scan_param)
{
aggr_.scan_param_allocator_.free(scan_param);
}
if (NULL != my_location)
{
aggr_.scanner_allocator_.free(my_location);
}
return ret;
}
TEST(TestObScanner, row_num)
{
ObScanner scanner;
const int table_num = 3;
const int row_num = 20;
const int column_num = 5;
buffer table_name[table_num];
buffer row_key[row_num];
buffer column_name[column_num];
for (int i = 0; i < table_num; i++)
{
table_name[i].assigne("T").appende(i);
}
for (int i = 0; i < row_num; i++)
{
row_key[i].assigne("R").appende(i);
}
for (int i = 0; i < column_num; i++)
{
column_name[i].assigne("C").appende(i);
}
for (int i = 0; i < table_num; i++)
{
for (int j = 0; j < row_num; j++)
{
for (int k = 0; k < column_num; k++)
{
ObCellInfo cell;
cell.table_name_ = table_name[i].get_obstring();
cell.row_key_ = row_key[j].get_obstring();
cell.column_name_ = column_name[k].get_obstring();
cell.value_.set_int(1);
scanner.add_cell(cell);
}
}
}
ASSERT_EQ(scanner.get_row_num(), table_num * row_num);
ASSERT_EQ(scanner.get_cell_num(), table_num * row_num * column_num);
scanner.set_whole_result_row_num(1000);
buffer buf(1 << 21);
scanner.serialize(buf.ptre(), buf.capacity(), buf.length());
printf("%ld\n", buf.length());
ObScanner dscanner;
int64_t pos = 0;
dscanner.deserialize(buf.ptre(), buf.length(), pos);
EXPECT_EQ(dscanner.get_row_num(), table_num * row_num);
EXPECT_EQ(dscanner.get_cell_num(), table_num * row_num * column_num);
EXPECT_EQ(dscanner.get_whole_result_row_num(), 1000);
scanner.clear();
scanner.set_mem_size_limit(256);
int j = 0, k = 0;
int err = OB_SUCCESS;
for (; j < row_num; j++)
{
k = 0;
for (; k < column_num; k++)
{
ObCellInfo cell;
cell.table_name_ = table_name[0].get_obstring();
cell.row_key_ = row_key[j].get_obstring();
cell.column_name_ = column_name[k].get_obstring();
cell.value_.set_int(1);
if (OB_SUCCESS != (err = scanner.add_cell(cell)))
{
break;
}
}
if (OB_SUCCESS != err)
{
break;
}
}
if (OB_SUCCESS != err)
{
scanner.rollback();
}
EXPECT_EQ(scanner.get_row_num(), j);
EXPECT_EQ(scanner.get_cell_num(), j * column_num);
buf.length() = 0;
scanner.serialize(buf.ptre(), buf.capacity(), buf.length());
printf("%ld\n", buf.length());
dscanner.reset();
pos = 0;
dscanner.deserialize(buf.ptre(), buf.length(), pos);
EXPECT_EQ(dscanner.get_row_num(), j);
EXPECT_EQ(dscanner.get_cell_num(), j * column_num);
}
int ObScannerLoader::get_decoded_scanner(ObScanner ¶m)
{
ObNewRange range;
ObString column;
ObString table_name;
char value[4096];
if (config_loaded_)
{
TBSYS_LOG(INFO, "creating scanner");
param.reset();
range.reset();
range.table_id_ = table_id_;
TBSYS_LOG(DEBUG, "range_start_=%s", to_cstring(range_start_));
TBSYS_LOG(DEBUG, "range_end_=%s", to_cstring(range_end_));
range.start_key_ = range_start_;
if (range_start_inclusive_)
range.border_flag_.set_inclusive_start();
else
range.border_flag_.unset_inclusive_start();
if (range_start_min_)
range.start_key_.set_min_row();
range.end_key_ = range_end_;
if (range_end_inclusive_)
range.border_flag_.set_inclusive_end();
else
range.border_flag_.unset_inclusive_end();
if (range_end_max_)
range.end_key_.set_max_row();
param.set_range(range);
//param.set_is_req_fullfilled(is_fullfilled_, actual_fullfilled_item_num_);
param.set_is_req_fullfilled(is_fullfilled_, fullfilled_item_num_);
param.set_data_version(data_version_);
FILE *fp = NULL;
if ((fp = fopen(data_file_name_, "rb")) == NULL) {
TBSYS_LOG(ERROR, "Fail to open %s", data_file_name_);
}
else
{
while(fgets(value, 4096, fp))
{
/* only digit and string accepted. if not dig, then string */
int start = 0;
int end = 0;
int cnt = 0;
char buf[4096];
int table_id = 0;
int column_cnt = 0;
ObRowkey row_key;
while(true)
{
while(isspace(value[start]))
{
start++;
}
end = start;
while(!isspace(value[end]) && value[end] != '\0')
{
end++;
}
if (start != end)
{
memset(buf, 0, 4096);
cnt = end - start;
strncpy(buf, &value[start], cnt);
buf[cnt] = '\0';
if (column_cnt == 0)
{
table_id = atoi(buf);
}
else if(column_cnt == 1)
{
// TODO build rowkey
//strcpy(row_key_buf, buf);
//row_key.assign(row_key_buf, cnt);
}
else
{
build_cell_info(table_id, row_key, buf, param);
}
column_cnt++;
start = end;
}
if (value[end] == '\0')
{
break;
}
}
}
fclose(fp);
}
}
return OB_SUCCESS;
}