int ObGetScanProxy::get_compact_scanner(const ObGetParam& get_param, ObScanner& orig_scanner,ObScanner& compact_scanner) { ObTabletManager::ObGetThreadContext*& get_context = tablet_manager_.get_cur_thread_get_contex(); const ObGetParam::ObRowIndex* row_index = get_param.get_row_index(); int64_t row_num = orig_scanner.get_row_num(); ColumnFilter* cf = GET_TSI_MULT(ColumnFilter,TSI_CS_COLUMNFILTER_1); ObTablet* tablet = NULL; int ret = OB_SUCCESS; ObRowkey rowkey; if ((NULL == get_context) || (NULL == row_index) || (NULL == cf)) { TBSYS_LOG(WARN,"get thread context of get failed"); ret = OB_ERROR; } else { int64_t compactsstable_version = get_context->min_compactsstable_version_; if (ObVersion::compare(compactsstable_version,orig_scanner.get_data_version()) > 0) { compact_scanner.set_data_version(compactsstable_version); for(int64_t i=0; (i < row_num) && (OB_SUCCESS == ret); ++i) { tablet = get_context->tablets_[i]; rowkey = get_param[row_index[i].offset_]->row_key_; build_get_column_filter(get_param,row_index[i].offset_,row_index[i].size_,*cf); if ((tablet != NULL) && (ret = get_compact_row(*tablet,rowkey,compactsstable_version,cf,compact_scanner)) != OB_SUCCESS) { TBSYS_LOG(WARN,"get data from compactsstable failed,ret=%d",ret); } } } } return ret; }
TEST(TestObScanner, get_row) { ObScanner scanner; const int table_num = 10; const int row_num = 20; const int column_num = 10; buffer table_name[table_num]; buffer row_key[row_num]; buffer column_name[column_num]; for (int i = 0; i < table_num; i++) { table_name[i].assigne("T").appende(i); } for (int i = 0; i < row_num; i++) { row_key[i].assigne("R").appende(i); } for (int i = 0; i < column_num; i++) { column_name[i].assigne("C").appende(i); } for (int i = 0; i < table_num; i++) { for (int j = 0; j < row_num; j++) { for (int k = 0; k < column_num; k++) { ObCellInfo cell; cell.table_name_ = table_name[i].get_obstring(); cell.row_key_ = row_key[j].get_obstring(); cell.column_name_ = column_name[k].get_obstring(); cell.value_.set_int(1); scanner.add_cell(cell); } } } ObScanner::RowIterator it = scanner.row_begin(); ASSERT_EQ(it == scanner.row_begin(), true); for (int i = 0; i < table_num; i++) { for (int j = 0; j < row_num; j++) { ObCellInfo *row; int64_t num = 0; ASSERT_EQ(OB_SUCCESS, scanner.next_row()); ASSERT_EQ(OB_SUCCESS, scanner.get_row(&row, &num)); ASSERT_EQ(column_num, num); } } ASSERT_EQ(OB_ITER_END, scanner.next_row()); buffer buf(1 << 21); scanner.serialize(buf.ptre(), buf.capacity(), buf.length()); printf("%ld\n", buf.length()); ObScanner dscanner; int64_t pos = 0; dscanner.deserialize(buf.ptre(), buf.length(), pos); for (int i = 0; i < table_num; i++) { for (int j = 0; j < row_num; j++) { ObCellInfo *row; int64_t num = 0; ASSERT_EQ(OB_SUCCESS, dscanner.next_row()); ASSERT_EQ(OB_SUCCESS, dscanner.get_row(&row, &num)); ASSERT_EQ(column_num, num); } } ASSERT_EQ(OB_ITER_END, dscanner.next_row()); scanner.clear(); scanner.set_mem_size_limit(256); int j = 0, k = 0; int err = OB_SUCCESS; for (; j < row_num; j++) { for (k = 0; k < column_num; k++) { ObCellInfo cell; cell.table_name_ = table_name[0].get_obstring(); cell.row_key_ = row_key[j].get_obstring(); cell.column_name_ = column_name[k].get_obstring(); cell.value_.set_int(1); if (OB_SUCCESS != (err = scanner.add_cell(cell))) { break; } } if (OB_SUCCESS != err) { break; } } if (OB_SUCCESS != err) { scanner.rollback(); } for (int i = 0; i < j; i++) { ObCellInfo *row; int64_t num = 0; ASSERT_EQ(OB_SUCCESS, scanner.next_row()); ASSERT_EQ(OB_SUCCESS, scanner.get_row(&row, &num)); ASSERT_EQ(column_num, num); } ASSERT_EQ(OB_ITER_END, scanner.next_row()); buf.length() = 0; scanner.serialize(buf.ptre(), buf.capacity(), buf.length()); printf("%ld\n", buf.length()); dscanner.reset(); pos = 0; dscanner.deserialize(buf.ptre(), buf.length(), pos); printf("row_num=%ld\n", dscanner.get_row_num()); for (int i = 0; i < j; i++) { ObCellInfo *row; int64_t num = 0; ASSERT_EQ(OB_SUCCESS, dscanner.next_row()); ASSERT_EQ(OB_SUCCESS, dscanner.get_row(&row, &num)); ASSERT_EQ(column_num, num); } ASSERT_EQ(OB_ITER_END, dscanner.next_row()); }
TEST(TestObScanner, row_num) { ObScanner scanner; const int table_num = 3; const int row_num = 20; const int column_num = 5; buffer table_name[table_num]; buffer row_key[row_num]; buffer column_name[column_num]; for (int i = 0; i < table_num; i++) { table_name[i].assigne("T").appende(i); } for (int i = 0; i < row_num; i++) { row_key[i].assigne("R").appende(i); } for (int i = 0; i < column_num; i++) { column_name[i].assigne("C").appende(i); } for (int i = 0; i < table_num; i++) { for (int j = 0; j < row_num; j++) { for (int k = 0; k < column_num; k++) { ObCellInfo cell; cell.table_name_ = table_name[i].get_obstring(); cell.row_key_ = row_key[j].get_obstring(); cell.column_name_ = column_name[k].get_obstring(); cell.value_.set_int(1); scanner.add_cell(cell); } } } ASSERT_EQ(scanner.get_row_num(), table_num * row_num); ASSERT_EQ(scanner.get_cell_num(), table_num * row_num * column_num); scanner.set_whole_result_row_num(1000); buffer buf(1 << 21); scanner.serialize(buf.ptre(), buf.capacity(), buf.length()); printf("%ld\n", buf.length()); ObScanner dscanner; int64_t pos = 0; dscanner.deserialize(buf.ptre(), buf.length(), pos); EXPECT_EQ(dscanner.get_row_num(), table_num * row_num); EXPECT_EQ(dscanner.get_cell_num(), table_num * row_num * column_num); EXPECT_EQ(dscanner.get_whole_result_row_num(), 1000); scanner.clear(); scanner.set_mem_size_limit(256); int j = 0, k = 0; int err = OB_SUCCESS; for (; j < row_num; j++) { k = 0; for (; k < column_num; k++) { ObCellInfo cell; cell.table_name_ = table_name[0].get_obstring(); cell.row_key_ = row_key[j].get_obstring(); cell.column_name_ = column_name[k].get_obstring(); cell.value_.set_int(1); if (OB_SUCCESS != (err = scanner.add_cell(cell))) { break; } } if (OB_SUCCESS != err) { break; } } if (OB_SUCCESS != err) { scanner.rollback(); } EXPECT_EQ(scanner.get_row_num(), j); EXPECT_EQ(scanner.get_cell_num(), j * column_num); buf.length() = 0; scanner.serialize(buf.ptre(), buf.capacity(), buf.length()); printf("%ld\n", buf.length()); dscanner.reset(); pos = 0; dscanner.deserialize(buf.ptre(), buf.length(), pos); EXPECT_EQ(dscanner.get_row_num(), j); EXPECT_EQ(dscanner.get_cell_num(), j * column_num); }
int ObMergerSortedOperator::add_sharding_result(ObScanner & sharding_res, const ObNewRange &query_range, bool &is_finish) { int err = OB_SUCCESS; if ((OB_SUCCESS == err) && (NULL == scan_param_)) { TBSYS_LOG(WARN,"operator was not initialized yet [scan_param_:%p]", scan_param_); err = OB_INVALID_ARGUMENT; } if ((OB_SUCCESS == err) && (sharding_result_count_ >= MAX_SHARDING_RESULT_COUNT)) { TBSYS_LOG(WARN,"array is full [MAX_SHARDING_RESULT_COUNT:%ld,sharding_result_count_:%ld]", MAX_SHARDING_RESULT_COUNT, sharding_result_count_); err = OB_ARRAY_OUT_OF_RANGE; } /// @todo (wushi [email protected]) check fullfill item number and size property bool is_fullfilled = false; int64_t fullfilled_item_num = 0; ObRowkey last_row_key; ObNewRange cs_tablet_range; if ((OB_SUCCESS == err) && (OB_SUCCESS != (err = sharding_res.get_is_req_fullfilled(is_fullfilled,fullfilled_item_num)))) { TBSYS_LOG(WARN,"fail to get fullfilled info from sharding result [err:%d]", err); } if (OB_SUCCESS == err) { if (OB_SUCCESS == (err = sharding_res.get_last_row_key(last_row_key))) { } else if (OB_ENTRY_NOT_EXIST == err) { err = OB_SUCCESS; } else { TBSYS_LOG(WARN,"fail to get last rowkey from sharding result [err:%d]", err); } } if (OB_SUCCESS == err) { if (is_fullfilled) { if ((OB_SUCCESS == err) && (OB_SUCCESS != (err = sharding_res.get_range(cs_tablet_range)))) { TBSYS_LOG(WARN,"fail to get tablet range from sharding result [err:%d]", err); } if (OB_SUCCESS == err) { if (ScanFlag::FORWARD == scan_param_->get_scan_direction()) { last_row_key = cs_tablet_range.end_key_; } else { last_row_key = cs_tablet_range.start_key_; } } } if (OB_SUCCESS == err) { fullfilled_item_num = sharding_res.get_row_num(); sharding_result_arr_[sharding_result_count_].init(sharding_res,query_range,*scan_param_, last_row_key, fullfilled_item_num); sharding_result_count_ ++; } } is_finish = false; if (OB_SUCCESS == err) { sort(is_finish, &sharding_res); } TBSYS_LOG(DEBUG, "add sharding result. is_finish=%d, err=%d", is_finish, err); return err; }