void multi_update_perf(const int64_t times, const int64_t rn, const int64_t cn) { ObRowCompaction rc; MockIterator iter; ObScanner scanner; const int64_t buf_len = 2 * 1024 * 1024; char *buffer = new char[buf_len]; int64_t pos = 0; build_iterator(iter, rn, cn); int64_t timeu = tbsys::CTimeUtil::getTime(); for (int64_t i = 0; i < times; i++) { scanner.reset(); iter.reset_iter(); rc.set_iterator(&iter); pos = 0; build_scanner(rc, scanner, buffer, buf_len, pos); } fprintf(stderr, "row_compaction times=%ld row=%ld cell=%ld/row timeu=%ld size=%ld\n", times, rn, cn, tbsys::CTimeUtil::getTime() - timeu, pos); FILE *fd = fopen("./rc.data", "w"); fwrite(buffer, 1, pos, fd); fclose(fd); timeu = tbsys::CTimeUtil::getTime(); for (int64_t i = 0; i < times; i++) { scanner.reset(); iter.reset_iter(); pos = 0; build_scanner(iter, scanner, buffer, buf_len, pos); } fprintf(stderr, "no_compaction times=%ld row=%ld cell=%ld/row timeu=%ld size=%ld\n", times, rn, cn, tbsys::CTimeUtil::getTime() - timeu, pos); fd = fopen("./nrc.data", "w"); fwrite(buffer, 1, pos, fd); fclose(fd); delete buffer; }
TEST(TestObScanner, get_row) { ObScanner scanner; const int table_num = 10; const int row_num = 20; const int column_num = 10; buffer table_name[table_num]; buffer row_key[row_num]; buffer column_name[column_num]; for (int i = 0; i < table_num; i++) { table_name[i].assigne("T").appende(i); } for (int i = 0; i < row_num; i++) { row_key[i].assigne("R").appende(i); } for (int i = 0; i < column_num; i++) { column_name[i].assigne("C").appende(i); } for (int i = 0; i < table_num; i++) { for (int j = 0; j < row_num; j++) { for (int k = 0; k < column_num; k++) { ObCellInfo cell; cell.table_name_ = table_name[i].get_obstring(); cell.row_key_ = row_key[j].get_obstring(); cell.column_name_ = column_name[k].get_obstring(); cell.value_.set_int(1); scanner.add_cell(cell); } } } ObScanner::RowIterator it = scanner.row_begin(); ASSERT_EQ(it == scanner.row_begin(), true); for (int i = 0; i < table_num; i++) { for (int j = 0; j < row_num; j++) { ObCellInfo *row; int64_t num = 0; ASSERT_EQ(OB_SUCCESS, scanner.next_row()); ASSERT_EQ(OB_SUCCESS, scanner.get_row(&row, &num)); ASSERT_EQ(column_num, num); } } ASSERT_EQ(OB_ITER_END, scanner.next_row()); buffer buf(1 << 21); scanner.serialize(buf.ptre(), buf.capacity(), buf.length()); printf("%ld\n", buf.length()); ObScanner dscanner; int64_t pos = 0; dscanner.deserialize(buf.ptre(), buf.length(), pos); for (int i = 0; i < table_num; i++) { for (int j = 0; j < row_num; j++) { ObCellInfo *row; int64_t num = 0; ASSERT_EQ(OB_SUCCESS, dscanner.next_row()); ASSERT_EQ(OB_SUCCESS, dscanner.get_row(&row, &num)); ASSERT_EQ(column_num, num); } } ASSERT_EQ(OB_ITER_END, dscanner.next_row()); scanner.clear(); scanner.set_mem_size_limit(256); int j = 0, k = 0; int err = OB_SUCCESS; for (; j < row_num; j++) { for (k = 0; k < column_num; k++) { ObCellInfo cell; cell.table_name_ = table_name[0].get_obstring(); cell.row_key_ = row_key[j].get_obstring(); cell.column_name_ = column_name[k].get_obstring(); cell.value_.set_int(1); if (OB_SUCCESS != (err = scanner.add_cell(cell))) { break; } } if (OB_SUCCESS != err) { break; } } if (OB_SUCCESS != err) { scanner.rollback(); } for (int i = 0; i < j; i++) { ObCellInfo *row; int64_t num = 0; ASSERT_EQ(OB_SUCCESS, scanner.next_row()); ASSERT_EQ(OB_SUCCESS, scanner.get_row(&row, &num)); ASSERT_EQ(column_num, num); } ASSERT_EQ(OB_ITER_END, scanner.next_row()); buf.length() = 0; scanner.serialize(buf.ptre(), buf.capacity(), buf.length()); printf("%ld\n", buf.length()); dscanner.reset(); pos = 0; dscanner.deserialize(buf.ptre(), buf.length(), pos); printf("row_num=%ld\n", dscanner.get_row_num()); for (int i = 0; i < j; i++) { ObCellInfo *row; int64_t num = 0; ASSERT_EQ(OB_SUCCESS, dscanner.next_row()); ASSERT_EQ(OB_SUCCESS, dscanner.get_row(&row, &num)); ASSERT_EQ(column_num, num); } ASSERT_EQ(OB_ITER_END, dscanner.next_row()); }
int ObMetaTable3::MyIterator::scan_beginning(ConstIterator &meta_it, const KeyRange &search_range, ObScanner& scanner) { int ret = OB_SUCCESS; scanner.reset(); ObString null_string; ObScanParam *scan_param = aggr_.scan_param_allocator_.alloc(); ObScanner *my_location = aggr_.scanner_allocator_.alloc(); if (NULL == scan_param) { TBSYS_LOG(ERROR, "failed to alloc scan param"); ret = OB_ALLOCATE_MEMORY_FAILED; } else if (NULL == my_location) { TBSYS_LOG(ERROR, "failed to alloc scanner"); ret = OB_ALLOCATE_MEMORY_FAILED; } else { scan_param->reset(); my_location->reset(); scan_param->set(my_tid_, null_string, search_range); if (OB_SUCCESS != (ret = scan_param->set_limit_info(0, SCAN_LIMIT_COUNT))) { TBSYS_LOG(WARN, "failed to set limit info, err=%d", ret); } // scan all columns else if (OB_SUCCESS != (ret = add_scan_columns(*scan_param))) { TBSYS_LOG(WARN, "failed to add scan columsn, err=%d", ret); } // find the locations of the tablet which the startkey is in else if (OB_SUCCESS != (ret = get_location(meta_it, search_range.start_key_, search_range.border_flag_.inclusive_start(), *my_location))) { TBSYS_LOG(WARN, "failed to get location, err=%d", ret); } else if (OB_SUCCESS != (ret = scan_param->set_location_info(*my_location))) { TBSYS_LOG(WARN, "failed to add location, err=%d", ret); } else if (OB_SUCCESS != (ret = aggr_.scan(*scan_param, scanner))) { TBSYS_LOG(WARN, "scan error, err=%d range=%s", ret, to_cstring(search_range)); } else { scanner.reset_row_iter(); read_scanner_row_count_ = 0; TBSYS_LOG(DEBUG, "scan meta table succ, range=%s", to_cstring(search_range)); } } if (NULL != scan_param) { aggr_.scan_param_allocator_.free(scan_param); } if (NULL != my_location) { aggr_.scanner_allocator_.free(my_location); } return ret; }
TEST(TestObScanner, row_num) { ObScanner scanner; const int table_num = 3; const int row_num = 20; const int column_num = 5; buffer table_name[table_num]; buffer row_key[row_num]; buffer column_name[column_num]; for (int i = 0; i < table_num; i++) { table_name[i].assigne("T").appende(i); } for (int i = 0; i < row_num; i++) { row_key[i].assigne("R").appende(i); } for (int i = 0; i < column_num; i++) { column_name[i].assigne("C").appende(i); } for (int i = 0; i < table_num; i++) { for (int j = 0; j < row_num; j++) { for (int k = 0; k < column_num; k++) { ObCellInfo cell; cell.table_name_ = table_name[i].get_obstring(); cell.row_key_ = row_key[j].get_obstring(); cell.column_name_ = column_name[k].get_obstring(); cell.value_.set_int(1); scanner.add_cell(cell); } } } ASSERT_EQ(scanner.get_row_num(), table_num * row_num); ASSERT_EQ(scanner.get_cell_num(), table_num * row_num * column_num); scanner.set_whole_result_row_num(1000); buffer buf(1 << 21); scanner.serialize(buf.ptre(), buf.capacity(), buf.length()); printf("%ld\n", buf.length()); ObScanner dscanner; int64_t pos = 0; dscanner.deserialize(buf.ptre(), buf.length(), pos); EXPECT_EQ(dscanner.get_row_num(), table_num * row_num); EXPECT_EQ(dscanner.get_cell_num(), table_num * row_num * column_num); EXPECT_EQ(dscanner.get_whole_result_row_num(), 1000); scanner.clear(); scanner.set_mem_size_limit(256); int j = 0, k = 0; int err = OB_SUCCESS; for (; j < row_num; j++) { k = 0; for (; k < column_num; k++) { ObCellInfo cell; cell.table_name_ = table_name[0].get_obstring(); cell.row_key_ = row_key[j].get_obstring(); cell.column_name_ = column_name[k].get_obstring(); cell.value_.set_int(1); if (OB_SUCCESS != (err = scanner.add_cell(cell))) { break; } } if (OB_SUCCESS != err) { break; } } if (OB_SUCCESS != err) { scanner.rollback(); } EXPECT_EQ(scanner.get_row_num(), j); EXPECT_EQ(scanner.get_cell_num(), j * column_num); buf.length() = 0; scanner.serialize(buf.ptre(), buf.capacity(), buf.length()); printf("%ld\n", buf.length()); dscanner.reset(); pos = 0; dscanner.deserialize(buf.ptre(), buf.length(), pos); EXPECT_EQ(dscanner.get_row_num(), j); EXPECT_EQ(dscanner.get_cell_num(), j * column_num); }
int ObScannerLoader::get_decoded_scanner(ObScanner ¶m) { ObNewRange range; ObString column; ObString table_name; char value[4096]; if (config_loaded_) { TBSYS_LOG(INFO, "creating scanner"); param.reset(); range.reset(); range.table_id_ = table_id_; TBSYS_LOG(DEBUG, "range_start_=%s", to_cstring(range_start_)); TBSYS_LOG(DEBUG, "range_end_=%s", to_cstring(range_end_)); range.start_key_ = range_start_; if (range_start_inclusive_) range.border_flag_.set_inclusive_start(); else range.border_flag_.unset_inclusive_start(); if (range_start_min_) range.start_key_.set_min_row(); range.end_key_ = range_end_; if (range_end_inclusive_) range.border_flag_.set_inclusive_end(); else range.border_flag_.unset_inclusive_end(); if (range_end_max_) range.end_key_.set_max_row(); param.set_range(range); //param.set_is_req_fullfilled(is_fullfilled_, actual_fullfilled_item_num_); param.set_is_req_fullfilled(is_fullfilled_, fullfilled_item_num_); param.set_data_version(data_version_); FILE *fp = NULL; if ((fp = fopen(data_file_name_, "rb")) == NULL) { TBSYS_LOG(ERROR, "Fail to open %s", data_file_name_); } else { while(fgets(value, 4096, fp)) { /* only digit and string accepted. if not dig, then string */ int start = 0; int end = 0; int cnt = 0; char buf[4096]; int table_id = 0; int column_cnt = 0; ObRowkey row_key; while(true) { while(isspace(value[start])) { start++; } end = start; while(!isspace(value[end]) && value[end] != '\0') { end++; } if (start != end) { memset(buf, 0, 4096); cnt = end - start; strncpy(buf, &value[start], cnt); buf[cnt] = '\0'; if (column_cnt == 0) { table_id = atoi(buf); } else if(column_cnt == 1) { // TODO build rowkey //strcpy(row_key_buf, buf); //row_key.assign(row_key_buf, cnt); } else { build_cell_info(table_id, row_key, buf, param); } column_cnt++; start = end; } if (value[end] == '\0') { break; } } } fclose(fp); } } return OB_SUCCESS; }