void total_scan(const char *fname, PageArena<char> &allocer, MockClient &client, const char *version_range) { ObScanner scanner; ObScanParam scan_param; read_scan_param(fname, SCAN_PARAM_SECTION, allocer, scan_param); scan_param.set_version_range(str2range(version_range)); scan_param.set_is_read_consistency(false); int64_t total_fullfilled_num = 0; int64_t total_row_counter = 0; int64_t total_timeu = 0; while (true) { int64_t timeu = tbsys::CTimeUtil::getTime(); int err = client.ups_scan(scan_param, scanner, timeout); timeu = tbsys::CTimeUtil::getTime() - timeu; if (OB_SUCCESS != err) { fprintf(stdout, "[%s] err=%d\n", __FUNCTION__, err); break; } else { int64_t row_counter = 0; while (OB_SUCCESS == scanner.next_cell()) { ObCellInfo *ci = NULL; bool is_row_changed = false; scanner.get_cell(&ci, &is_row_changed); //fprintf(stdout, "%s\n", updateserver::print_cellinfo(ci, "CLI_SCAN")); if (is_row_changed) { row_counter++; } } bool is_fullfilled = false; int64_t fullfilled_num = 0; ObRowkey last_rk; scanner.get_last_row_key(last_rk); scanner.get_is_req_fullfilled(is_fullfilled, fullfilled_num); fprintf(stdout, "[SINGLE_SCAN] is_fullfilled=%s fullfilled_num=%ld row_counter=%ld timeu=%ld last_row_key=[%s]\n", STR_BOOL(is_fullfilled), fullfilled_num, row_counter, timeu, to_cstring(last_rk)); total_fullfilled_num += fullfilled_num; total_row_counter += row_counter; total_timeu += timeu; if (is_fullfilled) { break; } else { const_cast<ObNewRange*>(scan_param.get_range())->start_key_ = last_rk; const_cast<ObNewRange*>(scan_param.get_range())->border_flag_.unset_min_value(); const_cast<ObNewRange*>(scan_param.get_range())->border_flag_.unset_inclusive_start(); } } } fprintf(stdout, "[TOTAL_SCAN] total_fullfilled_num=%ld total_row_counter=%ld total_timeu=%ld\n", total_fullfilled_num, total_row_counter, total_timeu); }
void print_scanner(ObScanner &scanner) { int64_t row_counter = 0; while (OB_SUCCESS == scanner.next_cell()) { ObCellInfo *ci = NULL; bool is_row_changed = false; scanner.get_cell(&ci, &is_row_changed); fprintf(stdout, "%s\n", common::print_cellinfo(ci, "CLI_SCAN")); if (is_row_changed) { row_counter++; } } bool is_fullfilled = false; int64_t fullfilled_num = 0; ObRowkey last_rk; scanner.get_last_row_key(last_rk); scanner.get_is_req_fullfilled(is_fullfilled, fullfilled_num); fprintf(stdout, "is_fullfilled=%s fullfilled_num=%ld row_counter=%ld last_row_key=[%s]\n", STR_BOOL(is_fullfilled), fullfilled_num, row_counter, to_cstring(last_rk)); }
int param_get(const char *fname, MockClient &client) { int err = OB_SUCCESS; int fd = open(fname, O_RDONLY); if (-1 == fd) { fprintf(stdout, "open [%s] fail errno=%u\n", fname, errno); err = OB_ERROR; } else { struct stat st; fstat(fd, &st); char *buf = (char*)malloc(st.st_size); if (NULL == buf) { fprintf(stdout, "malloc buf fail size=%ld\n", st.st_size); err = OB_ERROR; } else { ssize_t __attribute__((unused)) ret = read(fd, buf, st.st_size); int64_t pos = 0; ObGetParam get_param; if (OB_SUCCESS != get_param.deserialize(buf, st.st_size, pos)) { fprintf(stdout, "deserialize get_param fail\n"); err = OB_ERROR; } else { get_param.set_is_read_consistency(false); ObScanner scanner; int err = client.ups_get(get_param, scanner, timeout); fprintf(stdout, "[%s] err=%d\n", __FUNCTION__, err); if (OB_SUCCESS == err) { int64_t row_counter = 0; while (OB_SUCCESS == scanner.next_cell()) { ObCellInfo *ci = NULL; bool is_row_changed = false; scanner.get_cell(&ci, &is_row_changed); fprintf(stdout, "%s\n", common::print_cellinfo(ci, "CLI_GET")); if (is_row_changed) { row_counter++; } } bool is_fullfilled = false; int64_t fullfilled_num = 0; ObRowkey last_rk; scanner.get_last_row_key(last_rk); scanner.get_is_req_fullfilled(is_fullfilled, fullfilled_num); fprintf(stdout, "is_fullfilled=%s fullfilled_num=%ld row_counter=%ld last_row_key=[%s]\n", STR_BOOL(is_fullfilled), fullfilled_num, row_counter, to_cstring(last_rk)); } } free(buf); } close(fd); } return err; }
void scan_check_all(MutatorBuilder &mb, MockClient &client, const int64_t table_start_version, const bool using_id, const bool check) { for (int64_t i = 0; i < mb.get_schema_num(); ++i) { ObScanParam scan_param; prepare_scan_param(scan_param, mb.get_schema(i), table_start_version, using_id); int ret = OB_SUCCESS; bool is_fullfilled = false; int64_t fullfilled_item_num = 0; RowChecker rc; int64_t row_counter = 0; ObScanner scanner; while (!is_fullfilled && OB_SUCCESS == ret) { int64_t timeu = tbsys::CTimeUtil::getTime(); ret = client.ups_scan(scan_param, scanner, TIMEOUT_MS); TBSYS_LOG(INFO, "scan ret=%d timeu=%ld", ret, tbsys::CTimeUtil::getTime() - timeu); if (check && OB_SUCCESS == ret) { while (OB_SUCCESS == scanner.next_cell()) { ObCellInfo *ci = NULL; bool is_row_changed = false; if (OB_SUCCESS == scanner.get_cell(&ci, &is_row_changed)) { if (!using_id) { trans_name2id(*ci, mb.get_schema(i)); } if (is_row_changed && 0 != rc.cell_num()) { std::string row_key_str(rc.get_cur_rowkey().ptr(), 0, rc.get_cur_rowkey().length()); //fprintf(stderr, "[%.*s] ", rc.get_cur_rowkey().length(), rc.get_cur_rowkey().ptr()); bool get_row_bret = get_check_row(mb.get_schema(i), rc.get_cur_rowkey(), mb.get_cellinfo_builder(i), client, table_start_version, using_id); //fprintf(stderr, "[get_row check_ret=%d] ", bret); bool cell_info_bret = rc.check_row(mb.get_cellinfo_builder(i), mb.get_schema(i)); //fprintf(stderr, "[cell_info check_ret=%d]\n", bret); TBSYS_LOG(INFO, "[%s] [get_row check_ret=%d] [cell_info check_ret=%d]", row_key_str.c_str(), get_row_bret, cell_info_bret); } if (is_row_changed && 0 != rc.rowkey_num() && rc.is_prefix_changed(ci->row_key_)) { std::string row_key_str(rc.get_last_rowkey().ptr(), 0, rc.get_last_rowkey().length()); //fprintf(stderr, "[%.*s] ", rc.get_last_rowkey().length(), rc.get_last_rowkey().ptr()); bool bret = rc.check_rowkey(mb.get_rowkey_builder(i)); //fprintf(stderr, "row_key check_ret=%d\n", bret); TBSYS_LOG(INFO, "[%s] [row_key check_ret=%d]", row_key_str.c_str(), bret); } rc.add_cell(ci); if (is_row_changed) { rc.add_rowkey(ci->row_key_); row_counter++; } } } if (0 != rc.cell_num()) { std::string row_key_str(rc.get_cur_rowkey().ptr(), 0, rc.get_cur_rowkey().length()); //fprintf(stderr, "[%.*s] ", rc.get_cur_rowkey().length(), rc.get_cur_rowkey().ptr()); bool get_row_bret = get_check_row(mb.get_schema(i), rc.get_cur_rowkey(), mb.get_cellinfo_builder(i), client, table_start_version, using_id); //fprintf(stderr, "[get_row check_ret=%d] ", bret); bool cell_info_bret = rc.check_row(mb.get_cellinfo_builder(i), mb.get_schema(i)); //fprintf(stderr, "[cell_info check_ret=%d]\n", bret); TBSYS_LOG(INFO, "[%s] [get_row check_ret=%d] [cell_info check_ret=%d]", row_key_str.c_str(), get_row_bret, cell_info_bret); } } scanner.get_is_req_fullfilled(is_fullfilled, fullfilled_item_num); ObRange *range = const_cast<ObRange*>(scan_param.get_range()); scanner.get_last_row_key(range->start_key_); range->border_flag_.unset_min_value(); range->border_flag_.unset_inclusive_start(); } if (check && 0 != rc.rowkey_num()) { std::string row_key_str(rc.get_last_rowkey().ptr(), 0, rc.get_last_rowkey().length()); //fprintf(stderr, "[%.*s] ", rc.get_last_rowkey().length(), rc.get_last_rowkey().ptr()); bool bret = rc.check_rowkey(mb.get_rowkey_builder(i)); //fprintf(stderr, "row_key check_ret=%d\n", bret); TBSYS_LOG(INFO, "[%s] [row_key check_ret=%d]", row_key_str.c_str(), bret); } TBSYS_LOG(INFO, "table_id=%lu row_counter=%ld", mb.get_schema(i).get_table_id(), row_counter); } }
int ObMergerSortedOperator::add_sharding_result(ObScanner & sharding_res, const ObNewRange &query_range, bool &is_finish) { int err = OB_SUCCESS; if ((OB_SUCCESS == err) && (NULL == scan_param_)) { TBSYS_LOG(WARN,"operator was not initialized yet [scan_param_:%p]", scan_param_); err = OB_INVALID_ARGUMENT; } if ((OB_SUCCESS == err) && (sharding_result_count_ >= MAX_SHARDING_RESULT_COUNT)) { TBSYS_LOG(WARN,"array is full [MAX_SHARDING_RESULT_COUNT:%ld,sharding_result_count_:%ld]", MAX_SHARDING_RESULT_COUNT, sharding_result_count_); err = OB_ARRAY_OUT_OF_RANGE; } /// @todo (wushi [email protected]) check fullfill item number and size property bool is_fullfilled = false; int64_t fullfilled_item_num = 0; ObRowkey last_row_key; ObNewRange cs_tablet_range; if ((OB_SUCCESS == err) && (OB_SUCCESS != (err = sharding_res.get_is_req_fullfilled(is_fullfilled,fullfilled_item_num)))) { TBSYS_LOG(WARN,"fail to get fullfilled info from sharding result [err:%d]", err); } if (OB_SUCCESS == err) { if (OB_SUCCESS == (err = sharding_res.get_last_row_key(last_row_key))) { } else if (OB_ENTRY_NOT_EXIST == err) { err = OB_SUCCESS; } else { TBSYS_LOG(WARN,"fail to get last rowkey from sharding result [err:%d]", err); } } if (OB_SUCCESS == err) { if (is_fullfilled) { if ((OB_SUCCESS == err) && (OB_SUCCESS != (err = sharding_res.get_range(cs_tablet_range)))) { TBSYS_LOG(WARN,"fail to get tablet range from sharding result [err:%d]", err); } if (OB_SUCCESS == err) { if (ScanFlag::FORWARD == scan_param_->get_scan_direction()) { last_row_key = cs_tablet_range.end_key_; } else { last_row_key = cs_tablet_range.start_key_; } } } if (OB_SUCCESS == err) { fullfilled_item_num = sharding_res.get_row_num(); sharding_result_arr_[sharding_result_count_].init(sharding_res,query_range,*scan_param_, last_row_key, fullfilled_item_num); sharding_result_count_ ++; } } is_finish = false; if (OB_SUCCESS == err) { sort(is_finish, &sharding_res); } TBSYS_LOG(DEBUG, "add sharding result. is_finish=%d, err=%d", is_finish, err); return err; }
int get_next_param( const ObScanParam &org_scan_param, const ObScanner &prev_scan_result, ObScanParam *scan_param, ObMemBuf &range_buffer) { int err = OB_SUCCESS; const ObReadParam &org_read_param = org_scan_param; ObReadParam *read_param = scan_param; ObNewRange tablet_range; const ObNewRange *org_scan_range = NULL; ObNewRange cur_range; if(NULL != scan_param) { scan_param->reset(); *read_param = org_read_param; } bool request_fullfilled = false; int64_t fullfilled_row_num = 0; err = prev_scan_result.get_is_req_fullfilled(request_fullfilled,fullfilled_row_num); if (OB_SUCCESS == err) { org_scan_range = org_scan_param.get_range(); if (NULL == org_scan_range) { TBSYS_LOG(WARN, "unexpected error, can't get range from org scan param"); err = OB_INVALID_ARGUMENT; } } ObRowkey last_row_key; if (OB_SUCCESS == err) { if (request_fullfilled) { err = prev_scan_result.get_range(tablet_range); if (OB_SUCCESS == err) { if (true == is_finish_scan(org_scan_param, tablet_range)) { err = OB_ITER_END; } else if (ScanFlag::FORWARD == org_scan_param.get_scan_direction()) { last_row_key = tablet_range.end_key_; } else { last_row_key = tablet_range.start_key_; } } } else { err = prev_scan_result.get_last_row_key(last_row_key); if (OB_ENTRY_NOT_EXIST == err) { /// first time, remain unchanged } else if (OB_SUCCESS != err) { TBSYS_LOG(WARN, "unexpected error, scanner should contain at least one cell"); } } } if (OB_SUCCESS == err && NULL != scan_param) { cur_range =*org_scan_range; // forward if (ScanFlag::FORWARD == org_scan_param.get_scan_direction()) { cur_range.start_key_ = last_row_key; cur_range.border_flag_.unset_inclusive_start(); } else { cur_range.end_key_ = last_row_key; if (request_fullfilled) { // tablet range start key cur_range.border_flag_.set_inclusive_end(); } else { // the smallest rowkey cur_range.border_flag_.unset_inclusive_end(); } } } // first time remain unchanged if (OB_ENTRY_NOT_EXIST == err) { cur_range =*org_scan_range; err = OB_SUCCESS; } if (OB_SUCCESS == err && NULL != scan_param) { err = allocate_range_buffer(cur_range, range_buffer); } if (OB_SUCCESS == err && NULL != scan_param) { scan_param->set(org_scan_param.get_table_id(), org_scan_param.get_table_name(), cur_range); for (int32_t cell_idx = 0; cell_idx < org_scan_param.get_column_id_size() && OB_SUCCESS == err; cell_idx ++) { err = scan_param->add_column(org_scan_param.get_column_id()[cell_idx]); } if(OB_SUCCESS == err) { scan_param->set_scan_size(GET_SCAN_SIZE(org_scan_param.get_scan_size())); scan_param->set_scan_flag(org_scan_param.get_scan_flag()); } } return err; }