int ObMergerPrefetchData::modify_prefetch_param(ObGetParam & param) { ObVersionRange range; int ret = version_.get_version(range.end_version_.version_); if ((ret != OB_SUCCESS) || (range.end_version_ <= 0)) { TBSYS_LOG(WARN, "get last version failed:version[%ld], ret[%d]", (int64_t)range.end_version_, ret); } else { cur_version_ = range.end_version_; range.border_flag_.set_min_value(); range.border_flag_.set_inclusive_end(); // maybe not need someday param.set_is_result_cached(true); param.set_is_read_consistency(true); param.set_version_range(range); } return ret; }
// maybe not find a merge server int TaskFactory::get_table_tablet(const char * table_name, const uint64_t table_id, uint64_t & count) { int ret = OB_SUCCESS; ObScanParam scan_param; uint64_t max_len = 0; if (NULL == table_name) { TBSYS_LOG(ERROR, "check table name failed:table[%s], id[%lu]", table_name, table_id); ret = OB_ERROR; } else { ret = init_scan_param(table_name, table_id, max_len, scan_param); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init scan param failed:table[%s], ret[%d]", table_name, ret); } } if (OB_SUCCESS == ret) { // for the first table tablet ObRowkey row_key; ObObj temp_buffer[common::OB_MAX_ROWKEY_COLUMN_NUMBER]; row_key.assign(temp_buffer, common::OB_MAX_ROWKEY_COLUMN_NUMBER); ObGetParam param; ObScanner scanner; ObServer server; ObRowkey start_key; ObRowkey end_key; ObCellInfo * cell = NULL; ObScannerIterator iter; bool row_change = false; ObString name; name.assign(const_cast<char*>(table_name), (int32_t)strlen(table_name)); ObCellInfo temp_cell; temp_cell.table_id_ = table_id; temp_cell.column_id_ = 0; common::ModuleArena allocator; while ((OB_SUCCESS == ret) && (!row_key.is_max_row())) { param.reset(); param.set_is_read_consistency(false); temp_cell.row_key_ = row_key; ret = param.add_cell(temp_cell); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "add cell failed:ret[%d]", ret); break; } ret = rpc_->get(root_server_, timeout_, param, scanner); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get root table for tablet failed:table[%lu], ret[%d]", table_id, ret); break; } else { // skip the first row iter = scanner.begin(); ++iter; while ((iter != scanner.end()) && (OB_SUCCESS == (ret = iter.get_cell(&cell, &row_change))) && !row_change) { if (NULL == cell) { TBSYS_LOG(ERROR, "%s", "check cell failed"); ret = OB_INNER_STAT_ERROR; break; } ret = cell->row_key_.deep_copy(start_key, allocator); if (ret != OB_SUCCESS) { TBSYS_LOG(WARN, "deep copy the rowkey failed:ret[%d]", ret); } ++iter; } } // scanner iter end if (ret == OB_SUCCESS) { int64_t ip = 0; int64_t port = 0; int64_t version = 0; TabletLocation list; for (++iter; iter != scanner.end(); ++iter) { ret = iter.get_cell(&cell, &row_change); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret); break; } else if (row_change) // && (iter != last_iter)) { TaskInfo task; task.set_table_id(table_id); task.set_table_name(table_name); ret = init_new_task(name, start_key, end_key, scan_param, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret); break; } else { ret = insert_new_task(list, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret); break; } ++count; } list.clear(); start_key = end_key; } else { cell->row_key_.deep_copy(end_key, allocator); if ((cell->column_name_.compare("1_ms_port") == 0) || (cell->column_name_.compare("2_ms_port") == 0) || (cell->column_name_.compare("3_ms_port") == 0)) { ret = cell->value_.get_int(port); } else if ((cell->column_name_.compare("1_ipv4") == 0) || (cell->column_name_.compare("2_ipv4") == 0) || (cell->column_name_.compare("3_ipv4") == 0)) { ret = cell->value_.get_int(ip); } else if ((cell->column_name_.compare("1_tablet_version") == 0) || (cell->column_name_.compare("2_tablet_version") == 0) || (cell->column_name_.compare("3_tablet_version") == 0)) { ret = cell->value_.get_int(version); if (OB_SUCCESS == ret) { if (0 == port || port == 0) { TBSYS_LOG(WARN, "%s", "check port or ip failed"); } else { server.set_ipv4_addr(int32_t(ip), int32_t(port)); ObTabletLocation addr(version, server); if (OB_SUCCESS != (ret = list.add(addr))) { TBSYS_LOG(ERROR, "add addr failed:server[%d], port[%d], ret[%d]", server.get_ipv4(), server.get_port(), ret); } else { TBSYS_LOG(DEBUG, "add addr succ:server[%s], version:%ld", server.to_cstring(), version); } } ip = port = version = 0; } } if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret); break; } } } // for the last row if ((OB_SUCCESS == ret) && (start_key != end_key)) { TaskInfo task; task.set_table_id(table_id); task.set_table_name(table_name); ret = init_new_task(name, start_key, end_key, scan_param, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret); } else { ret = insert_new_task(list, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret); } ++count; } if (OB_SUCCESS == ret) { // copy all the end key objs to row_key for (int64_t i = 0; i < end_key.length(); ++i) { const_cast<ObObj *> (row_key.ptr())[i] = end_key.ptr()[i]; row_key.assign(const_cast<ObObj *> (row_key.ptr()), end_key.length()); } // modify last row key for next get root table if (!end_key.is_max_row()) { const_cast<ObObj *>(row_key.ptr())[end_key.length()].set_max_value(); row_key.assign(const_cast<ObObj *> (row_key.ptr()), end_key.length() + 1); } } } list.clear(); } } } return ret; }
int MutatorBuilder::build_get_param(ObGetParam &get_param, PageArena<char> &allocer, const int64_t table_start_version, const bool using_id, const int64_t row_num, const int64_t cell_num_per_row) { int ret = OB_SUCCESS; get_param.reset(); UNUSED(table_start_version); static int64_t row_info_array[OB_MAX_TABLE_NUMBER]; static int64_t suffix_num_array[OB_MAX_TABLE_NUMBER]; static int64_t counter = 0; static const int64_t ROW_INFO_CACHE_FLUSH_PERIOD = 100; //static const int64_t ROW_INFO_CACHE_FLUSH_PERIOD = 1000000000; if (0 == (counter++ % ROW_INFO_CACHE_FLUSH_PERIOD)) { memset(row_info_array, 0, sizeof(row_info_array)); memset(suffix_num_array, 0, sizeof(suffix_num_array)); } for (int64_t i = 0; i < row_num; i++) { int64_t rand = 0; lrand48_r(&table_rand_seed_, &rand); int64_t table_pos = range_rand(0, table_num_ - 1, rand); const ObSchema &cur_schema = schema_mgr_.begin()[table_pos]; if (0 == row_info_array[table_pos] && 0 == (row_info_array[table_pos] = get_cur_rowkey_info_(cur_schema, using_id, rb_array_[table_pos]->get_prefix_start()))) { assert(OB_SUCCESS == ret); ret = OB_ERROR; break; } if (0 == suffix_num_array[table_pos] && 0 == (suffix_num_array[table_pos] = get_suffix_num_(cur_schema, using_id, rb_array_[table_pos]->get_prefix_start()))) { assert(OB_SUCCESS == ret); ret = OB_ERROR; break; } ObCellInfo ci; ci.row_key_ = rb_array_[table_pos]->get_random_rowkey(row_info_array[table_pos], suffix_num_array[table_pos], allocer); if (using_id) { ci.table_id_ = cur_schema.get_table_id(); } else { ci.table_name_.assign_ptr(const_cast<char*>(cur_schema.get_table_name()), static_cast<int32_t>(strlen(cur_schema.get_table_name()))); } if (using_id) { ci.column_id_ = SEED_COLUMN_ID; ci.column_name_.assign_ptr(NULL, 0); } else { ci.column_id_ = OB_INVALID_ID; ci.column_name_.assign_ptr((char*)SEED_COLUMN_NAME, static_cast<int32_t>(strlen(SEED_COLUMN_NAME))); } get_param.add_cell(ci); if (using_id) { ci.column_id_ = CELL_NUM_COLUMN_ID; ci.column_name_.assign_ptr(NULL, 0); } else { ci.column_id_ = OB_INVALID_ID; ci.column_name_.assign_ptr((char*)CELL_NUM_COLUMN_NAME, static_cast<int32_t>(strlen(CELL_NUM_COLUMN_NAME))); } get_param.add_cell(ci); lrand48_r(&table_rand_seed_, &rand); int64_t cell_num = range_rand(1, cell_num_per_row, rand); int64_t column_num = cur_schema.column_end() - cur_schema.column_begin(); for (int64_t j = 0; j < cell_num; j++) { lrand48_r(&table_rand_seed_, &rand); int64_t column_pos = range_rand(0, column_num - 1, rand); //int64_t column_pos = 0; const ObColumnSchema &column_schema = cur_schema.column_begin()[column_pos]; if (using_id) { ci.column_id_ = column_schema.get_id(); ci.column_name_.assign_ptr(NULL, 0); } else { ci.column_id_ = OB_INVALID_ID; ci.column_name_.assign_ptr(const_cast<char*>(column_schema.get_name()), static_cast<int32_t>(strlen(column_schema.get_name()))); } get_param.add_cell(ci); } } ObVersionRange version_range; version_range.start_version_ = table_start_version_; version_range.border_flag_.set_max_value(); version_range.border_flag_.set_inclusive_start(); get_param.set_version_range(version_range); char* is_read_consistency_cfg = getenv("is_read_consistency"); bool is_read_consistency = (NULL != is_read_consistency_cfg && 0 == strcmp(is_read_consistency_cfg, "true")); get_param.set_is_read_consistency(is_read_consistency); assert(get_param.get_cell_size() > 0); return ret; }
// maybe not find a merge server int TaskFactory::get_table_tablet(const char * table_name, const uint64_t table_id, uint64_t & count) { int ret = OB_SUCCESS; ObScanParam scan_param; uint64_t max_len = 0; if (NULL == table_name) { TBSYS_LOG(ERROR, "check table name failed:table[%s], id[%lu]", table_name, table_id); ret = OB_ERROR; } else { ret = init_scan_param(table_name, table_id, max_len, scan_param); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init scan param failed:table[%s], ret[%d]", table_name, ret); } } if (OB_SUCCESS == ret) { TaskInfo task; task.set_table_id(table_id); task.set_table_name(table_name); // for the first table tablet ObString row_key; char temp_buffer[1]; memset(temp_buffer, 0, sizeof(temp_buffer)); row_key.assign(temp_buffer, sizeof(temp_buffer)); ObGetParam param; ObScanner scanner; ObServer server; ObString start_key; ObString end_key; ObCellInfo * cell = NULL; ObScannerIterator iter; bool row_change = false; ObString name; name.assign(const_cast<char*>(table_name), strlen(table_name)); ObCellInfo temp_cell; temp_cell.table_id_ = table_id; temp_cell.column_id_ = 0; const uint64_t MAX_LEN = 1024; char last_tablet_rowkey[MAX_LEN] = ""; const int32_t MAX_SERVER_ADDR_SIZE = 128; char server_addr[MAX_SERVER_ADDR_SIZE]; while ((OB_SUCCESS == ret) && (!is_max_rowkey(max_len, row_key.ptr(), row_key.length() - 1))) { param.reset(); param.set_is_read_consistency(false); temp_cell.row_key_ = row_key; ret = param.add_cell(temp_cell); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "add cell failed:ret[%d]", ret); break; } ret = rpc_->get(root_server_, timeout_, param, scanner); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get root table for tablet failed:table[%lu], ret[%d]", table_id, ret); break; } else { // skip the first row iter = scanner.begin(); ++iter; while ((iter != scanner.end()) && (OB_SUCCESS == (ret = iter.get_cell(&cell, &row_change))) && !row_change) { if (NULL == cell) { TBSYS_LOG(ERROR, "%s", "check cell failed"); ret = OB_INNER_STAT_ERROR; break; } start_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); ++iter; } } if (ret == OB_SUCCESS) { int64_t ip = 0; int64_t port = 0; int64_t version = 0; TabletLocation list; for (++iter; iter != scanner.end(); ++iter) { ret = iter.get_cell(&cell, &row_change); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret); break; } else if (row_change) // && (iter != last_iter)) { ret = init_new_task(name, start_key, end_key, scan_param, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret); break; } else { ret = insert_new_task(list, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret); break; } ++count; } list.clear(); start_key = end_key; end_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); } else { end_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); if ((cell->column_name_.compare("1_ms_port") == 0) || (cell->column_name_.compare("2_ms_port") == 0) || (cell->column_name_.compare("3_ms_port") == 0)) { ret = cell->value_.get_int(port); } else if ((cell->column_name_.compare("1_ipv4") == 0) || (cell->column_name_.compare("2_ipv4") == 0) || (cell->column_name_.compare("3_ipv4") == 0)) { ret = cell->value_.get_int(ip); } else if ((cell->column_name_.compare("1_tablet_version") == 0) || (cell->column_name_.compare("2_tablet_version") == 0) || (cell->column_name_.compare("3_tablet_version") == 0)) { ret = cell->value_.get_int(version); if (OB_SUCCESS == ret) { if (0 == port || port == 0) { TBSYS_LOG(WARN, "%s", "check port or ip failed"); } else { server.set_ipv4_addr(ip, port); ObTabletLocation addr(version, server); if (OB_SUCCESS != (ret = list.add(addr))) { TBSYS_LOG(ERROR, "add addr failed:server[%d], port[%d], ret[%d]", server.get_ipv4(), server.get_port(), ret); } else { server.to_string(server_addr, MAX_SERVER_ADDR_SIZE); TBSYS_LOG(DEBUG, "add addr succ:server[%s], version:%ld", server_addr, version); } } ip = port = version = 0; } } if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret); break; } } } // for the last row if ((OB_SUCCESS == ret) && (start_key != end_key)) { ret = init_new_task(name, start_key, end_key, scan_param, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret); } else { ret = insert_new_task(list, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret); } ++count; } if (OB_SUCCESS == ret) { // modify last row key for next get root table uint64_t len = end_key.length(); if (MAX_LEN > len + 1) { memcpy(last_tablet_rowkey, end_key.ptr(), len); last_tablet_rowkey[len] = 0; row_key.assign(last_tablet_rowkey, len + 1); } else { TBSYS_LOG(ERROR, "check end key failed:len[%lu]", len); } } } list.clear(); } } } return ret; }