int GetNotExistRows::form_get_param(ObGetParam &get_param, const uint32_t min_key_include, const uint32_t max_key_include, void *&arg) { UNUSED(min_key_include); int err = OB_SUCCESS; arg = NULL; const int32_t get_cell_count = 32; const uint32_t random_selector = 1024*16; uint32_t start_key_val = static_cast<uint32_t>(max_key_include + 1 + random()%random_selector); uint32_t key; get_param.reset(true); ObCellInfo cell; cell.table_name_ = msolap::target_table_name; cell.column_name_ = ObGetParam::OB_GET_ALL_COLUMN_NAME; for (int32_t i = 0; (OB_SUCCESS == err) && (i < get_cell_count); i++, start_key_val ++) { key = htonl(start_key_val); cell.row_key_.assign((char*)&key,sizeof(key)); if (OB_SUCCESS != (err = get_param.add_cell(cell))) { TBSYS_LOG(WARN,"fail to add cell to ObGetParam [err:%d]", err); } } if (OB_SUCCESS == err) { arg = &get_param; } return err; }
int MultiGetP::form_get_param(ObGetParam &get_param, const uint32_t min_key_include, const uint32_t max_key_include, void *&arg) { int err = OB_SUCCESS; arg = NULL; get_param.reset(true); char c_name; ObCellInfo cell; cell.table_name_ = msolap::target_table_name; cell.column_name_.assign(&c_name, sizeof(c_name)); int64_t cell_count = get_cell_count; if (cell_count > max_key_include - min_key_include + 1) { cell_count = max_key_include - min_key_include + 1; } int64_t random_selector = (max_key_include - min_key_include + 1)/cell_count; /// bool get_last_cell = false; uint32_t start_key = min_key_include; uint32_t start_key_val = 0; char strkey[32]; for (int64_t cell_idx = 0; (OB_SUCCESS == err) && (start_key <= max_key_include) && (cell_idx < cell_count); cell_idx ++,start_key = static_cast<int32_t>(start_key + random_selector)) { uint32_t cur_key = static_cast<uint32_t>(start_key + random() % random_selector); c_name = msolap::olap_get_column_name(msolap::min_column_id + random()%(msolap::max_column_id - msolap::min_column_id + 1)); start_key_val = htonl(cur_key); snprintf(strkey, 32, "%d", start_key_val); cell.row_key_ = make_rowkey(strkey, &allocator_); assert(cell.column_name_.ptr()[0] <= 'z' && cell.column_name_.ptr()[0] >= 'a'); if (OB_SUCCESS != (err = get_param.add_cell(cell))) { TBSYS_LOG(WARN,"fail to add cell to ObGetParam [err:%d,c_name:%c]", err, c_name); } } if (OB_SUCCESS == err) { arg = &get_param; } return err; }
int ObSKeyInfoKey::build_get_param(ObGetParam& get_param) { int ret = OB_SUCCESS; ObCellInfo cell_info; ObRowkey rowkey; ObObj rowkey_obj; ObVersionRange ver_range; int64_t pos = 0; if (NULL == buffer_) { buffer_ = reinterpret_cast<char*>(ob_malloc(BUF_SIZE, ObModIds::OB_SKEY_INFO_KEY)); } ret = serialization::encode_i64(buffer_, BUF_SIZE, pos, skey_version_); if (OB_SUCCESS != ret) { TBSYS_LOG(ERROR, "encode failed pkey_version=%ld", skey_version_); } else { rowkey_obj.set_varchar(ObString(0, static_cast<int32_t>(pos), buffer_)); rowkey.assign(&rowkey_obj, 1); get_param.reset(); get_param.set_is_result_cached(true); cell_info.table_id_ = SKEY_TABLE_ID; cell_info.row_key_ = rowkey; cell_info.column_id_ = SKEY_COL_ID; ret = get_param.add_cell(cell_info); if (OB_SUCCESS != ret) { TBSYS_LOG(ERROR, "Get param add cell failed table_id[%lu]", cell_info.table_id_); } else { ver_range.start_version_ = 0; ver_range.end_version_ = INT64_MAX - 1; ver_range.border_flag_.set_inclusive_start(); ver_range.border_flag_.set_inclusive_end(); get_param.set_version_range(ver_range); } } return ret; }
int GetSingleRowAllColumn::form_get_param(ObGetParam &get_param, const uint32_t min_key_include, const uint32_t max_key_include, void *&arg) { int err = OB_SUCCESS; arg = NULL; uint32_t start_key_val = htonl(static_cast<int32_t>(random()%(max_key_include - min_key_include + 1) + min_key_include)); get_param.reset(true); ObCellInfo cell; cell.table_name_ = msolap::target_table_name; cell.column_name_ = ObGetParam::OB_GET_ALL_COLUMN_NAME; cell.row_key_.assign((char*)&start_key_val,sizeof(start_key_val)); if (OB_SUCCESS != (err = get_param.add_cell(cell))) { TBSYS_LOG(WARN,"fail to add cell to ObGetParam [err:%d]", err); } if (OB_SUCCESS == err) { arg = &get_param; } return err; }
// maybe not find a merge server int TaskFactory::get_table_tablet(const char * table_name, const uint64_t table_id, uint64_t & count) { int ret = OB_SUCCESS; ObScanParam scan_param; uint64_t max_len = 0; if (NULL == table_name) { TBSYS_LOG(ERROR, "check table name failed:table[%s], id[%lu]", table_name, table_id); ret = OB_ERROR; } else { ret = init_scan_param(table_name, table_id, max_len, scan_param); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init scan param failed:table[%s], ret[%d]", table_name, ret); } } if (OB_SUCCESS == ret) { // for the first table tablet ObRowkey row_key; ObObj temp_buffer[common::OB_MAX_ROWKEY_COLUMN_NUMBER]; row_key.assign(temp_buffer, common::OB_MAX_ROWKEY_COLUMN_NUMBER); ObGetParam param; ObScanner scanner; ObServer server; ObRowkey start_key; ObRowkey end_key; ObCellInfo * cell = NULL; ObScannerIterator iter; bool row_change = false; ObString name; name.assign(const_cast<char*>(table_name), (int32_t)strlen(table_name)); ObCellInfo temp_cell; temp_cell.table_id_ = table_id; temp_cell.column_id_ = 0; common::ModuleArena allocator; while ((OB_SUCCESS == ret) && (!row_key.is_max_row())) { param.reset(); param.set_is_read_consistency(false); temp_cell.row_key_ = row_key; ret = param.add_cell(temp_cell); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "add cell failed:ret[%d]", ret); break; } ret = rpc_->get(root_server_, timeout_, param, scanner); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get root table for tablet failed:table[%lu], ret[%d]", table_id, ret); break; } else { // skip the first row iter = scanner.begin(); ++iter; while ((iter != scanner.end()) && (OB_SUCCESS == (ret = iter.get_cell(&cell, &row_change))) && !row_change) { if (NULL == cell) { TBSYS_LOG(ERROR, "%s", "check cell failed"); ret = OB_INNER_STAT_ERROR; break; } ret = cell->row_key_.deep_copy(start_key, allocator); if (ret != OB_SUCCESS) { TBSYS_LOG(WARN, "deep copy the rowkey failed:ret[%d]", ret); } ++iter; } } // scanner iter end if (ret == OB_SUCCESS) { int64_t ip = 0; int64_t port = 0; int64_t version = 0; TabletLocation list; for (++iter; iter != scanner.end(); ++iter) { ret = iter.get_cell(&cell, &row_change); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret); break; } else if (row_change) // && (iter != last_iter)) { TaskInfo task; task.set_table_id(table_id); task.set_table_name(table_name); ret = init_new_task(name, start_key, end_key, scan_param, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret); break; } else { ret = insert_new_task(list, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret); break; } ++count; } list.clear(); start_key = end_key; } else { cell->row_key_.deep_copy(end_key, allocator); if ((cell->column_name_.compare("1_ms_port") == 0) || (cell->column_name_.compare("2_ms_port") == 0) || (cell->column_name_.compare("3_ms_port") == 0)) { ret = cell->value_.get_int(port); } else if ((cell->column_name_.compare("1_ipv4") == 0) || (cell->column_name_.compare("2_ipv4") == 0) || (cell->column_name_.compare("3_ipv4") == 0)) { ret = cell->value_.get_int(ip); } else if ((cell->column_name_.compare("1_tablet_version") == 0) || (cell->column_name_.compare("2_tablet_version") == 0) || (cell->column_name_.compare("3_tablet_version") == 0)) { ret = cell->value_.get_int(version); if (OB_SUCCESS == ret) { if (0 == port || port == 0) { TBSYS_LOG(WARN, "%s", "check port or ip failed"); } else { server.set_ipv4_addr(int32_t(ip), int32_t(port)); ObTabletLocation addr(version, server); if (OB_SUCCESS != (ret = list.add(addr))) { TBSYS_LOG(ERROR, "add addr failed:server[%d], port[%d], ret[%d]", server.get_ipv4(), server.get_port(), ret); } else { TBSYS_LOG(DEBUG, "add addr succ:server[%s], version:%ld", server.to_cstring(), version); } } ip = port = version = 0; } } if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret); break; } } } // for the last row if ((OB_SUCCESS == ret) && (start_key != end_key)) { TaskInfo task; task.set_table_id(table_id); task.set_table_name(table_name); ret = init_new_task(name, start_key, end_key, scan_param, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret); } else { ret = insert_new_task(list, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret); } ++count; } if (OB_SUCCESS == ret) { // copy all the end key objs to row_key for (int64_t i = 0; i < end_key.length(); ++i) { const_cast<ObObj *> (row_key.ptr())[i] = end_key.ptr()[i]; row_key.assign(const_cast<ObObj *> (row_key.ptr()), end_key.length()); } // modify last row key for next get root table if (!end_key.is_max_row()) { const_cast<ObObj *>(row_key.ptr())[end_key.length()].set_max_value(); row_key.assign(const_cast<ObObj *> (row_key.ptr()), end_key.length() + 1); } } } list.clear(); } } } return ret; }
int MutatorBuilder::build_get_param(ObGetParam &get_param, PageArena<char> &allocer, const int64_t table_start_version, const bool using_id, const int64_t row_num, const int64_t cell_num_per_row) { int ret = OB_SUCCESS; get_param.reset(); UNUSED(table_start_version); static int64_t row_info_array[OB_MAX_TABLE_NUMBER]; static int64_t suffix_num_array[OB_MAX_TABLE_NUMBER]; static int64_t counter = 0; static const int64_t ROW_INFO_CACHE_FLUSH_PERIOD = 100; //static const int64_t ROW_INFO_CACHE_FLUSH_PERIOD = 1000000000; if (0 == (counter++ % ROW_INFO_CACHE_FLUSH_PERIOD)) { memset(row_info_array, 0, sizeof(row_info_array)); memset(suffix_num_array, 0, sizeof(suffix_num_array)); } for (int64_t i = 0; i < row_num; i++) { int64_t rand = 0; lrand48_r(&table_rand_seed_, &rand); int64_t table_pos = range_rand(0, table_num_ - 1, rand); const ObSchema &cur_schema = schema_mgr_.begin()[table_pos]; if (0 == row_info_array[table_pos] && 0 == (row_info_array[table_pos] = get_cur_rowkey_info_(cur_schema, using_id, rb_array_[table_pos]->get_prefix_start()))) { assert(OB_SUCCESS == ret); ret = OB_ERROR; break; } if (0 == suffix_num_array[table_pos] && 0 == (suffix_num_array[table_pos] = get_suffix_num_(cur_schema, using_id, rb_array_[table_pos]->get_prefix_start()))) { assert(OB_SUCCESS == ret); ret = OB_ERROR; break; } ObCellInfo ci; ci.row_key_ = rb_array_[table_pos]->get_random_rowkey(row_info_array[table_pos], suffix_num_array[table_pos], allocer); if (using_id) { ci.table_id_ = cur_schema.get_table_id(); } else { ci.table_name_.assign_ptr(const_cast<char*>(cur_schema.get_table_name()), static_cast<int32_t>(strlen(cur_schema.get_table_name()))); } if (using_id) { ci.column_id_ = SEED_COLUMN_ID; ci.column_name_.assign_ptr(NULL, 0); } else { ci.column_id_ = OB_INVALID_ID; ci.column_name_.assign_ptr((char*)SEED_COLUMN_NAME, static_cast<int32_t>(strlen(SEED_COLUMN_NAME))); } get_param.add_cell(ci); if (using_id) { ci.column_id_ = CELL_NUM_COLUMN_ID; ci.column_name_.assign_ptr(NULL, 0); } else { ci.column_id_ = OB_INVALID_ID; ci.column_name_.assign_ptr((char*)CELL_NUM_COLUMN_NAME, static_cast<int32_t>(strlen(CELL_NUM_COLUMN_NAME))); } get_param.add_cell(ci); lrand48_r(&table_rand_seed_, &rand); int64_t cell_num = range_rand(1, cell_num_per_row, rand); int64_t column_num = cur_schema.column_end() - cur_schema.column_begin(); for (int64_t j = 0; j < cell_num; j++) { lrand48_r(&table_rand_seed_, &rand); int64_t column_pos = range_rand(0, column_num - 1, rand); //int64_t column_pos = 0; const ObColumnSchema &column_schema = cur_schema.column_begin()[column_pos]; if (using_id) { ci.column_id_ = column_schema.get_id(); ci.column_name_.assign_ptr(NULL, 0); } else { ci.column_id_ = OB_INVALID_ID; ci.column_name_.assign_ptr(const_cast<char*>(column_schema.get_name()), static_cast<int32_t>(strlen(column_schema.get_name()))); } get_param.add_cell(ci); } } ObVersionRange version_range; version_range.start_version_ = table_start_version_; version_range.border_flag_.set_max_value(); version_range.border_flag_.set_inclusive_start(); get_param.set_version_range(version_range); char* is_read_consistency_cfg = getenv("is_read_consistency"); bool is_read_consistency = (NULL != is_read_consistency_cfg && 0 == strcmp(is_read_consistency_cfg, "true")); get_param.set_is_read_consistency(is_read_consistency); assert(get_param.get_cell_size() > 0); return ret; }
// maybe not find a merge server int TaskFactory::get_table_tablet(const char * table_name, const uint64_t table_id, uint64_t & count) { int ret = OB_SUCCESS; ObScanParam scan_param; uint64_t max_len = 0; if (NULL == table_name) { TBSYS_LOG(ERROR, "check table name failed:table[%s], id[%lu]", table_name, table_id); ret = OB_ERROR; } else { ret = init_scan_param(table_name, table_id, max_len, scan_param); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init scan param failed:table[%s], ret[%d]", table_name, ret); } } if (OB_SUCCESS == ret) { TaskInfo task; task.set_table_id(table_id); task.set_table_name(table_name); // for the first table tablet ObString row_key; char temp_buffer[1]; memset(temp_buffer, 0, sizeof(temp_buffer)); row_key.assign(temp_buffer, sizeof(temp_buffer)); ObGetParam param; ObScanner scanner; ObServer server; ObString start_key; ObString end_key; ObCellInfo * cell = NULL; ObScannerIterator iter; bool row_change = false; ObString name; name.assign(const_cast<char*>(table_name), strlen(table_name)); ObCellInfo temp_cell; temp_cell.table_id_ = table_id; temp_cell.column_id_ = 0; const uint64_t MAX_LEN = 1024; char last_tablet_rowkey[MAX_LEN] = ""; const int32_t MAX_SERVER_ADDR_SIZE = 128; char server_addr[MAX_SERVER_ADDR_SIZE]; while ((OB_SUCCESS == ret) && (!is_max_rowkey(max_len, row_key.ptr(), row_key.length() - 1))) { param.reset(); param.set_is_read_consistency(false); temp_cell.row_key_ = row_key; ret = param.add_cell(temp_cell); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "add cell failed:ret[%d]", ret); break; } ret = rpc_->get(root_server_, timeout_, param, scanner); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get root table for tablet failed:table[%lu], ret[%d]", table_id, ret); break; } else { // skip the first row iter = scanner.begin(); ++iter; while ((iter != scanner.end()) && (OB_SUCCESS == (ret = iter.get_cell(&cell, &row_change))) && !row_change) { if (NULL == cell) { TBSYS_LOG(ERROR, "%s", "check cell failed"); ret = OB_INNER_STAT_ERROR; break; } start_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); ++iter; } } if (ret == OB_SUCCESS) { int64_t ip = 0; int64_t port = 0; int64_t version = 0; TabletLocation list; for (++iter; iter != scanner.end(); ++iter) { ret = iter.get_cell(&cell, &row_change); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret); break; } else if (row_change) // && (iter != last_iter)) { ret = init_new_task(name, start_key, end_key, scan_param, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret); break; } else { ret = insert_new_task(list, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret); break; } ++count; } list.clear(); start_key = end_key; end_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); } else { end_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); if ((cell->column_name_.compare("1_ms_port") == 0) || (cell->column_name_.compare("2_ms_port") == 0) || (cell->column_name_.compare("3_ms_port") == 0)) { ret = cell->value_.get_int(port); } else if ((cell->column_name_.compare("1_ipv4") == 0) || (cell->column_name_.compare("2_ipv4") == 0) || (cell->column_name_.compare("3_ipv4") == 0)) { ret = cell->value_.get_int(ip); } else if ((cell->column_name_.compare("1_tablet_version") == 0) || (cell->column_name_.compare("2_tablet_version") == 0) || (cell->column_name_.compare("3_tablet_version") == 0)) { ret = cell->value_.get_int(version); if (OB_SUCCESS == ret) { if (0 == port || port == 0) { TBSYS_LOG(WARN, "%s", "check port or ip failed"); } else { server.set_ipv4_addr(ip, port); ObTabletLocation addr(version, server); if (OB_SUCCESS != (ret = list.add(addr))) { TBSYS_LOG(ERROR, "add addr failed:server[%d], port[%d], ret[%d]", server.get_ipv4(), server.get_port(), ret); } else { server.to_string(server_addr, MAX_SERVER_ADDR_SIZE); TBSYS_LOG(DEBUG, "add addr succ:server[%s], version:%ld", server_addr, version); } } ip = port = version = 0; } } if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret); break; } } } // for the last row if ((OB_SUCCESS == ret) && (start_key != end_key)) { ret = init_new_task(name, start_key, end_key, scan_param, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret); } else { ret = insert_new_task(list, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret); } ++count; } if (OB_SUCCESS == ret) { // modify last row key for next get root table uint64_t len = end_key.length(); if (MAX_LEN > len + 1) { memcpy(last_tablet_rowkey, end_key.ptr(), len); last_tablet_rowkey[len] = 0; row_key.assign(last_tablet_rowkey, len + 1); } else { TBSYS_LOG(ERROR, "check end key failed:len[%lu]", len); } } } list.clear(); } } } return ret; }