TEST_F(TestTabletLocationItem, test_valid) { ObMergerTabletLocationList list; ObTabletLocation temp_server; ObServer chunkserver; for (int32_t i = 0; i < ObMergerTabletLocationList::MAX_REPLICA_COUNT; ++i) { chunkserver.set_ipv4_addr(i + 300000, i + 1024); temp_server.tablet_version_ = i; temp_server.chunkserver_ = chunkserver; EXPECT_TRUE(OB_SUCCESS == list.add(temp_server)); } // all invalid ObMergerTabletLocation server; for (int32_t i = 0; i < ObMergerTabletLocationList::MAX_REPLICA_COUNT; ++i) { chunkserver.set_ipv4_addr(i + 300000, i + 1024); server.server_.chunkserver_ = chunkserver; server.server_.tablet_version_ = i; EXPECT_TRUE(OB_SUCCESS == list.set_item_invalid(server)); EXPECT_TRUE(list[i].err_times_ == ObMergerTabletLocation::MAX_ERR_TIMES); } list.set_item_valid(1100); EXPECT_TRUE(list.get_timestamp() == 1100); for (int64_t i = 0; i < ObMergerTabletLocationList::MAX_REPLICA_COUNT; ++i) { EXPECT_TRUE(list[i].err_times_ == 0); } }
int main(int argc, char* argv[]) { ObServer cs; cs.set_ipv4_addr("localhost", 3500); BaseClient client; client.initialize(); ObServer root; root.set_ipv4_addr("10.232.35.40", 2500); ThreadSpecificBuffer buffer; ObRootServerRpcStub server; do { int ret = server.init(root, client.get_rpc()); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init failed"); break; } int64_t timeout = 2000; int64_t timestamp = 0; // heartbeat ret = server.renew_heartbeat(timeout, cs, timestamp); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "renew heartbeat failed:ret[%d]", ret); break; } else { printf("get new lease time[%ld]\n", timestamp); } sleep(1); // fetch schema timestamp = 1282900503726484; ObSchemaManager schema; ret = server.fetch_schema(timeout, timestamp, schema); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "fetch schema failed:ret[%d]", ret); break; } else { printf("fetch schema succ:appname[%s]\n", schema.get_app_name()); schema.print_info(); } } while(0); client.destory(); return 0; }
TEST_F(TestTabletLocationItem, test_sort) { ObMergerTabletLocationList list; ObServer root_server; const char * addr = "localhost"; root_server.set_ipv4_addr(addr, 8888); int32_t server_ip = root_server.get_ipv4(); // one server ObTabletLocation location; srand(100); ObServer temp; temp.set_ipv4_addr(rand() % 10245, 1023); //location.tablet_id_ = 1; location.chunkserver_ = temp; list.add(location); EXPECT_TRUE(OB_SUCCESS == list.sort(root_server)); int64_t count = list.size(); for (int64_t i = 0; i < count - 1; ++i) { EXPECT_TRUE(abs(list[i].server_.chunkserver_.get_ipv4() - server_ip) <= abs(list[i+1].server_.chunkserver_.get_ipv4() - server_ip)); } // two server temp.set_ipv4_addr(rand() % 10245, 1024); location.chunkserver_ = temp; list.add(location); EXPECT_TRUE(OB_SUCCESS == list.sort(root_server)); count = list.size(); for (int64_t i = 0; i < count - 1; ++i) { EXPECT_TRUE(abs(list[i].server_.chunkserver_.get_ipv4() - server_ip) <= abs(list[i+1].server_.chunkserver_.get_ipv4() - server_ip)); } // three server location.chunkserver_ = root_server; list.add(location); EXPECT_TRUE(OB_SUCCESS == list.sort(root_server)); count = list.size(); for (int64_t i = 0; i < count - 1; ++i) { EXPECT_TRUE(abs(list[i].server_.chunkserver_.get_ipv4() - server_ip) <= abs(list[i+1].server_.chunkserver_.get_ipv4() - server_ip)); } }
TEST_F(TestTabletLocationItem, test_invalid) { ObMergerTabletLocationList list; ObTabletLocation temp_server; ObServer chunkserver; for (int32_t i = 0; i < ObMergerTabletLocationList::MAX_REPLICA_COUNT; ++i) { chunkserver.set_ipv4_addr(i + 300000, i + 1024); temp_server.tablet_version_ = i; temp_server.chunkserver_ = chunkserver; EXPECT_TRUE(OB_SUCCESS == list.add(temp_server)); } ObMergerTabletLocation server; // not exist for (int32_t i = 0; i < ObMergerTabletLocationList::MAX_REPLICA_COUNT; ++i) { chunkserver.set_ipv4_addr(i + 200000, i + 1024); server.server_.chunkserver_ = chunkserver; server.server_.tablet_version_ = i; EXPECT_TRUE(OB_SUCCESS != list.set_item_invalid(server)); EXPECT_TRUE(list[i].err_times_ != ObMergerTabletLocation::MAX_ERR_TIMES); } // set second invalid chunkserver.set_ipv4_addr(1 + 300000, 1 + 1024); server.server_.chunkserver_ = chunkserver; server.server_.tablet_version_ = 1; EXPECT_TRUE(OB_SUCCESS == list.set_item_invalid(server)); EXPECT_TRUE(list[1].err_times_ == ObMergerTabletLocation::MAX_ERR_TIMES); EXPECT_TRUE(list[0].err_times_ != ObMergerTabletLocation::MAX_ERR_TIMES); EXPECT_TRUE(list[2].err_times_ != ObMergerTabletLocation::MAX_ERR_TIMES); // all invalid for (int32_t i = 0; i < ObMergerTabletLocationList::MAX_REPLICA_COUNT; ++i) { chunkserver.set_ipv4_addr(i + 300000, i + 1024); server.server_.chunkserver_ = chunkserver; server.server_.tablet_version_ = i; EXPECT_TRUE(OB_SUCCESS == list.set_item_invalid(server)); EXPECT_TRUE(list[i].err_times_ == ObMergerTabletLocation::MAX_ERR_TIMES); } for (int64_t i = 0; i < ObMergerTabletLocationList::MAX_REPLICA_COUNT; ++i) { server.server_.tablet_version_ = i + 100; EXPECT_TRUE(OB_SUCCESS == list.set_item_invalid(server)); } }
int MockRootServer::handle_get_updater(ObPacket * ob_packet) { int ret = OB_SUCCESS; ObDataBuffer* data = ob_packet->get_buffer(); if (NULL == data) { ret = OB_ERROR; } tbnet::Connection* connection = ob_packet->get_connection(); ThreadSpecificBuffer::Buffer* thread_buffer = response_packet_buffer_.get_buffer(); if (NULL == thread_buffer) { ret = OB_ERROR; } else { thread_buffer->reset(); ObDataBuffer out_buffer(thread_buffer->current(), thread_buffer->remain()); ObResultCode result_msg; result_msg.result_code_ = ret; ret = result_msg.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position()); // new version int32_t channel_id = ob_packet->getChannelId(); ObServer server; server.set_ipv4_addr("localhost", MockUpdateServer::UPDATE_SERVER_PORT); ret = server.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position()); ret = send_response(OB_REPORT_TABLETS_RESPONSE, 1, out_buffer, connection, channel_id); } return ret; }
int ObMonitor::select_master(node_list_type *pnodelist, ObServer &new_master) { int err = OB_ERROR; if(pnodelist == NULL) { TBSYS_LOG(WARN, "invalid argument"); err = OB_INVALID_ARGUMENT; } if(OB_ERROR == err) { node_list_type::iterator it = (*pnodelist).begin(); while(it != (*pnodelist).end()) { if((*it)->status == SLAVE_SYNC) { new_master.set_ipv4_addr((*it)->node_server.get_ipv4(), (*it)->node_server.get_port()); (*it)->status = MASTER_ACTIVE; err = OB_SUCCESS; break; } it ++; } } return err; }
TEST_F(TestTabletLocationItem, test_deserialize) { ObMergerTabletLocation server; ObMergerTabletLocationList list; list.set_timestamp(1000); ObTabletLocation temp_server; ObServer chunkserver; for (int32_t i = 0; i < ObMergerTabletLocationList::MAX_REPLICA_COUNT; ++i) { chunkserver.set_ipv4_addr(i + 300000, i + 1024); temp_server.tablet_version_ = i; temp_server.chunkserver_ = chunkserver; EXPECT_TRUE(OB_SUCCESS == list.add(temp_server)); EXPECT_TRUE(i + 1 == list.size()); } ObStringBuf buffer; list.set_buffer(buffer); list.print_info(); char start[1024] = "0x00000000"; char end[1024] = "0xFFFFFFFF"; ObRange range; range.start_key_.assign(start, (int32_t)strlen(start)); range.end_key_.assign(end, (int32_t)strlen(end)); int ret = list.set_tablet_range(range); EXPECT_TRUE(OB_SUCCESS == ret); int64_t size = list.get_serialize_size(); EXPECT_TRUE(size != 0); char * temp = new char[size]; EXPECT_TRUE(NULL != temp); int64_t pos = 0; EXPECT_TRUE(OB_SUCCESS != list.serialize(temp, size - 1 , pos)); pos = 0; EXPECT_TRUE(OB_SUCCESS == list.serialize(temp, size, pos)); EXPECT_TRUE(pos == size); ObMergerTabletLocationList list2; list2.set_buffer(buffer); pos = 0; EXPECT_TRUE(OB_SUCCESS == list2.deserialize(temp, size, pos)); list2.print_info(); list.print_info(); char start1[1024] = "0x0000000000"; char end1[1024] = "0xFFFFFFFFFF"; range.start_key_.assign(start1, (int32_t)strlen(start1)); range.end_key_.assign(end1, (int32_t)strlen(end1)); EXPECT_TRUE(OB_SUCCESS == list.set_tablet_range(range)); size = list.get_serialize_size(); EXPECT_TRUE(size != 0); delete []temp; temp = new char[size]; EXPECT_TRUE(NULL != temp); pos = 0; EXPECT_TRUE(OB_SUCCESS == list.serialize(temp, size, pos)); pos = 0; EXPECT_TRUE(OB_SUCCESS != list2.deserialize(temp, size, pos)); // pos = 0; list2.reset(); EXPECT_TRUE(OB_SUCCESS == list2.deserialize(temp, size, pos)); }
TEST_F(TestTabletLocationItem, test_serialize) { ObMergerTabletLocation server; ObMergerTabletLocationList list; list.set_timestamp(1000); ObTabletLocation temp_server; ObServer chunkserver; for (int32_t i = 0; i < ObMergerTabletLocationList::MAX_REPLICA_COUNT; ++i) { chunkserver.set_ipv4_addr(i + 300000, i + 1024); temp_server.tablet_version_ = i; temp_server.chunkserver_ = chunkserver; EXPECT_TRUE(OB_SUCCESS == list.add(temp_server)); EXPECT_TRUE(i + 1 == list.size()); } list.print_info(); int64_t size = list.get_serialize_size(); EXPECT_TRUE(size != 0); char * temp = new char[size]; EXPECT_TRUE(NULL != temp); int64_t pos = 0; EXPECT_TRUE(OB_SUCCESS != list.serialize(temp, size - 1 , pos)); pos = 0; EXPECT_TRUE(OB_SUCCESS == list.serialize(temp, size, pos)); EXPECT_TRUE(pos == size); ObMergerTabletLocationList list2; pos = 0; EXPECT_TRUE(OB_SUCCESS == list2.deserialize(temp, size, pos)); list2.print_info(); list.print_info(); }
ObServer &ObDeleteReplicasTest::get_addr(int32_t idx) { char buff[128]; snprintf(buff, 128, "10.10.10.%d", idx); int port = 26666; addr_.set_ipv4_addr(buff, port); return addr_; }
void run_migrate_test_case(ParamSet& param, char * cmd) { bool print_help = parse_param(param,cmd); if ( true == print_help ) printhelp(); else { printParam(param); ObServer cs; cs.set_ipv4_addr(param.server_ip, param.port); ObServer ds; ds.set_ipv4_addr(param.dest_ip, param.dest_port); BaseClient client; if (OB_SUCCESS != client.initialize()) { TBSYS_LOG(ERROR, "initialize client failed."); } migrate_test_case(client, cs, ds, param.table_id, param.user_id, param.item_type); client.destory(); printf("================MIGRATE TEST DONE================\n"); } }
ObAppStats::ObAppStats(ObClientRpcStub &stub, const ObRole server_type, const char* config_file_name) : ObServerStats(stub, server_type) { root_server_array_.init(MAX_SERVER_NUM, root_servers_, 0); tbsys::CConfig config; if(config.load(config_file_name)) { fprintf(stderr, "load file %s error\n", config_file_name); } else { master_index_ = -1; int count = config.getInt(SERVER_SECTION, "cluster_count", 0); if (count <= 0) { fprintf(stderr, "cluster_count %d cannot < 0\n", count); } else { const int MAX_LEN = 64; char ipname[MAX_LEN]; char portname[MAX_LEN]; char ip[MAX_LEN]; char mastername[MAX_LEN]; int32_t port = 0; int is_master = 0; ObServer server; for (int i = 0; i < count ; ++i) { snprintf(ipname, MAX_LEN, "rs%d_ip", i); snprintf(portname, MAX_LEN, "rs%d_port", i); snprintf(mastername, MAX_LEN, "rs%d_is_master", i); if (OB_SUCCESS != load_string(config, ip, MAX_LEN, SERVER_SECTION, ipname, true)) { fprintf(stderr, "%s is null\n", ipname); break; } port = config.getInt(SERVER_SECTION, portname, 0); if (port <= 0) { fprintf(stderr, "%s is 0\n", portname); break; } is_master = config.getInt(SERVER_SECTION, mastername, 0); if (is_master > 0) master_index_ = i; server.set_ipv4_addr(ip, port); root_server_array_.push_back(server); } } } }
int parse_obsql_options(int argc, char *argv[], ObServer &root_server) { int ret = 0; const char *addr = NULL; int32_t port = 0; static struct option longopts[] = { {"hostip", required_argument, NULL, 'h'}, {"port", required_argument, NULL, 'p'}, {"quiet", required_argument, NULL, 'q'}, {"help", no_argument, NULL, '?'}, {NULL, 0, NULL, 0} }; while((ret = getopt_long(argc, argv,"h:p:q?", longopts, NULL)) != -1) { switch(ret) { case 'h': addr = optarg; break; case 'p': port = atoi(optarg); break; case 'q': TBSYS_LOGGER.setLogLevel("ERROR"); break; case '?': if (strcmp(argv[optind - 1], "-?") == 0 || strcmp(argv[optind - 1], "--help") == 0) usage(); break; default: fprintf(stderr, "%s is not identified\n",optarg); usage(); exit(1); break; }; } ret = root_server.set_ipv4_addr(addr, port); if (!ret) { fprintf(stderr, "root_server addr invalid, addr=%s, port=%d\n", addr, port); return OB_ERROR; } return OB_SUCCESS; }
TEST_F(TestBlackList, init) { ObMergerServerBlackList list; ObUpsList ups_list; EXPECT_TRUE(OB_SUCCESS != list.init(10, 10000, MERGE_SERVER, ups_list)); ObUpsInfo info; ObServer server; for (int i = 0; i < MAX_COUNT; ++i) { server.set_ipv4_addr(i+1024, i+2048); info.addr_ = server; info.ms_read_percentage_ = 10; ups_list.ups_count_ = i + 1; ups_list.ups_array_[i] = info; } EXPECT_TRUE(OB_SUCCESS == list.init(10, 10000, MERGE_SERVER, ups_list)); }
void* thread_run(void* arg) { ParamSet *param = reinterpret_cast<ParamSet*>(arg); ObServer cs; cs.set_ipv4_addr(param->server_ip, param->port); BaseClient client; if (OB_SUCCESS != client.initialize()) { TBSYS_LOG(ERROR, "initialize client failed."); //return OB_ERROR; } // get_test_case(client, cs, param->table_id, param->user_id, param->item_type, param->item_id); //scan_test_case(client, cs, param->table_id, param->user_id, param->item_type, param->column_count); // create_test_case(client, cs, param->table_id,param->user_id,param->item_type); // migrate_test_case(); // merge_test_case(); return (void *)0; }
void run_drop_test_case(ParamSet& param, char * cmd) { bool print_help = parse_param(param,cmd); if ( true == print_help ) printhelp(); else { printParam(param); ObServer cs; cs.set_ipv4_addr(param.server_ip, param.port); BaseClient client; if (OB_SUCCESS != client.initialize()) { TBSYS_LOG(ERROR, "initialize client failed."); } drop_test_case(client, cs, param.memtable_frozen_version); client.destory(); printf("==============DROP TEST DONE==================\n"); } }
void run_scan_test_case(ParamSet& param, char * cmd) { bool print_help = parse_param(param,cmd); if ( true == print_help ) printhelp(); else { printParam(param); ObServer cs; cs.set_ipv4_addr(param.server_ip, param.port); BaseClient client; if (OB_SUCCESS != client.initialize()) { TBSYS_LOG(ERROR, "initialize client failed."); } scan_test_case(client, cs, param.table_id, param.user_id, static_cast<int8_t>(param.item_type), static_cast<int32_t>(param.column_count)); client.destory(); printf("================SCAN TEST DONE===============\n"); } }
void run_get_test_case_common(ParamSet& param, char * cmd) { bool print_help = parse_param(param,cmd); if ( true == print_help ) printhelp(); else { printParam(param); ObServer cs; cs.set_ipv4_addr(param.server_ip, param.port); BaseClient client; if (OB_SUCCESS != client.initialize()) { TBSYS_LOG(ERROR, "initialize client failed."); } get_test_case_common(client, cs, param.table_id, param.start_key, param.column_count); client.destory(); printf("===============GET TEST DONE==============\n"); } }
void run_merge_scan_case(ParamSet& param, char * cmd) { bool print_help = parse_param(param,cmd); if ( true == print_help ) printhelp(); else { printParam(param); ObServer cs; cs.set_ipv4_addr(param.server_ip, param.port); BaseClient client; if (OB_SUCCESS != client.initialize()) { TBSYS_LOG(ERROR, "initialize client failed."); } merge_scan_case(client, cs, param.table_name, param.start_key, param.end_key, param.key_size, param.is_hex_key); client.destory(); printf("===============GET TEST DONE==============\n"); } }
int main(int argc, char ** argv) { UNUSED(argc); UNUSED(argv); int ret = OB_SUCCESS; //char * test = "00596331192663719"; char * test = "00912219024438372"; ObString rowkey; rowkey.assign(test, strlen(test)); char temp[1024]; hex_to_str(rowkey.ptr(), rowkey.length(), temp, 1024); printf("test2str:test[%s], str[%s]\n", test, temp); int64_t ip = 539289610; //488957962; //522512394; ObServer server; server.set_ipv4_addr(ip, 1024); static const int32_t MAX_SERVER_ADDR_SIZE = 128; char server_addr[MAX_SERVER_ADDR_SIZE]; server.to_string(server_addr, MAX_SERVER_ADDR_SIZE); printf("server:[%ld], server[%s]\n", ip, server_addr); return ret; }
TEST_F(TestBlackList, fail_check) { ObMergerServerBlackList list; ObUpsList ups_list; EXPECT_TRUE(OB_SUCCESS != list.init(10, 10000, MERGE_SERVER, ups_list)); ObServer server; for (int i = 0; i < MAX_COUNT; ++i) { EXPECT_TRUE(true == list.check(i)); list.fail(i, server); } ObUpsInfo info; for (int i = 0; i < MAX_COUNT; ++i) { server.set_ipv4_addr(i+1024, i+2048); info.addr_ = server; info.ms_read_percentage_ = 10; ups_list.ups_count_ = i + 1; ups_list.ups_array_[i] = info; } EXPECT_TRUE(OB_SUCCESS == list.init(10, 10000, MERGE_SERVER, ups_list)); for (int i = MAX_COUNT; i < MAX_COUNT + 10; ++i) { server.set_ipv4_addr(i+1024, i+2048); EXPECT_TRUE(true == list.check(i)); for (int j = 0; j < 10; ++j) { list.fail(i - MAX_COUNT, server); } EXPECT_TRUE(list.get_valid_count() == MAX_COUNT); } EXPECT_TRUE(list.get_valid_count() == MAX_COUNT); for (int i = 0; i < MAX_COUNT; ++i) { server.set_ipv4_addr(i+1024, i+2048); EXPECT_TRUE(true == list.check(i)); for (int j = 0; j < 10; ++j) { EXPECT_TRUE(true == list.check(i)); EXPECT_TRUE(list.get_valid_count() == (MAX_COUNT - i)); list.fail(i, server); } list.fail(i, server); EXPECT_TRUE(list.get_valid_count() == (MAX_COUNT - i - 1)); EXPECT_TRUE(false == list.check(i)); } EXPECT_TRUE(list.get_valid_count() == 0); list.reset(); EXPECT_TRUE(list.get_valid_count() == MAX_COUNT); for (int i = 0; i < MAX_COUNT; ++i) { server.set_ipv4_addr(i+1024, i+2048); EXPECT_TRUE(true == list.check(i)); } for (int i = 0; i < MAX_COUNT; ++i) { server.set_ipv4_addr(i+1024, i+2048); EXPECT_TRUE(true == list.check(i)); for (int j = 0; j < 10; ++j) { EXPECT_TRUE(true == list.check(i)); EXPECT_TRUE(list.get_valid_count() == (MAX_COUNT - i)); list.fail(i, server); } list.fail(i, server); EXPECT_TRUE(list.get_valid_count() == (MAX_COUNT - i - 1)); EXPECT_TRUE(false == list.check(i)); } // timeout to alive again usleep(10000); for (int i = 0; i < MAX_COUNT; ++i) { server.set_ipv4_addr(i+1024, i+2048); EXPECT_TRUE(true == list.check(i)); EXPECT_TRUE(list.get_valid_count() == MAX_COUNT); } usleep(10000); for (int i = 0; i < MAX_COUNT; ++i) { server.set_ipv4_addr(i+1024, i+2048); EXPECT_TRUE(true == list.check(i)); EXPECT_TRUE(list.get_valid_count() == MAX_COUNT); } }
// maybe not find a merge server int TaskFactory::get_table_tablet(const char * table_name, const uint64_t table_id, uint64_t & count) { int ret = OB_SUCCESS; ObScanParam scan_param; uint64_t max_len = 0; if (NULL == table_name) { TBSYS_LOG(ERROR, "check table name failed:table[%s], id[%lu]", table_name, table_id); ret = OB_ERROR; } else { ret = init_scan_param(table_name, table_id, max_len, scan_param); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init scan param failed:table[%s], ret[%d]", table_name, ret); } } if (OB_SUCCESS == ret) { TaskInfo task; task.set_table_id(table_id); task.set_table_name(table_name); // for the first table tablet ObString row_key; char temp_buffer[1]; memset(temp_buffer, 0, sizeof(temp_buffer)); row_key.assign(temp_buffer, sizeof(temp_buffer)); ObGetParam param; ObScanner scanner; ObServer server; ObString start_key; ObString end_key; ObCellInfo * cell = NULL; ObScannerIterator iter; bool row_change = false; ObString name; name.assign(const_cast<char*>(table_name), strlen(table_name)); ObCellInfo temp_cell; temp_cell.table_id_ = table_id; temp_cell.column_id_ = 0; const uint64_t MAX_LEN = 1024; char last_tablet_rowkey[MAX_LEN] = ""; const int32_t MAX_SERVER_ADDR_SIZE = 128; char server_addr[MAX_SERVER_ADDR_SIZE]; while ((OB_SUCCESS == ret) && (!is_max_rowkey(max_len, row_key.ptr(), row_key.length() - 1))) { param.reset(); param.set_is_read_consistency(false); temp_cell.row_key_ = row_key; ret = param.add_cell(temp_cell); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "add cell failed:ret[%d]", ret); break; } ret = rpc_->get(root_server_, timeout_, param, scanner); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get root table for tablet failed:table[%lu], ret[%d]", table_id, ret); break; } else { // skip the first row iter = scanner.begin(); ++iter; while ((iter != scanner.end()) && (OB_SUCCESS == (ret = iter.get_cell(&cell, &row_change))) && !row_change) { if (NULL == cell) { TBSYS_LOG(ERROR, "%s", "check cell failed"); ret = OB_INNER_STAT_ERROR; break; } start_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); ++iter; } } if (ret == OB_SUCCESS) { int64_t ip = 0; int64_t port = 0; int64_t version = 0; TabletLocation list; for (++iter; iter != scanner.end(); ++iter) { ret = iter.get_cell(&cell, &row_change); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret); break; } else if (row_change) // && (iter != last_iter)) { ret = init_new_task(name, start_key, end_key, scan_param, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret); break; } else { ret = insert_new_task(list, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret); break; } ++count; } list.clear(); start_key = end_key; end_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); } else { end_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); if ((cell->column_name_.compare("1_ms_port") == 0) || (cell->column_name_.compare("2_ms_port") == 0) || (cell->column_name_.compare("3_ms_port") == 0)) { ret = cell->value_.get_int(port); } else if ((cell->column_name_.compare("1_ipv4") == 0) || (cell->column_name_.compare("2_ipv4") == 0) || (cell->column_name_.compare("3_ipv4") == 0)) { ret = cell->value_.get_int(ip); } else if ((cell->column_name_.compare("1_tablet_version") == 0) || (cell->column_name_.compare("2_tablet_version") == 0) || (cell->column_name_.compare("3_tablet_version") == 0)) { ret = cell->value_.get_int(version); if (OB_SUCCESS == ret) { if (0 == port || port == 0) { TBSYS_LOG(WARN, "%s", "check port or ip failed"); } else { server.set_ipv4_addr(ip, port); ObTabletLocation addr(version, server); if (OB_SUCCESS != (ret = list.add(addr))) { TBSYS_LOG(ERROR, "add addr failed:server[%d], port[%d], ret[%d]", server.get_ipv4(), server.get_port(), ret); } else { server.to_string(server_addr, MAX_SERVER_ADDR_SIZE); TBSYS_LOG(DEBUG, "add addr succ:server[%s], version:%ld", server_addr, version); } } ip = port = version = 0; } } if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret); break; } } } // for the last row if ((OB_SUCCESS == ret) && (start_key != end_key)) { ret = init_new_task(name, start_key, end_key, scan_param, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret); } else { ret = insert_new_task(list, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret); } ++count; } if (OB_SUCCESS == ret) { // modify last row key for next get root table uint64_t len = end_key.length(); if (MAX_LEN > len + 1) { memcpy(last_tablet_rowkey, end_key.ptr(), len); last_tablet_rowkey[len] = 0; row_key.assign(last_tablet_rowkey, len + 1); } else { TBSYS_LOG(ERROR, "check end key failed:len[%lu]", len); } } } list.clear(); } } } return ret; }
int ObRootServerRpcStub::get_tablet_info(const common::ObSchemaManagerV2& schema, const uint64_t table_id, const ObRange& range, ObTabletLocation location [],int32_t& size) { int ret = OB_SUCCESS; int32_t index = 0; const ObChunkServer& chunk_server = ObChunkServerMain::get_instance()->get_chunk_server(); const int64_t timeout = chunk_server.get_param().get_network_time_out(); if (OB_INVALID_ID == table_id || size <= 0) { TBSYS_LOG(ERROR,"invalid table id"); ret = OB_ERROR; } ObScanParam param; ObScanner scanner; ObString table_name; const ObTableSchema *table = schema.get_table_schema(table_id); if (NULL == table) { TBSYS_LOG(ERROR,"can not find table %lu",table_id); ret = OB_ERROR; } else { table_name.assign_ptr(const_cast<char *>(table->get_table_name()),strlen(table->get_table_name())); } if (OB_SUCCESS == ret) { param.set(OB_INVALID_ID,table_name,range); //use table name param.set_is_read_consistency(false); } if ((OB_SUCCESS == ret) && ((ret = scan(root_server_,timeout,param,scanner)) != OB_SUCCESS) ) { TBSYS_LOG(ERROR,"get tablet from rootserver failed:[%d]",ret); } ObServer server; char tmp_buf[32]; ObString start_key; ObString end_key; ObCellInfo * cell = NULL; ObScannerIterator iter; bool row_change = false; if (OB_SUCCESS == ret) { int64_t ip = 0; int64_t port = 0; int64_t version = 0; iter = scanner.begin(); ret = iter.get_cell(&cell, &row_change); row_change = false; while((OB_SUCCESS == ret) && index < size) { if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret); } else if (row_change) { TBSYS_LOG(DEBUG,"row changed,ignore"); hex_dump(cell->row_key_.ptr(),cell->row_key_.length(),false,TBSYS_LOG_LEVEL_INFO); break; //just get one row } else if (cell != NULL) { end_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); if ((cell->column_name_.compare("1_port") == 0) || (cell->column_name_.compare("2_port") == 0) || (cell->column_name_.compare("3_port") == 0)) { ret = cell->value_.get_int(port); TBSYS_LOG(DEBUG,"port is %ld",port); } else if ((cell->column_name_.compare("1_ipv4") == 0) || (cell->column_name_.compare("2_ipv4") == 0) || (cell->column_name_.compare("3_ipv4") == 0)) { ret = cell->value_.get_int(ip); TBSYS_LOG(DEBUG,"ip is %ld",ip); } else if (cell->column_name_.compare("1_tablet_version") == 0 || cell->column_name_.compare("2_tablet_version") == 0 || cell->column_name_.compare("3_tablet_version") == 0) { ret = cell->value_.get_int(version); hex_dump(cell->row_key_.ptr(),cell->row_key_.length(),false,TBSYS_LOG_LEVEL_INFO); TBSYS_LOG(DEBUG,"tablet_version is %d",version); } if (OB_SUCCESS == ret) { if (0 != port && 0 != ip && 0 != version) { server.set_ipv4_addr(ip, port); server.to_string(tmp_buf,sizeof(tmp_buf)); TBSYS_LOG(INFO,"add tablet s:%s,%ld",tmp_buf,version); ObTabletLocation addr(version, server); location[index++] = addr; ip = port = version = 0; } } else { TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret); } if (++iter == scanner.end()) break; ret = iter.get_cell(&cell, &row_change); } else { //impossible } } } if (OB_SUCCESS == ret) { size = index; TBSYS_LOG(INFO,"get %d tablets from rootserver",size); } return ret; }
// waring:all return cell in a row must be same as root table's columns, // and the second row is this row allocated chunkserver list int ObMergerRootRpcProxy::scan_root_table(ObMergerTabletLocationCache * cache, const uint64_t table_id, const ObString & row_key, const ObServer & addr, ObMergerTabletLocationList & location) { assert(location.get_buffer() != NULL); int ret = OB_SUCCESS; bool find_right_tablet = false; ObScanner scanner; // root table id = 0 ret = rpc_stub_->fetch_tablet_location(rpc_timeout_, root_server_, 0, table_id, row_key, scanner); if (ret != OB_SUCCESS) { TBSYS_LOG(WARN, "fetch tablet location failed:table_id[%lu], length[%d], ret[%d]", table_id, row_key.length(), ret); hex_dump(row_key.ptr(), row_key.length(), true); } else { ObRange range; range.border_flag_.unset_inclusive_start(); range.border_flag_.set_inclusive_end(); ObString start_key; ObString end_key; ObServer server; ObCellInfo * cell = NULL; bool row_change = false; ObScannerIterator iter = scanner.begin(); TBSYS_LOG(DEBUG, "%s", "parse scanner result for get some tablet locations"); // all return cell in a row must be same as root table's columns ++iter; while ((iter != scanner.end()) && (OB_SUCCESS == (ret = iter.get_cell(&cell, &row_change))) && !row_change) { if (NULL == cell) { ret = OB_INNER_STAT_ERROR; break; } start_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); ++iter; } if (ret == OB_SUCCESS) { int64_t ip = 0; int64_t port = 0; bool second_row = true; // next cell ObMergerTabletLocationList list; for (++iter; iter != scanner.end(); ++iter) { ret = iter.get_cell(&cell, &row_change); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret); break; } else if (row_change) // && (iter != last_iter)) { range.table_id_ = table_id; if (NULL == start_key.ptr()) { range.border_flag_.set_min_value(); } else { range.border_flag_.unset_min_value(); range.start_key_ = start_key; } range.border_flag_.unset_max_value(); range.end_key_ = end_key; start_key = end_key; end_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); list.set_timestamp(tbsys::CTimeUtil::getTime()); list.sort(addr); // not deep copy the range list.set_tablet_range(range); // the second row is this row allocated chunkserver list if (second_row) { second_row = false; if ((row_key <= range.end_key_) && (row_key > range.start_key_)) { find_right_tablet = true; location = list; assert(location.get_buffer() != NULL); location.set_tablet_range(range); } else { ret = OB_DATA_NOT_SERVE; TBSYS_LOG(ERROR, "check range not include this key:ret[%d]", ret); hex_dump(row_key.ptr(), row_key.length()); range.hex_dump(); break; } } // add to cache if (OB_SUCCESS != cache->set(range, list)) { TBSYS_LOG(WARN, "%s", "add the range to cache failed"); } list.clear(); } else { end_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); if ((cell->column_name_.compare("1_port") == 0) || (cell->column_name_.compare("2_port") == 0) || (cell->column_name_.compare("3_port") == 0)) { ret = cell->value_.get_int(port); } else if ((cell->column_name_.compare("1_ipv4") == 0) || (cell->column_name_.compare("2_ipv4") == 0) || (cell->column_name_.compare("3_ipv4") == 0)) { ret = cell->value_.get_int(ip); if (OB_SUCCESS == ret) { if (port == 0) { TBSYS_LOG(WARN, "check port failed:ip[%ld], port[%ld]", ip, port); } server.set_ipv4_addr(static_cast<int32_t>(ip), static_cast<int32_t>(port)); ObTabletLocation addr(0, server); if (OB_SUCCESS != (ret = list.add(addr))) { TBSYS_LOG(ERROR, "add addr failed:ip[%ld], port[%ld], ret[%d]", ip, port, ret); break; } else { TBSYS_LOG(DEBUG, "add addr succ:ip[%ld], port[%ld]", ip, port); } ip = port = 0; } } if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret); break; } } } // for the last row if ((OB_SUCCESS == ret) && (start_key != end_key)) { range.table_id_ = table_id; if (NULL == start_key.ptr()) { range.border_flag_.set_min_value(); } else { range.border_flag_.unset_min_value(); range.start_key_ = start_key; } range.border_flag_.unset_max_value(); range.end_key_ = end_key; list.set_timestamp(tbsys::CTimeUtil::getTime()); // not deep copy the range list.set_tablet_range(range); list.sort(addr); // double check add all range->locationlist to cache if ((row_key <= range.end_key_) && (row_key > range.start_key_)) { find_right_tablet = true; location = list; // deep copy range assert(location.get_buffer() != NULL); location.set_tablet_range(range); } else if (second_row) { range.hex_dump(); ret = OB_DATA_NOT_SERVE; TBSYS_LOG(ERROR, "check range not include this key:ret[%d]", ret); } // add to list to cache if (OB_SUCCESS != cache->set(range, list)) { range.hex_dump(); TBSYS_LOG(WARN, "%s", "add the range to cache failed"); } } } else { TBSYS_LOG(ERROR, "check get first row cell failed:ret[%d]", ret); } } if ((OB_SUCCESS == ret) && (0 == location.size())) { TBSYS_LOG(ERROR, "check get location size failed:table_id[%ld], find[%d], count[%ld]", table_id, find_right_tablet, location.size()); hex_dump(row_key.ptr(), row_key.length()); ret = OB_INNER_STAT_ERROR; } ms_get_counter_set().inc(ObMergerCounterIds::C_SCAN_ROOT_TABLE); return ret; }
// maybe not find a merge server int TaskFactory::get_table_tablet(const char * table_name, const uint64_t table_id, uint64_t & count) { int ret = OB_SUCCESS; ObScanParam scan_param; uint64_t max_len = 0; if (NULL == table_name) { TBSYS_LOG(ERROR, "check table name failed:table[%s], id[%lu]", table_name, table_id); ret = OB_ERROR; } else { ret = init_scan_param(table_name, table_id, max_len, scan_param); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init scan param failed:table[%s], ret[%d]", table_name, ret); } } if (OB_SUCCESS == ret) { // for the first table tablet ObRowkey row_key; ObObj temp_buffer[common::OB_MAX_ROWKEY_COLUMN_NUMBER]; row_key.assign(temp_buffer, common::OB_MAX_ROWKEY_COLUMN_NUMBER); ObGetParam param; ObScanner scanner; ObServer server; ObRowkey start_key; ObRowkey end_key; ObCellInfo * cell = NULL; ObScannerIterator iter; bool row_change = false; ObString name; name.assign(const_cast<char*>(table_name), (int32_t)strlen(table_name)); ObCellInfo temp_cell; temp_cell.table_id_ = table_id; temp_cell.column_id_ = 0; common::ModuleArena allocator; while ((OB_SUCCESS == ret) && (!row_key.is_max_row())) { param.reset(); param.set_is_read_consistency(false); temp_cell.row_key_ = row_key; ret = param.add_cell(temp_cell); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "add cell failed:ret[%d]", ret); break; } ret = rpc_->get(root_server_, timeout_, param, scanner); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get root table for tablet failed:table[%lu], ret[%d]", table_id, ret); break; } else { // skip the first row iter = scanner.begin(); ++iter; while ((iter != scanner.end()) && (OB_SUCCESS == (ret = iter.get_cell(&cell, &row_change))) && !row_change) { if (NULL == cell) { TBSYS_LOG(ERROR, "%s", "check cell failed"); ret = OB_INNER_STAT_ERROR; break; } ret = cell->row_key_.deep_copy(start_key, allocator); if (ret != OB_SUCCESS) { TBSYS_LOG(WARN, "deep copy the rowkey failed:ret[%d]", ret); } ++iter; } } // scanner iter end if (ret == OB_SUCCESS) { int64_t ip = 0; int64_t port = 0; int64_t version = 0; TabletLocation list; for (++iter; iter != scanner.end(); ++iter) { ret = iter.get_cell(&cell, &row_change); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret); break; } else if (row_change) // && (iter != last_iter)) { TaskInfo task; task.set_table_id(table_id); task.set_table_name(table_name); ret = init_new_task(name, start_key, end_key, scan_param, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret); break; } else { ret = insert_new_task(list, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret); break; } ++count; } list.clear(); start_key = end_key; } else { cell->row_key_.deep_copy(end_key, allocator); if ((cell->column_name_.compare("1_ms_port") == 0) || (cell->column_name_.compare("2_ms_port") == 0) || (cell->column_name_.compare("3_ms_port") == 0)) { ret = cell->value_.get_int(port); } else if ((cell->column_name_.compare("1_ipv4") == 0) || (cell->column_name_.compare("2_ipv4") == 0) || (cell->column_name_.compare("3_ipv4") == 0)) { ret = cell->value_.get_int(ip); } else if ((cell->column_name_.compare("1_tablet_version") == 0) || (cell->column_name_.compare("2_tablet_version") == 0) || (cell->column_name_.compare("3_tablet_version") == 0)) { ret = cell->value_.get_int(version); if (OB_SUCCESS == ret) { if (0 == port || port == 0) { TBSYS_LOG(WARN, "%s", "check port or ip failed"); } else { server.set_ipv4_addr(int32_t(ip), int32_t(port)); ObTabletLocation addr(version, server); if (OB_SUCCESS != (ret = list.add(addr))) { TBSYS_LOG(ERROR, "add addr failed:server[%d], port[%d], ret[%d]", server.get_ipv4(), server.get_port(), ret); } else { TBSYS_LOG(DEBUG, "add addr succ:server[%s], version:%ld", server.to_cstring(), version); } } ip = port = version = 0; } } if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret); break; } } } // for the last row if ((OB_SUCCESS == ret) && (start_key != end_key)) { TaskInfo task; task.set_table_id(table_id); task.set_table_name(table_name); ret = init_new_task(name, start_key, end_key, scan_param, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret); } else { ret = insert_new_task(list, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret); } ++count; } if (OB_SUCCESS == ret) { // copy all the end key objs to row_key for (int64_t i = 0; i < end_key.length(); ++i) { const_cast<ObObj *> (row_key.ptr())[i] = end_key.ptr()[i]; row_key.assign(const_cast<ObObj *> (row_key.ptr()), end_key.length()); } // modify last row key for next get root table if (!end_key.is_max_row()) { const_cast<ObObj *>(row_key.ptr())[end_key.length()].set_max_value(); row_key.assign(const_cast<ObObj *> (row_key.ptr()), end_key.length() + 1); } } } list.clear(); } } } return ret; }
TEST_F(TestGetRequestEvent, test_get) { ObMergerRpcStub stub; ThreadSpecificBuffer buffer; ObPacketFactory factory; tbnet::Transport transport; tbnet::DefaultPacketStreamer streamer; streamer.setPacketFactory(&factory); transport.start(); ObClientManager client_manager; EXPECT_TRUE(OB_SUCCESS == client_manager.initialize(&transport, &streamer)); EXPECT_TRUE(OB_SUCCESS == stub.init(&buffer, &client_manager)); ObServer root_server; root_server.set_ipv4_addr(addr, MockRootServer::ROOT_SERVER_PORT); ObServer update_server; update_server.set_ipv4_addr(addr, MockUpdateServer::UPDATE_SERVER_PORT); ObServer merge_server; merge_server.set_ipv4_addr(addr, 10256); ObMergerRpcProxy proxy(3, timeout, update_server); root_server.set_ipv4_addr(addr, MockRootServer::ROOT_SERVER_PORT); ObMergerRootRpcProxy rpc(0, timeout, root_server); EXPECT_TRUE(OB_SUCCESS == rpc.init(&stub)); ObTabletLocationCache * location = new ObMergerTabletLocationCache; EXPECT_TRUE(NULL != location); EXPECT_TRUE(OB_SUCCESS == location->init(50000 * 5, 1000, 10000)); ObMergerLocationCacheProxy cache(merge_server, &rpc, location); // init tablet cache char temp[256] = ""; char temp_end[256] = ""; ObServer server; const uint64_t START_ROW = 100L; const uint64_t MAX_ROW = 300L; for (uint64_t i = START_ROW; i < MAX_ROW - 100; i += 100) { server.set_ipv4_addr(addr, MockChunkServer::CHUNK_SERVER_PORT); ObTabletLocation addr(i, server); ObTabletLocationList list; EXPECT_TRUE(OB_SUCCESS == list.add(addr)); EXPECT_TRUE(OB_SUCCESS == list.add(addr)); EXPECT_TRUE(OB_SUCCESS == list.add(addr)); snprintf(temp, 100, "row_%lu", i); snprintf(temp_end, 100, "row_%lu", i + 100); ObString start_key(100, strlen(temp), temp); ObString end_key(100, strlen(temp_end), temp_end); ObRange range; range.table_id_ = 234; range.start_key_ = start_key; range.end_key_ = end_key; list.set_timestamp(tbsys::CTimeUtil::getTime()); EXPECT_TRUE(OB_SUCCESS == location->set(range, list)); } ObMergerTabletLocation succ_addr; ObTabletLocationList list; EXPECT_TRUE(OB_SUCCESS == proxy.init(&stub, &cache, NULL)); // start root server MockRootServer root; tbsys::CThread root_server_thread; MockServerRunner test_root_server(root); root_server_thread.start(&test_root_server, NULL); // start chunk server MockChunkServer chunk; tbsys::CThread chunk_server_thread; MockServerRunner test_chunk_server(chunk); chunk_server_thread.start(&test_chunk_server, NULL); sleep(2); ObMergerAsyncRpcStub async; ObMergerLocationCacheProxy location_proxy(root_server, &rpc, location); ObGetRequestEvent request(&location_proxy, &async); ObGetParam get_param; ObCellInfo cell; ObString row_key; snprintf(temp, 100, "row_101"); row_key.assign(temp, strlen(temp)); cell.table_id_ = 234; cell.column_id_ = 111; cell.row_key_ = row_key; // add same cells EXPECT_TRUE(OB_SUCCESS == get_param.add_cell(cell)); EXPECT_TRUE(OB_SUCCESS == get_param.add_cell(cell)); EXPECT_TRUE(OB_SUCCESS == get_param.add_cell(cell)); EXPECT_TRUE(OB_SUCCESS == get_param.add_cell(cell)); EXPECT_TRUE(OB_SUCCESS == get_param.add_cell(cell)); EXPECT_TRUE(OB_SUCCESS == get_param.add_cell(cell)); EXPECT_TRUE(OB_SUCCESS == get_param.add_cell(cell)); // not init EXPECT_TRUE(OB_SUCCESS != request.set_request_param(get_param, timeout)); EXPECT_TRUE(OB_SUCCESS != request.wait(timeout)); EXPECT_TRUE(OB_SUCCESS == async.init(&buffer, &client_manager)); EXPECT_TRUE(OB_SUCCESS == request.init(100, 10)); EXPECT_TRUE(OB_SUCCESS == request.set_request_param(get_param, timeout)); EXPECT_TRUE(OB_SUCCESS == request.wait(timeout)); uint64_t count = 0; ObScannerIterator iter; ObCellInfo * cell_info = NULL; int ret = OB_SUCCESS; while (OB_ITER_END != (ret = request.next_cell())) { EXPECT_TRUE(OB_SUCCESS == ret); EXPECT_TRUE(OB_SUCCESS == request.get_cell(&cell_info)); EXPECT_TRUE(cell_info != NULL); printf("client:%.*s\n", cell_info->row_key_.length(), cell_info->row_key_.ptr()); ++count; } EXPECT_TRUE(OB_SUCCESS == request.reset()); root.stop(); chunk.stop(); sleep(3); transport.stop(); if (location != NULL) { delete location; location = NULL; } }