void dump_scanner(ObScanner &scanner) { ObScannerIterator iter; int total_count = 0; int row_count = 0; bool is_row_changed = false; for (iter = scanner.begin(); iter != scanner.end(); iter++) { ObCellInfo *cell_info; iter.get_cell(&cell_info, &is_row_changed); if (is_row_changed) { ++row_count; fprintf(stderr,"table_id:%lu, rowkey:\n", cell_info->table_id_); hex_dump(cell_info->row_key_.ptr(), cell_info->row_key_.length()); } fprintf(stderr, "%s\n", common::print_cellinfo(cell_info, "CLI_GET")); ++total_count; } fprintf(stderr, "row_count=%d,total_count=%d\n", row_count, total_count); }
// maybe not find a merge server int TaskFactory::get_table_tablet(const char * table_name, const uint64_t table_id, uint64_t & count) { int ret = OB_SUCCESS; ObScanParam scan_param; uint64_t max_len = 0; if (NULL == table_name) { TBSYS_LOG(ERROR, "check table name failed:table[%s], id[%lu]", table_name, table_id); ret = OB_ERROR; } else { ret = init_scan_param(table_name, table_id, max_len, scan_param); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init scan param failed:table[%s], ret[%d]", table_name, ret); } } if (OB_SUCCESS == ret) { // for the first table tablet ObRowkey row_key; ObObj temp_buffer[common::OB_MAX_ROWKEY_COLUMN_NUMBER]; row_key.assign(temp_buffer, common::OB_MAX_ROWKEY_COLUMN_NUMBER); ObGetParam param; ObScanner scanner; ObServer server; ObRowkey start_key; ObRowkey end_key; ObCellInfo * cell = NULL; ObScannerIterator iter; bool row_change = false; ObString name; name.assign(const_cast<char*>(table_name), (int32_t)strlen(table_name)); ObCellInfo temp_cell; temp_cell.table_id_ = table_id; temp_cell.column_id_ = 0; common::ModuleArena allocator; while ((OB_SUCCESS == ret) && (!row_key.is_max_row())) { param.reset(); param.set_is_read_consistency(false); temp_cell.row_key_ = row_key; ret = param.add_cell(temp_cell); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "add cell failed:ret[%d]", ret); break; } ret = rpc_->get(root_server_, timeout_, param, scanner); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get root table for tablet failed:table[%lu], ret[%d]", table_id, ret); break; } else { // skip the first row iter = scanner.begin(); ++iter; while ((iter != scanner.end()) && (OB_SUCCESS == (ret = iter.get_cell(&cell, &row_change))) && !row_change) { if (NULL == cell) { TBSYS_LOG(ERROR, "%s", "check cell failed"); ret = OB_INNER_STAT_ERROR; break; } ret = cell->row_key_.deep_copy(start_key, allocator); if (ret != OB_SUCCESS) { TBSYS_LOG(WARN, "deep copy the rowkey failed:ret[%d]", ret); } ++iter; } } // scanner iter end if (ret == OB_SUCCESS) { int64_t ip = 0; int64_t port = 0; int64_t version = 0; TabletLocation list; for (++iter; iter != scanner.end(); ++iter) { ret = iter.get_cell(&cell, &row_change); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret); break; } else if (row_change) // && (iter != last_iter)) { TaskInfo task; task.set_table_id(table_id); task.set_table_name(table_name); ret = init_new_task(name, start_key, end_key, scan_param, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret); break; } else { ret = insert_new_task(list, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret); break; } ++count; } list.clear(); start_key = end_key; } else { cell->row_key_.deep_copy(end_key, allocator); if ((cell->column_name_.compare("1_ms_port") == 0) || (cell->column_name_.compare("2_ms_port") == 0) || (cell->column_name_.compare("3_ms_port") == 0)) { ret = cell->value_.get_int(port); } else if ((cell->column_name_.compare("1_ipv4") == 0) || (cell->column_name_.compare("2_ipv4") == 0) || (cell->column_name_.compare("3_ipv4") == 0)) { ret = cell->value_.get_int(ip); } else if ((cell->column_name_.compare("1_tablet_version") == 0) || (cell->column_name_.compare("2_tablet_version") == 0) || (cell->column_name_.compare("3_tablet_version") == 0)) { ret = cell->value_.get_int(version); if (OB_SUCCESS == ret) { if (0 == port || port == 0) { TBSYS_LOG(WARN, "%s", "check port or ip failed"); } else { server.set_ipv4_addr(int32_t(ip), int32_t(port)); ObTabletLocation addr(version, server); if (OB_SUCCESS != (ret = list.add(addr))) { TBSYS_LOG(ERROR, "add addr failed:server[%d], port[%d], ret[%d]", server.get_ipv4(), server.get_port(), ret); } else { TBSYS_LOG(DEBUG, "add addr succ:server[%s], version:%ld", server.to_cstring(), version); } } ip = port = version = 0; } } if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret); break; } } } // for the last row if ((OB_SUCCESS == ret) && (start_key != end_key)) { TaskInfo task; task.set_table_id(table_id); task.set_table_name(table_name); ret = init_new_task(name, start_key, end_key, scan_param, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret); } else { ret = insert_new_task(list, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret); } ++count; } if (OB_SUCCESS == ret) { // copy all the end key objs to row_key for (int64_t i = 0; i < end_key.length(); ++i) { const_cast<ObObj *> (row_key.ptr())[i] = end_key.ptr()[i]; row_key.assign(const_cast<ObObj *> (row_key.ptr()), end_key.length()); } // modify last row key for next get root table if (!end_key.is_max_row()) { const_cast<ObObj *>(row_key.ptr())[end_key.length()].set_max_value(); row_key.assign(const_cast<ObObj *> (row_key.ptr()), end_key.length() + 1); } } } list.clear(); } } } return ret; }
TEST(TestObScanner, add_cell) { ObScanner fem; fem.set_is_req_fullfilled(true, 1011); fem.set_data_version(170); char buf[BUFSIZ]; int64_t pospos = 0; fem.serialize(buf, BUFSIZ, pospos); FILE *fd; fd = fopen("tmptest", "w+"); fwrite(buf, 1, pospos, fd); fclose(fd); ObScanner os; ObScanner ost; /*** added by wushi ***/ int64_t fullfilled_item = 0; int64_t tmp = 0; bool is_fullfilled = true; bool tmp_bool = false; /// initialize ASSERT_EQ(os.get_is_req_fullfilled(tmp_bool, fullfilled_item),0); EXPECT_FALSE(tmp_bool); ASSERT_EQ(fullfilled_item,0); /// set fullfilled is_fullfilled = true; fullfilled_item = 1000; os.set_is_req_fullfilled(is_fullfilled,fullfilled_item); ASSERT_EQ(os.get_is_req_fullfilled(tmp_bool, tmp),0); ASSERT_EQ(tmp_bool,is_fullfilled); ASSERT_EQ(tmp,fullfilled_item); /*** added by wushi ***/ ObCellInfo oci; oci.table_name_.assign((char*)"table1", 6); oci.row_key_.assign((char*)"row1", 4); oci.column_name_.assign((char*)"column1", 7); oci.value_.set_int(0xff); ost.set_mem_size_limit(20); ASSERT_EQ(OB_SIZE_OVERFLOW, ost.add_cell(oci)); os.set_mem_size_limit(1024 * 1024 * 2); ASSERT_EQ(OB_SUCCESS, os.add_cell(oci)); oci.column_name_.assign((char*)"column2", 7); oci.value_.set_int(0xfe); ASSERT_EQ(OB_SUCCESS, os.add_cell(oci)); ObScannerIterator iter = os.begin(); ObCellInfo tci; ASSERT_EQ(OB_SUCCESS, iter.get_cell(tci)); ASSERT_EQ(string("table1"), string(tci.table_name_.ptr(), tci.table_name_.length())); ASSERT_EQ(string("row1"), string(tci.row_key_.ptr(), tci.row_key_.length())); ASSERT_EQ(string("column1"), string(tci.column_name_.ptr(), tci.column_name_.length())); ASSERT_EQ(OB_SUCCESS, tci.value_.get_int(tmp)); ASSERT_EQ(0xff, tmp); iter++; ASSERT_EQ(OB_SUCCESS, iter.get_cell(tci)); ASSERT_EQ(string("table1"), string(tci.table_name_.ptr(), tci.table_name_.length())); ASSERT_EQ(string("row1"), string(tci.row_key_.ptr(), tci.row_key_.length())); ASSERT_EQ(string("column2"), string(tci.column_name_.ptr(), tci.column_name_.length())); ASSERT_EQ(OB_SUCCESS, tci.value_.get_int(tmp)); ASSERT_EQ(0xfe, tmp); iter++; ASSERT_EQ(true, os.end() == iter); //ObCellInfo oci; char row_key_buffer[1024]; oci.table_name_.assign((char*)"table1", 6); sprintf(row_key_buffer, "row1"); oci.row_key_.assign(row_key_buffer, 4); oci.column_name_.assign((char*)"column1", 7); oci.value_.set_int(0xff); ASSERT_EQ(OB_SUCCESS, os.add_cell(oci)); oci.table_name_.assign((char*)"table1", 6); sprintf(row_key_buffer, "row1"); oci.row_key_.assign(row_key_buffer, 4); oci.column_name_.assign((char*)"column2", 7); oci.value_.set_int(0xee); os.set_mem_size_limit(20); ASSERT_EQ(OB_SIZE_OVERFLOW, os.add_cell(oci)); os.set_mem_size_limit(1024 * 1024 * 2); ASSERT_EQ(OB_SUCCESS, os.add_cell(oci)); oci.table_name_.assign((char*)"table1", 6); sprintf(row_key_buffer, "row2"); oci.row_key_.assign(row_key_buffer, 4); oci.column_name_.assign((char*)"column1", 7); oci.value_.set_int(0xdd); ASSERT_EQ(OB_SUCCESS, os.add_cell(oci)); oci.table_name_.assign((char*)"table2", 6); sprintf(row_key_buffer, "row2"); oci.row_key_.assign(row_key_buffer, 4); oci.column_name_.assign((char*)"column1", 7); oci.value_.set_int(0xcc); ASSERT_EQ(OB_SUCCESS, os.add_cell(oci)); oci.table_name_.assign((char*)"table3", 6); sprintf(row_key_buffer, "row2"); oci.row_key_.assign(row_key_buffer, 4); oci.column_name_.assign((char*)"column1", 7); oci.value_.set_int(0xbb); ASSERT_EQ(OB_SUCCESS, os.add_cell(oci)); fprintf(stdout, "size=%ld\n", os.get_serialize_size()); char buffer[2048]; int64_t pos = 0; ASSERT_EQ(OB_SUCCESS, os.serialize(buffer, 1024, pos)); ASSERT_EQ(pos, os.get_serialize_size()); fd = fopen("./test.data.before_rollback", "w+"); fwrite(buffer, 1, 1024, fd); fclose(fd); for (iter = os.begin(); iter != os.end(); iter++) { ObCellInfo ci; ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci)); fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s]\n", ci.table_name_.length(), ci.table_name_.ptr(), ci.row_key_.length(), ci.row_key_.ptr(), ci.column_name_.length(), ci.column_name_.ptr()); } fprintf(stdout, "==============================\n"); while (OB_SUCCESS == os.next_cell()) { ObCellInfo *ci = NULL; bool is_row_changed; ASSERT_EQ(OB_SUCCESS, os.get_cell(&ci, &is_row_changed)); fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s] " "table_id=[%lu] column_id=[%lu] row_changed=[%d]\n", ci->table_name_.length(), ci->table_name_.ptr(), ci->row_key_.length(), ci->row_key_.ptr(), ci->column_name_.length(),ci->column_name_.ptr(), ci->table_id_, ci->column_id_, is_row_changed); } os.reset_iter(); fprintf(stdout, "==============================\n"); ASSERT_EQ(OB_SUCCESS, os.rollback()); fprintf(stdout, "size=%ld\n", os.get_serialize_size()); pos = 0; ASSERT_EQ(OB_SUCCESS, os.serialize(buffer, 1024, pos)); ASSERT_EQ(pos, os.get_serialize_size()); fd = fopen("./test.data.after_rollback", "w+"); fwrite(buffer, 1, 1024, fd); fclose(fd); for (iter = os.begin(); iter != os.end(); iter++) { ObCellInfo ci; ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci)); fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s]\n", ci.table_name_.length(), ci.table_name_.ptr(), ci.row_key_.length(), ci.row_key_.ptr(), ci.column_name_.length(), ci.column_name_.ptr()); ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci)); fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s]\n", ci.table_name_.length(), ci.table_name_.ptr(), ci.row_key_.length(), ci.row_key_.ptr(), ci.column_name_.length(), ci.column_name_.ptr()); } fprintf(stdout, "==============================\n"); while (OB_SUCCESS == os.next_cell()) { ObCellInfo *ci = NULL; bool is_row_changed; ASSERT_EQ(OB_SUCCESS, os.get_cell(&ci, &is_row_changed)); fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s] " "table_id=[%lu] column_id=[%lu] row_changed=[%d]\n", ci->table_name_.length(), ci->table_name_.ptr(), ci->row_key_.length(), ci->row_key_.ptr(), ci->column_name_.length(),ci->column_name_.ptr(), ci->table_id_, ci->column_id_, is_row_changed); ASSERT_EQ(OB_SUCCESS, os.get_cell(&ci, &is_row_changed)); fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s] " "table_id=[%lu] column_id=[%lu] row_changed=[%d]\n", ci->table_name_.length(), ci->table_name_.ptr(), ci->row_key_.length(), ci->row_key_.ptr(), ci->column_name_.length(),ci->column_name_.ptr(), ci->table_id_, ci->column_id_, is_row_changed); } os.reset_iter(); fprintf(stdout, "==============================\n"); pos = 0; ASSERT_EQ(OB_SUCCESS, os.deserialize(buffer, os.get_serialize_size(), pos)); ASSERT_EQ(pos, os.get_serialize_size()); for (iter = os.begin(); iter != os.end(); iter++) { ObCellInfo ci; ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci)); fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s]\n", ci.table_name_.length(), ci.table_name_.ptr(), ci.row_key_.length(), ci.row_key_.ptr(), ci.column_name_.length(), ci.column_name_.ptr()); ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci)); fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s]\n", ci.table_name_.length(), ci.table_name_.ptr(), ci.row_key_.length(), ci.row_key_.ptr(), ci.column_name_.length(), ci.column_name_.ptr()); } fprintf(stdout, "==============================\n"); while (OB_SUCCESS == os.next_cell()) { ObCellInfo *ci = NULL; bool is_row_changed; ASSERT_EQ(OB_SUCCESS, os.get_cell(&ci, &is_row_changed)); fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s] " "table_id=[%lu] column_id=[%lu] row_changed=[%d]\n", ci->table_name_.length(), ci->table_name_.ptr(), ci->row_key_.length(), ci->row_key_.ptr(), ci->column_name_.length(),ci->column_name_.ptr(), ci->table_id_, ci->column_id_, is_row_changed); } os.reset_iter(); /// set fullfilled // ASSERT_EQ(os.get_is_req_fullfilled(tmp_bool, tmp),0); // ASSERT_EQ(tmp_bool,is_fullfilled); // ASSERT_EQ(tmp,fullfilled_item); { ObRange range; range.border_flag_.set_inclusive_start(); range.start_key_.assign((char*)"11111", 5); range.end_key_.assign((char*)"22222", 5); ObRange range2; range2.border_flag_.set_inclusive_start(); range2.start_key_.assign((char*)"11111", 5); range2.end_key_.assign((char*)"22222", 5); int64_t pos = 0; int64_t data_len = 0; is_fullfilled = true; fullfilled_item = 1212; ObScanner os1; ObScanner os2; ASSERT_EQ(os1.set_is_req_fullfilled(is_fullfilled, fullfilled_item),0); os1.set_data_version(100); ASSERT_EQ(os1.set_range(range),0); range.border_flag_.set_inclusive_end(); char buf[1024]; ASSERT_EQ(os1.serialize(buf,sizeof(buf),pos),0); data_len = pos; pos = 0; ASSERT_EQ(os2.deserialize(buf, data_len, pos),0); memset(buf, 0x00, 1024); ASSERT_EQ(os2.get_is_req_fullfilled(tmp_bool, tmp),0); EXPECT_EQ(is_fullfilled, tmp_bool); ASSERT_EQ(fullfilled_item, tmp); ASSERT_EQ(os1.get_data_version(),100); ObRange range3; ASSERT_EQ(OB_SUCCESS, os2.get_range(range3)); ASSERT_EQ(range2.border_flag_.get_data(), range3.border_flag_.get_data()); ASSERT_EQ(string(range3.start_key_.ptr(), range3.start_key_.length()), string(range2.start_key_.ptr(), range2.start_key_.length())); ASSERT_EQ(string(range3.end_key_.ptr(), range3.end_key_.length()), string(range2.end_key_.ptr(), range2.end_key_.length())); const int times = 1000000; ObScanner os3; ObCellInfo cinull; cinull.table_name_.assign((char*)"hello", 5); cinull.value_.set_ext(ObActionFlag::OP_INSERT); ASSERT_EQ(OB_ERROR, os3.add_cell(cinull)); cinull.value_.set_ext(ObActionFlag::OP_DEL_ROW); ASSERT_EQ(OB_SUCCESS, os3.add_cell(cinull)); cinull.value_.set_ext(ObActionFlag::OP_ROW_DOES_NOT_EXIST); ASSERT_EQ(OB_SUCCESS, os3.add_cell(cinull)); int i = 0; for (; i < times; i++) { if (OB_SUCCESS != os3.add_cell(oci)) { break; } } const int64_t buf_len = 1 << 22; char* buff = (char*)ob_malloc(buf_len); int64_t ppos = 0; int64_t len = os3.get_serialize_size(); ASSERT_EQ(OB_BUF_NOT_ENOUGH, os3.serialize(buff, len - 4, ppos)); ASSERT_EQ(OB_SUCCESS, os3.serialize(buff, buf_len, ppos)); int64_t npos = 0; ObScanner os4; ASSERT_EQ(OB_ERROR, os4.deserialize(buff, 10, npos)); ASSERT_EQ(OB_SUCCESS, os4.deserialize(buff, ppos, npos)); ASSERT_EQ(npos, ppos); int64_t otmp; oci.value_.get_int(otmp); ObCellInfo *oo; int t = 0; ASSERT_EQ(OB_SUCCESS, os4.next_cell()); ASSERT_EQ(OB_SUCCESS, os4.get_cell(&oo)); ASSERT_EQ(0, oo->row_key_.length()); ASSERT_EQ(string(cinull.table_name_.ptr(), cinull.table_name_.length()), string(oo->table_name_.ptr(), oo->table_name_.length())); ASSERT_EQ(OB_SUCCESS, oo->value_.get_ext(tmp)); ASSERT_EQ(+ObActionFlag::OP_DEL_ROW, tmp); ASSERT_EQ(OB_SUCCESS, os4.next_cell()); ASSERT_EQ(OB_SUCCESS, os4.get_cell(&oo)); ASSERT_EQ(string(cinull.table_name_.ptr(), cinull.table_name_.length()), string(oo->table_name_.ptr(), oo->table_name_.length())); ASSERT_EQ(OB_SUCCESS, oo->value_.get_ext(tmp)); ASSERT_EQ(+ObActionFlag::OP_ROW_DOES_NOT_EXIST, tmp); while (OB_SUCCESS == os4.next_cell()) { ASSERT_EQ(OB_SUCCESS, os4.get_cell(&oo)); t++; ASSERT_EQ(string(oci.table_name_.ptr(), oci.table_name_.length()), string(oo->table_name_.ptr(), oo->table_name_.length())); ASSERT_EQ(string(oci.row_key_.ptr(), oci.row_key_.length()), string(oo->row_key_.ptr(), oo->row_key_.length())); ASSERT_EQ(string(oci.column_name_.ptr(), oci.column_name_.length()), string(oo->column_name_.ptr(), oo->column_name_.length())); ASSERT_EQ(OB_SUCCESS, oo->value_.get_int(tmp)); ASSERT_EQ(otmp, tmp); } ASSERT_EQ(t, i); ob_free(buff); } ObCellInfo st; st.table_name_.assign((char*)"table1", 6); st.row_key_.assign((char*)"row1", 4); st.column_name_.assign((char*)"column1", 7); st.value_.set_int(0xff); ObScanner ss; ss.add_cell(st); st.table_name_.assign((char*)"table2", 6); st.row_key_.assign((char*)"row1", 4); st.column_name_.assign((char*)"column1", 7); st.value_.set_int(0xff); ss.add_cell(st); ObCellInfo *sto; bool is_row_changed; ASSERT_EQ(OB_SUCCESS, ss.next_cell()); ASSERT_EQ(OB_SUCCESS, ss.get_cell(&sto, &is_row_changed)); ASSERT_EQ(true, is_row_changed); ASSERT_EQ(OB_SUCCESS, ss.next_cell()); ASSERT_EQ(OB_SUCCESS, ss.get_cell(&sto, &is_row_changed)); ASSERT_EQ(true, is_row_changed); ASSERT_EQ(OB_ITER_END, ss.next_cell()); }
// waring:all return cell in a row must be same as root table's columns, // and the second row is this row allocated chunkserver list int ObMergerRootRpcProxy::scan_root_table(ObMergerTabletLocationCache * cache, const uint64_t table_id, const ObString & row_key, const ObServer & addr, ObMergerTabletLocationList & location) { assert(location.get_buffer() != NULL); int ret = OB_SUCCESS; bool find_right_tablet = false; ObScanner scanner; // root table id = 0 ret = rpc_stub_->fetch_tablet_location(rpc_timeout_, root_server_, 0, table_id, row_key, scanner); if (ret != OB_SUCCESS) { TBSYS_LOG(WARN, "fetch tablet location failed:table_id[%lu], length[%d], ret[%d]", table_id, row_key.length(), ret); hex_dump(row_key.ptr(), row_key.length(), true); } else { ObRange range; range.border_flag_.unset_inclusive_start(); range.border_flag_.set_inclusive_end(); ObString start_key; ObString end_key; ObServer server; ObCellInfo * cell = NULL; bool row_change = false; ObScannerIterator iter = scanner.begin(); TBSYS_LOG(DEBUG, "%s", "parse scanner result for get some tablet locations"); // all return cell in a row must be same as root table's columns ++iter; while ((iter != scanner.end()) && (OB_SUCCESS == (ret = iter.get_cell(&cell, &row_change))) && !row_change) { if (NULL == cell) { ret = OB_INNER_STAT_ERROR; break; } start_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); ++iter; } if (ret == OB_SUCCESS) { int64_t ip = 0; int64_t port = 0; bool second_row = true; // next cell ObMergerTabletLocationList list; for (++iter; iter != scanner.end(); ++iter) { ret = iter.get_cell(&cell, &row_change); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret); break; } else if (row_change) // && (iter != last_iter)) { range.table_id_ = table_id; if (NULL == start_key.ptr()) { range.border_flag_.set_min_value(); } else { range.border_flag_.unset_min_value(); range.start_key_ = start_key; } range.border_flag_.unset_max_value(); range.end_key_ = end_key; start_key = end_key; end_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); list.set_timestamp(tbsys::CTimeUtil::getTime()); list.sort(addr); // not deep copy the range list.set_tablet_range(range); // the second row is this row allocated chunkserver list if (second_row) { second_row = false; if ((row_key <= range.end_key_) && (row_key > range.start_key_)) { find_right_tablet = true; location = list; assert(location.get_buffer() != NULL); location.set_tablet_range(range); } else { ret = OB_DATA_NOT_SERVE; TBSYS_LOG(ERROR, "check range not include this key:ret[%d]", ret); hex_dump(row_key.ptr(), row_key.length()); range.hex_dump(); break; } } // add to cache if (OB_SUCCESS != cache->set(range, list)) { TBSYS_LOG(WARN, "%s", "add the range to cache failed"); } list.clear(); } else { end_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); if ((cell->column_name_.compare("1_port") == 0) || (cell->column_name_.compare("2_port") == 0) || (cell->column_name_.compare("3_port") == 0)) { ret = cell->value_.get_int(port); } else if ((cell->column_name_.compare("1_ipv4") == 0) || (cell->column_name_.compare("2_ipv4") == 0) || (cell->column_name_.compare("3_ipv4") == 0)) { ret = cell->value_.get_int(ip); if (OB_SUCCESS == ret) { if (port == 0) { TBSYS_LOG(WARN, "check port failed:ip[%ld], port[%ld]", ip, port); } server.set_ipv4_addr(static_cast<int32_t>(ip), static_cast<int32_t>(port)); ObTabletLocation addr(0, server); if (OB_SUCCESS != (ret = list.add(addr))) { TBSYS_LOG(ERROR, "add addr failed:ip[%ld], port[%ld], ret[%d]", ip, port, ret); break; } else { TBSYS_LOG(DEBUG, "add addr succ:ip[%ld], port[%ld]", ip, port); } ip = port = 0; } } if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret); break; } } } // for the last row if ((OB_SUCCESS == ret) && (start_key != end_key)) { range.table_id_ = table_id; if (NULL == start_key.ptr()) { range.border_flag_.set_min_value(); } else { range.border_flag_.unset_min_value(); range.start_key_ = start_key; } range.border_flag_.unset_max_value(); range.end_key_ = end_key; list.set_timestamp(tbsys::CTimeUtil::getTime()); // not deep copy the range list.set_tablet_range(range); list.sort(addr); // double check add all range->locationlist to cache if ((row_key <= range.end_key_) && (row_key > range.start_key_)) { find_right_tablet = true; location = list; // deep copy range assert(location.get_buffer() != NULL); location.set_tablet_range(range); } else if (second_row) { range.hex_dump(); ret = OB_DATA_NOT_SERVE; TBSYS_LOG(ERROR, "check range not include this key:ret[%d]", ret); } // add to list to cache if (OB_SUCCESS != cache->set(range, list)) { range.hex_dump(); TBSYS_LOG(WARN, "%s", "add the range to cache failed"); } } } else { TBSYS_LOG(ERROR, "check get first row cell failed:ret[%d]", ret); } } if ((OB_SUCCESS == ret) && (0 == location.size())) { TBSYS_LOG(ERROR, "check get location size failed:table_id[%ld], find[%d], count[%ld]", table_id, find_right_tablet, location.size()); hex_dump(row_key.ptr(), row_key.length()); ret = OB_INNER_STAT_ERROR; } ms_get_counter_set().inc(ObMergerCounterIds::C_SCAN_ROOT_TABLE); return ret; }
int main(int argc, char** argv) { const int32_t MOCK_SERVER_LISTEN_PORT = 8888; MockClient client; ObServer dst_host; const char* dst_addr = "localhost"; dst_host.set_ipv4_addr(dst_addr, MOCK_SERVER_LISTEN_PORT); client.init(dst_host); // init cell info static const int64_t ROW_NUM = 1; static const int64_t COL_NUM = 10; ObCellInfo cell_infos[ROW_NUM][COL_NUM]; char row_key_strs[ROW_NUM][50]; uint64_t table_id = 10; // init cell infos for (int64_t i = 0; i < ROW_NUM; ++i) { sprintf(row_key_strs[i], "row_key_%08ld", i); for (int64_t j = 0; j < COL_NUM; ++j) { cell_infos[i][j].table_id_ = table_id; cell_infos[i][j].row_key_.assign(row_key_strs[i], strlen(row_key_strs[i])); //cell_infos[i][j].op_info_.set_sem_ob(); cell_infos[i][j].column_id_ = j + 1; cell_infos[i][j].value_.set_int(1000 + i * COL_NUM + j); } } // scan memtable ObScanner scanner; ObScanParam scan_param; //ObOperateInfo op_info; ObString table_name; ObRange range; const int64_t version = 999; range.start_key_ = cell_infos[0][0].row_key_; range.end_key_ = cell_infos[ROW_NUM - 1][0].row_key_; range.border_flag_.set_inclusive_start(); range.border_flag_.set_inclusive_end(); scan_param.set(table_id, table_name, range); //scan_param.set_timestamp(version); //scan_param.set_is_read_frozen_only(false); ObVersionRange version_range; version_range.start_version_ = version; version_range.end_version_ = version; version_range.border_flag_.set_inclusive_start(); version_range.border_flag_.set_inclusive_end(); scan_param.set_version_range(version_range); for (int64_t i = 0; i < COL_NUM; ++i) { scan_param.add_column(cell_infos[0][i].column_id_); } int64_t count = 0; const int64_t timeout = 10 * 1000L; int err = client.ups_scan(scan_param, scanner, timeout); fprintf(stderr, "ups_scan err=%d\n", err); // check result count = 0; ObScannerIterator iter; for (iter = scanner.begin(); iter != scanner.end(); iter++) { ObCellInfo ci; ObCellInfo expected = cell_infos[count / COL_NUM][count % COL_NUM]; EXPECT_EQ(OB_SUCCESS, iter.get_cell(ci)); check_cell(expected, ci); ++count; } EXPECT_EQ(ROW_NUM * COL_NUM, count); fprintf(stderr, "cell_num=%ld\n", count); client.destroy(); return 0; }
int ObRootServerRpcStub::get_tablet_info(const common::ObSchemaManagerV2& schema, const uint64_t table_id, const ObRange& range, ObTabletLocation location [],int32_t& size) { int ret = OB_SUCCESS; int32_t index = 0; const ObChunkServer& chunk_server = ObChunkServerMain::get_instance()->get_chunk_server(); const int64_t timeout = chunk_server.get_param().get_network_time_out(); if (OB_INVALID_ID == table_id || size <= 0) { TBSYS_LOG(ERROR,"invalid table id"); ret = OB_ERROR; } ObScanParam param; ObScanner scanner; ObString table_name; const ObTableSchema *table = schema.get_table_schema(table_id); if (NULL == table) { TBSYS_LOG(ERROR,"can not find table %lu",table_id); ret = OB_ERROR; } else { table_name.assign_ptr(const_cast<char *>(table->get_table_name()),strlen(table->get_table_name())); } if (OB_SUCCESS == ret) { param.set(OB_INVALID_ID,table_name,range); //use table name param.set_is_read_consistency(false); } if ((OB_SUCCESS == ret) && ((ret = scan(root_server_,timeout,param,scanner)) != OB_SUCCESS) ) { TBSYS_LOG(ERROR,"get tablet from rootserver failed:[%d]",ret); } ObServer server; char tmp_buf[32]; ObString start_key; ObString end_key; ObCellInfo * cell = NULL; ObScannerIterator iter; bool row_change = false; if (OB_SUCCESS == ret) { int64_t ip = 0; int64_t port = 0; int64_t version = 0; iter = scanner.begin(); ret = iter.get_cell(&cell, &row_change); row_change = false; while((OB_SUCCESS == ret) && index < size) { if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret); } else if (row_change) { TBSYS_LOG(DEBUG,"row changed,ignore"); hex_dump(cell->row_key_.ptr(),cell->row_key_.length(),false,TBSYS_LOG_LEVEL_INFO); break; //just get one row } else if (cell != NULL) { end_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); if ((cell->column_name_.compare("1_port") == 0) || (cell->column_name_.compare("2_port") == 0) || (cell->column_name_.compare("3_port") == 0)) { ret = cell->value_.get_int(port); TBSYS_LOG(DEBUG,"port is %ld",port); } else if ((cell->column_name_.compare("1_ipv4") == 0) || (cell->column_name_.compare("2_ipv4") == 0) || (cell->column_name_.compare("3_ipv4") == 0)) { ret = cell->value_.get_int(ip); TBSYS_LOG(DEBUG,"ip is %ld",ip); } else if (cell->column_name_.compare("1_tablet_version") == 0 || cell->column_name_.compare("2_tablet_version") == 0 || cell->column_name_.compare("3_tablet_version") == 0) { ret = cell->value_.get_int(version); hex_dump(cell->row_key_.ptr(),cell->row_key_.length(),false,TBSYS_LOG_LEVEL_INFO); TBSYS_LOG(DEBUG,"tablet_version is %d",version); } if (OB_SUCCESS == ret) { if (0 != port && 0 != ip && 0 != version) { server.set_ipv4_addr(ip, port); server.to_string(tmp_buf,sizeof(tmp_buf)); TBSYS_LOG(INFO,"add tablet s:%s,%ld",tmp_buf,version); ObTabletLocation addr(version, server); location[index++] = addr; ip = port = version = 0; } } else { TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret); } if (++iter == scanner.end()) break; ret = iter.get_cell(&cell, &row_change); } else { //impossible } } } if (OB_SUCCESS == ret) { size = index; TBSYS_LOG(INFO,"get %d tablets from rootserver",size); } return ret; }
// maybe not find a merge server int TaskFactory::get_table_tablet(const char * table_name, const uint64_t table_id, uint64_t & count) { int ret = OB_SUCCESS; ObScanParam scan_param; uint64_t max_len = 0; if (NULL == table_name) { TBSYS_LOG(ERROR, "check table name failed:table[%s], id[%lu]", table_name, table_id); ret = OB_ERROR; } else { ret = init_scan_param(table_name, table_id, max_len, scan_param); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init scan param failed:table[%s], ret[%d]", table_name, ret); } } if (OB_SUCCESS == ret) { TaskInfo task; task.set_table_id(table_id); task.set_table_name(table_name); // for the first table tablet ObString row_key; char temp_buffer[1]; memset(temp_buffer, 0, sizeof(temp_buffer)); row_key.assign(temp_buffer, sizeof(temp_buffer)); ObGetParam param; ObScanner scanner; ObServer server; ObString start_key; ObString end_key; ObCellInfo * cell = NULL; ObScannerIterator iter; bool row_change = false; ObString name; name.assign(const_cast<char*>(table_name), strlen(table_name)); ObCellInfo temp_cell; temp_cell.table_id_ = table_id; temp_cell.column_id_ = 0; const uint64_t MAX_LEN = 1024; char last_tablet_rowkey[MAX_LEN] = ""; const int32_t MAX_SERVER_ADDR_SIZE = 128; char server_addr[MAX_SERVER_ADDR_SIZE]; while ((OB_SUCCESS == ret) && (!is_max_rowkey(max_len, row_key.ptr(), row_key.length() - 1))) { param.reset(); param.set_is_read_consistency(false); temp_cell.row_key_ = row_key; ret = param.add_cell(temp_cell); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "add cell failed:ret[%d]", ret); break; } ret = rpc_->get(root_server_, timeout_, param, scanner); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get root table for tablet failed:table[%lu], ret[%d]", table_id, ret); break; } else { // skip the first row iter = scanner.begin(); ++iter; while ((iter != scanner.end()) && (OB_SUCCESS == (ret = iter.get_cell(&cell, &row_change))) && !row_change) { if (NULL == cell) { TBSYS_LOG(ERROR, "%s", "check cell failed"); ret = OB_INNER_STAT_ERROR; break; } start_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); ++iter; } } if (ret == OB_SUCCESS) { int64_t ip = 0; int64_t port = 0; int64_t version = 0; TabletLocation list; for (++iter; iter != scanner.end(); ++iter) { ret = iter.get_cell(&cell, &row_change); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret); break; } else if (row_change) // && (iter != last_iter)) { ret = init_new_task(name, start_key, end_key, scan_param, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret); break; } else { ret = insert_new_task(list, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret); break; } ++count; } list.clear(); start_key = end_key; end_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); } else { end_key.assign(cell->row_key_.ptr(), cell->row_key_.length()); if ((cell->column_name_.compare("1_ms_port") == 0) || (cell->column_name_.compare("2_ms_port") == 0) || (cell->column_name_.compare("3_ms_port") == 0)) { ret = cell->value_.get_int(port); } else if ((cell->column_name_.compare("1_ipv4") == 0) || (cell->column_name_.compare("2_ipv4") == 0) || (cell->column_name_.compare("3_ipv4") == 0)) { ret = cell->value_.get_int(ip); } else if ((cell->column_name_.compare("1_tablet_version") == 0) || (cell->column_name_.compare("2_tablet_version") == 0) || (cell->column_name_.compare("3_tablet_version") == 0)) { ret = cell->value_.get_int(version); if (OB_SUCCESS == ret) { if (0 == port || port == 0) { TBSYS_LOG(WARN, "%s", "check port or ip failed"); } else { server.set_ipv4_addr(ip, port); ObTabletLocation addr(version, server); if (OB_SUCCESS != (ret = list.add(addr))) { TBSYS_LOG(ERROR, "add addr failed:server[%d], port[%d], ret[%d]", server.get_ipv4(), server.get_port(), ret); } else { server.to_string(server_addr, MAX_SERVER_ADDR_SIZE); TBSYS_LOG(DEBUG, "add addr succ:server[%s], version:%ld", server_addr, version); } } ip = port = version = 0; } } if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret); break; } } } // for the last row if ((OB_SUCCESS == ret) && (start_key != end_key)) { ret = init_new_task(name, start_key, end_key, scan_param, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret); } else { ret = insert_new_task(list, task); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret); } ++count; } if (OB_SUCCESS == ret) { // modify last row key for next get root table uint64_t len = end_key.length(); if (MAX_LEN > len + 1) { memcpy(last_tablet_rowkey, end_key.ptr(), len); last_tablet_rowkey[len] = 0; row_key.assign(last_tablet_rowkey, len + 1); } else { TBSYS_LOG(ERROR, "check end key failed:len[%lu]", len); } } } list.clear(); } } } return ret; }
int TaskOutputFile::convert_result(const ObScanner &result, ObDataBuffer &buffer) const { bool first_line = true; int ret = OB_SUCCESS; int len = 0; ObCellInfo *cell = NULL; ObScannerIterator itr = result.begin(); TableConf::ColumnIterator col_itr; if (conf_ != NULL) { col_itr= conf_->column_begin(); } else { TBSYS_LOG(WARN, "please specify table conf"); return OB_ERROR; } while (itr != result.end()) { bool row_changed = false; ret = itr.get_cell(&cell, &row_changed); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "get cell failed ret[%d]", ret); break; } //append end of line for the last rowkey if (first_line) { row_changed = false; first_line = false; } else { if (row_changed == true) { if (ret == OB_SUCCESS) { ret = append_end_rec(buffer, rec_delima_); col_itr = conf_->column_begin(); } } else { ret = append_delima(buffer, delima_); } if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "append delima or end or record failed"); break; } } if (split_rowkey_ && conf_) { ret = handle_rk_null(buffer, cell, col_itr, false); } if (col_itr == conf_->column_end()) { ret = OB_ERROR; TBSYS_LOG(ERROR, "task_worker conf is diff with master conf, please check it"); break; } if (cell == NULL || cell->column_name_.compare(*col_itr)) { ret = OB_ERROR; TBSYS_LOG(ERROR, "column name mismatch ,cell column name = %s, expected name = %s", std:: string(cell->column_name_.ptr(), cell->column_name_.length()).c_str(), *col_itr); break; } if (ret == OB_SUCCESS) /* normal column */ { if (conf_->is_revise_column(cell->column_name_)) { ret = conf_->revise_column(*cell, buffer); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "revise column failed, column_id=%ld", cell->column_id_); break; } } else { len = serialize_cell(cell, buffer); if (len < 0) { TBSYS_LOG(ERROR, "serialize cell failed"); ret = OB_ERROR; break; } } } if (col_itr != conf_->column_end()) { col_itr++; /* move to next column */ } if (ret == OB_SUCCESS && conf_ != NULL) { ret = handle_rk_null(buffer, cell, col_itr, true); if (ret != OB_SUCCESS) { break; } } itr++; } if (ret == OB_SUCCESS) { if (col_itr != conf_->column_end()) { TBSYS_LOG(ERROR, "error happens column does not all proceeded"); } if (ret == OB_SUCCESS) { ret = append_end_rec(buffer, rec_delima_); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "last row, append end rec failed"); } } } return ret; }
int MockChunkServer::handle_scan_table(ObPacket * ob_packet) { int ret = OB_SUCCESS; ObDataBuffer* data = ob_packet->get_buffer(); if (NULL == data) { ret = OB_ERROR; } ObScanParam param; if (OB_SUCCESS == ret) { ret = param.deserialize(data->get_data(), data->get_capacity(), data->get_position()); if (ret != OB_SUCCESS) { TBSYS_LOG(ERROR, "%s", "check param failed"); } } tbnet::Connection* connection = ob_packet->get_connection(); ThreadSpecificBuffer::Buffer* thread_buffer = response_packet_buffer_.get_buffer(); if (NULL == thread_buffer) { ret = OB_ERROR; } else { thread_buffer->reset(); ObDataBuffer out_buffer(thread_buffer->current(), thread_buffer->remain()); ObResultCode result_msg; result_msg.result_code_ = ret; ret = result_msg.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position()); // fake cell ObCellInfo cell; ObScanner scanner; ObRowkey row_key; ObString column_name; char temp[256] = ""; #if 1 cell.table_id_ = 101; for (uint64_t i = 0; i < 10; ++i) { snprintf(temp, 256, "chunk_%lu_scan_row_key:%lu", i, i); row_key = make_rowkey(temp, &allocator_); cell.row_key_ = row_key; cell.column_id_ = i + 1; cell.value_.set_int(2234 + i); scanner.add_cell(cell); } #else cell.table_id_ = 123; for (uint64_t i = 100; i < 200; ++i) { snprintf(temp, 256, "row_%lu", i); row_key = make_rowkey(temp, &allocator_); cell.row_key_ = row_key; cell.column_id_ = 101; cell.value_.set_int(2234 + i); scanner.add_cell(cell); } #endif /* begin add by xiaochu */ //Scanner Range must be set other wise the ms client will report error ObNewRange range; /* /// This will cause rowkey mismatch //char *start= "chunk_0_scan_row_key:0"; //char *end = "chunk_9_scan_row_key:9"; */ char *start= (char*)"row_100"; char *end = (char*)"row_200"; range.start_key_ = make_rowkey(start, &allocator_);; range.end_key_ = make_rowkey(end, &allocator_); range.table_id_ = 103; scanner.set_range(range); scanner.set_is_req_fullfilled(true, 10); /* end add by xiaochu */ int32_t channel_id = ob_packet->getChannelId(); ret = scanner.serialize(out_buffer.get_data(), out_buffer.get_capacity(), out_buffer.get_position()); ObScannerIterator iter; for (iter = scanner.begin(); iter != scanner.end(); ++iter) { iter.get_cell(cell); printf("server_temp:%s\n", to_cstring(cell.row_key_)); }// ret = send_response(OB_SCAN_RESPONSE, 1, out_buffer, connection, channel_id); } TBSYS_LOG(INFO, "handle scan table result:ret[%d]", ret); return ret; }
TEST_F(TestRpcStub, test_scan_servers) { ObMergerRpcStub stub; ThreadSpecificBuffer buffer; ObPacketFactory factory; tbnet::Transport transport; tbnet::DefaultPacketStreamer streamer; streamer.setPacketFactory(&factory); transport.start(); ObClientManager client_manager; EXPECT_TRUE(OB_SUCCESS == client_manager.initialize(&transport, &streamer)); EXPECT_TRUE(OB_SUCCESS == stub.init(&buffer, &client_manager)); ObMergerTabletLocationList list; ObServer chunk_server; chunk_server.set_ipv4_addr(addr, MockChunkServer::CHUNK_SERVER_PORT); ObTabletLocation addr; //addr.tablet_id_ = 100; addr.chunkserver_ = chunk_server; list.add(addr); list.add(addr); list.add(addr); // start root server MockChunkServer server; MockServerRunner test_chunk_server(server); tbsys::CThread chunk_server_thread; chunk_server_thread.start(&test_chunk_server, NULL); sleep(2); ObScanParam param; ObCellInfo cell; ObRange range; ObString name; ObScanner scanner; param.set(1, name, range); EXPECT_TRUE(OB_SUCCESS == stub.scan(timeout, chunk_server, param, scanner)); EXPECT_TRUE(!scanner.is_empty()); uint64_t count = 0; ObScannerIterator iter; for (iter = scanner.begin(); iter != scanner.end(); ++iter) { EXPECT_TRUE(OB_SUCCESS == iter.get_cell(cell)); //EXPECT_TRUE(cell.column_id_ == count); printf("client:%.*s\n", cell.row_key_.length(), cell.row_key_.ptr()); ++count; } // return 10 cells EXPECT_TRUE(count == 10); ObMergerTabletLocation succ_addr; bool update = false; EXPECT_TRUE(OB_SUCCESS == stub.scan(timeout, list, param, succ_addr, scanner, update)); EXPECT_TRUE(!scanner.is_empty()); EXPECT_TRUE(update == false); for (iter = scanner.begin(); iter != scanner.end(); ++iter) { EXPECT_TRUE(OB_SUCCESS == iter.get_cell(cell)); printf("client:%.*s\n", cell.row_key_.length(), cell.row_key_.ptr()); } transport.stop(); server.stop(); sleep(10); }