void TaskManager::dump_tablets(const char *file)
{
FILE * f = fopen(file, "w");
char *key_buf = NULL;
if (NULL == f)
{
TBSYS_LOG(ERROR, "can't write tablets list file, %s", file);
return;
}
key_buf = new (std::nothrow) char[2 * OB_MAX_ROW_KEY_LENGTH];
if (NULL == key_buf)
{
TBSYS_LOG(ERROR, "can't allocate memory");
fclose(f);
return;
}
map<uint64_t, TaskInfo>::iterator itr = wait_queue_.begin();
while (itr != wait_queue_.end())
{
TabletLocation loc = itr->second.get_location();
fprintf(f, "TaskId:%ld, ReplicaSize=%ld, TableId:%ld\n",
itr->first, loc.size(), itr->second.get_table_id());
const ObRange *range = itr->second.get_param().get_range();
int len = hex_to_str(range->start_key_.ptr(), range->start_key_.length(),
key_buf, 2 * OB_MAX_ROW_KEY_LENGTH);
key_buf[2 * len] = 0;
fprintf(f, "S:%s\n", key_buf);
len = hex_to_str(range->end_key_.ptr(), range->end_key_.length(),
key_buf, 2 * OB_MAX_ROW_KEY_LENGTH);
key_buf[2 * len] = 0;
fprintf(f, "E:%s\n", key_buf);
for (int i = 0;i < loc.size();i++)
{
loc[i].chunkserver_.to_string(key_buf, 2 * OB_MAX_ROW_KEY_LENGTH);
fprintf(f, "MergeServer=%s\n", key_buf);
}
fprintf(f, "\n");
itr++;
}
fclose(f);
delete [] key_buf;
}
// add a new task
int TaskManager::insert_task(const TabletLocation & location, TaskInfo & task)
{
int ret = OB_SUCCESS;
task.set_location(location);
task.set_token(task_token_);
int64_t timestamp = tbsys::CTimeUtil::getTime();
task.set_timestamp(timestamp);
std::map<ObServer, int64_t>::const_iterator it;
tbsys::CThreadGuard lock(&lock_);
task.set_id(++task_id_alloc_);
++total_task_count_;
wait_queue_.insert(pair<uint64_t, TaskInfo>(task.get_id(), task));
for (int64_t i = 0; i < location.size(); ++i)
{
// TODO server counter for select task
it = server_manager_.find(location[i].chunkserver_);
if (it != server_manager_.end())
{
server_manager_[location[i].chunkserver_] = it->second + 1;
}
else
{
server_manager_.insert(pair<ObServer, int64_t>(location[i].chunkserver_, 1));
}
}
TBSYS_LOG(DEBUG, "insert task succ:id[%lu], count[%lu]", task_id_alloc_, total_task_count_);
return ret;
}
int TaskFactory::insert_new_task(const TabletLocation & list, TaskInfo & task)
{
int ret = OB_SUCCESS;
// alarm instantly
if (0 == list.size())
{
TBSYS_LOG(ERROR, "check task server list count failed:task[%lu], count[%lu]",
task.get_id(), list.size());
}
// remain do insert task
ret = task_manager_->insert_task(list, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "insert a new task failed:ret[%d]", ret);
}
return ret;
}
int TaskFactory::insert_new_task(const TabletLocation & list, TaskInfo & task)
{
int ret = OB_SUCCESS;
// alarm instantly
if (0 == list.size())
{
TBSYS_LOG(ERROR, "check task server list count failed:task[%lu], count[%lu]",
task.get_id(), list.size());
}
// remain do insert task
ret = task_manager_->insert_task(list, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "insert a new task failed:ret[%d]", ret);
}
else
{
TBSYS_LOG(TRACE, "insert a new task succ:task[%lu], table_name[%.*s], tablet[%s]",
task.get_id(), task.get_param().get_table_name().length(), task.get_param().get_table_name().ptr(),
to_cstring(*task.get_param().get_range()));
}
return ret;
}
// add a new task
int TaskManager::insert_task(const TabletLocation & location, TaskInfo & task)
{
int ret = OB_SUCCESS;
task.set_location(location);
task.set_token(task_token_);
int64_t timestamp = tbsys::CTimeUtil::getTime();
task.set_timestamp(timestamp);
std::map<ObServer, int64_t>::const_iterator it;
tbsys::CThreadGuard lock(&lock_);
task.set_id(++task_id_alloc_);
++total_task_count_;
wait_queue_.insert(pair<uint64_t, TaskInfo>(task.get_id(), task));
for (int64_t i = 0; i < location.size(); ++i)
{
// TODO server counter for select task
it = server_manager_.find(location[i].chunkserver_);
if (it != server_manager_.end())
{
server_manager_[location[i].chunkserver_] = it->second + 1;
}
else
{
server_manager_.insert(pair<ObServer, int64_t>(location[i].chunkserver_, 1));
}
}
TBSYS_LOG(DEBUG, "insert task succ:id[%lu], table_name[%.*s], range[%s], count[%lu]", task_id_alloc_,
task.get_param().get_table_name().length(), task.get_param().get_table_name().ptr(),
to_cstring(*task.get_param().get_range()), total_task_count_);
#if false
std::map<uint64_t, TaskInfo>::iterator temp_it;
for (temp_it = wait_queue_.begin(); temp_it != wait_queue_.end(); ++temp_it)
{
TBSYS_LOG(TRACE, "dump task range:task[%ld], range[%p:%s], key_obj[%p:%p]", temp_it->first,
temp_it->second.get_param().get_range(), to_cstring(*temp_it->second.get_param().get_range()),
temp_it->second.get_param().get_range()->start_key_.ptr(), temp_it->second.get_param().get_range()->end_key_.ptr());
}
#endif
return ret;
}
// maybe not find a merge server
int TaskFactory::get_table_tablet(const char * table_name, const uint64_t table_id, uint64_t & count)
{
int ret = OB_SUCCESS;
ObScanParam scan_param;
uint64_t max_len = 0;
if (NULL == table_name)
{
TBSYS_LOG(ERROR, "check table name failed:table[%s], id[%lu]", table_name, table_id);
ret = OB_ERROR;
}
else
{
ret = init_scan_param(table_name, table_id, max_len, scan_param);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init scan param failed:table[%s], ret[%d]", table_name, ret);
}
}
if (OB_SUCCESS == ret)
{
// for the first table tablet
ObRowkey row_key;
ObObj temp_buffer[common::OB_MAX_ROWKEY_COLUMN_NUMBER];
row_key.assign(temp_buffer, common::OB_MAX_ROWKEY_COLUMN_NUMBER);
ObGetParam param;
ObScanner scanner;
ObServer server;
ObRowkey start_key;
ObRowkey end_key;
ObCellInfo * cell = NULL;
ObScannerIterator iter;
bool row_change = false;
ObString name;
name.assign(const_cast<char*>(table_name), (int32_t)strlen(table_name));
ObCellInfo temp_cell;
temp_cell.table_id_ = table_id;
temp_cell.column_id_ = 0;
common::ModuleArena allocator;
while ((OB_SUCCESS == ret) && (!row_key.is_max_row()))
{
param.reset();
param.set_is_read_consistency(false);
temp_cell.row_key_ = row_key;
ret = param.add_cell(temp_cell);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "add cell failed:ret[%d]", ret);
break;
}
ret = rpc_->get(root_server_, timeout_, param, scanner);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "get root table for tablet failed:table[%lu], ret[%d]", table_id, ret);
break;
}
else
{
// skip the first row
iter = scanner.begin();
++iter;
while ((iter != scanner.end())
&& (OB_SUCCESS == (ret = iter.get_cell(&cell, &row_change))) && !row_change)
{
if (NULL == cell)
{
TBSYS_LOG(ERROR, "%s", "check cell failed");
ret = OB_INNER_STAT_ERROR;
break;
}
ret = cell->row_key_.deep_copy(start_key, allocator);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(WARN, "deep copy the rowkey failed:ret[%d]", ret);
}
++iter;
}
}
// scanner iter end
if (ret == OB_SUCCESS)
{
int64_t ip = 0;
int64_t port = 0;
int64_t version = 0;
TabletLocation list;
for (++iter; iter != scanner.end(); ++iter)
{
ret = iter.get_cell(&cell, &row_change);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret);
break;
}
else if (row_change) // && (iter != last_iter))
{
TaskInfo task;
task.set_table_id(table_id);
task.set_table_name(table_name);
ret = init_new_task(name, start_key, end_key, scan_param, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret);
break;
}
else
{
ret = insert_new_task(list, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret);
break;
}
++count;
}
list.clear();
start_key = end_key;
}
else
{
cell->row_key_.deep_copy(end_key, allocator);
if ((cell->column_name_.compare("1_ms_port") == 0)
|| (cell->column_name_.compare("2_ms_port") == 0)
|| (cell->column_name_.compare("3_ms_port") == 0))
{
ret = cell->value_.get_int(port);
}
else if ((cell->column_name_.compare("1_ipv4") == 0)
|| (cell->column_name_.compare("2_ipv4") == 0)
|| (cell->column_name_.compare("3_ipv4") == 0))
{
ret = cell->value_.get_int(ip);
}
else if ((cell->column_name_.compare("1_tablet_version") == 0)
|| (cell->column_name_.compare("2_tablet_version") == 0)
|| (cell->column_name_.compare("3_tablet_version") == 0))
{
ret = cell->value_.get_int(version);
if (OB_SUCCESS == ret)
{
if (0 == port || port == 0)
{
TBSYS_LOG(WARN, "%s", "check port or ip failed");
}
else
{
server.set_ipv4_addr(int32_t(ip), int32_t(port));
ObTabletLocation addr(version, server);
if (OB_SUCCESS != (ret = list.add(addr)))
{
TBSYS_LOG(ERROR, "add addr failed:server[%d], port[%d], ret[%d]",
server.get_ipv4(), server.get_port(), ret);
}
else
{
TBSYS_LOG(DEBUG, "add addr succ:server[%s], version:%ld", server.to_cstring(), version);
}
}
ip = port = version = 0;
}
}
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret);
break;
}
}
}
// for the last row
if ((OB_SUCCESS == ret) && (start_key != end_key))
{
TaskInfo task;
task.set_table_id(table_id);
task.set_table_name(table_name);
ret = init_new_task(name, start_key, end_key, scan_param, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret);
}
else
{
ret = insert_new_task(list, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret);
}
++count;
}
if (OB_SUCCESS == ret)
{
// copy all the end key objs to row_key
for (int64_t i = 0; i < end_key.length(); ++i)
{
const_cast<ObObj *> (row_key.ptr())[i] = end_key.ptr()[i];
row_key.assign(const_cast<ObObj *> (row_key.ptr()), end_key.length());
}
// modify last row key for next get root table
if (!end_key.is_max_row())
{
const_cast<ObObj *>(row_key.ptr())[end_key.length()].set_max_value();
row_key.assign(const_cast<ObObj *> (row_key.ptr()), end_key.length() + 1);
}
}
}
list.clear();
}
}
}
return ret;
}
// maybe not find a merge server
int TaskFactory::get_table_tablet(const char * table_name, const uint64_t table_id, uint64_t & count)
{
int ret = OB_SUCCESS;
ObScanParam scan_param;
uint64_t max_len = 0;
if (NULL == table_name)
{
TBSYS_LOG(ERROR, "check table name failed:table[%s], id[%lu]", table_name, table_id);
ret = OB_ERROR;
}
else
{
ret = init_scan_param(table_name, table_id, max_len, scan_param);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init scan param failed:table[%s], ret[%d]", table_name, ret);
}
}
if (OB_SUCCESS == ret)
{
TaskInfo task;
task.set_table_id(table_id);
task.set_table_name(table_name);
// for the first table tablet
ObString row_key;
char temp_buffer[1];
memset(temp_buffer, 0, sizeof(temp_buffer));
row_key.assign(temp_buffer, sizeof(temp_buffer));
ObGetParam param;
ObScanner scanner;
ObServer server;
ObString start_key;
ObString end_key;
ObCellInfo * cell = NULL;
ObScannerIterator iter;
bool row_change = false;
ObString name;
name.assign(const_cast<char*>(table_name), strlen(table_name));
ObCellInfo temp_cell;
temp_cell.table_id_ = table_id;
temp_cell.column_id_ = 0;
const uint64_t MAX_LEN = 1024;
char last_tablet_rowkey[MAX_LEN] = "";
const int32_t MAX_SERVER_ADDR_SIZE = 128;
char server_addr[MAX_SERVER_ADDR_SIZE];
while ((OB_SUCCESS == ret)
&& (!is_max_rowkey(max_len, row_key.ptr(), row_key.length() - 1)))
{
param.reset();
param.set_is_read_consistency(false);
temp_cell.row_key_ = row_key;
ret = param.add_cell(temp_cell);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "add cell failed:ret[%d]", ret);
break;
}
ret = rpc_->get(root_server_, timeout_, param, scanner);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "get root table for tablet failed:table[%lu], ret[%d]", table_id, ret);
break;
}
else
{
// skip the first row
iter = scanner.begin();
++iter;
while ((iter != scanner.end())
&& (OB_SUCCESS == (ret = iter.get_cell(&cell, &row_change))) && !row_change)
{
if (NULL == cell)
{
TBSYS_LOG(ERROR, "%s", "check cell failed");
ret = OB_INNER_STAT_ERROR;
break;
}
start_key.assign(cell->row_key_.ptr(), cell->row_key_.length());
++iter;
}
}
if (ret == OB_SUCCESS)
{
int64_t ip = 0;
int64_t port = 0;
int64_t version = 0;
TabletLocation list;
for (++iter; iter != scanner.end(); ++iter)
{
ret = iter.get_cell(&cell, &row_change);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "get cell from scanner iterator failed:ret[%d]", ret);
break;
}
else if (row_change) // && (iter != last_iter))
{
ret = init_new_task(name, start_key, end_key, scan_param, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret);
break;
}
else
{
ret = insert_new_task(list, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret);
break;
}
++count;
}
list.clear();
start_key = end_key;
end_key.assign(cell->row_key_.ptr(), cell->row_key_.length());
}
else
{
end_key.assign(cell->row_key_.ptr(), cell->row_key_.length());
if ((cell->column_name_.compare("1_ms_port") == 0)
|| (cell->column_name_.compare("2_ms_port") == 0)
|| (cell->column_name_.compare("3_ms_port") == 0))
{
ret = cell->value_.get_int(port);
}
else if ((cell->column_name_.compare("1_ipv4") == 0)
|| (cell->column_name_.compare("2_ipv4") == 0)
|| (cell->column_name_.compare("3_ipv4") == 0))
{
ret = cell->value_.get_int(ip);
}
else if ((cell->column_name_.compare("1_tablet_version") == 0)
|| (cell->column_name_.compare("2_tablet_version") == 0)
|| (cell->column_name_.compare("3_tablet_version") == 0))
{
ret = cell->value_.get_int(version);
if (OB_SUCCESS == ret)
{
if (0 == port || port == 0)
{
TBSYS_LOG(WARN, "%s", "check port or ip failed");
}
else
{
server.set_ipv4_addr(ip, port);
ObTabletLocation addr(version, server);
if (OB_SUCCESS != (ret = list.add(addr)))
{
TBSYS_LOG(ERROR, "add addr failed:server[%d], port[%d], ret[%d]",
server.get_ipv4(), server.get_port(), ret);
}
else
{
server.to_string(server_addr, MAX_SERVER_ADDR_SIZE);
TBSYS_LOG(DEBUG, "add addr succ:server[%s], version:%ld", server_addr, version);
}
}
ip = port = version = 0;
}
}
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "check get value failed:ret[%d]", ret);
break;
}
}
}
// for the last row
if ((OB_SUCCESS == ret) && (start_key != end_key))
{
ret = init_new_task(name, start_key, end_key, scan_param, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "init new task failed:ret[%d]", ret);
}
else
{
ret = insert_new_task(list, task);
if (ret != OB_SUCCESS)
{
TBSYS_LOG(ERROR, "insert new task failed:ret[%d]", ret);
}
++count;
}
if (OB_SUCCESS == ret)
{
// modify last row key for next get root table
uint64_t len = end_key.length();
if (MAX_LEN > len + 1)
{
memcpy(last_tablet_rowkey, end_key.ptr(), len);
last_tablet_rowkey[len] = 0;
row_key.assign(last_tablet_rowkey, len + 1);
}
else
{
TBSYS_LOG(ERROR, "check end key failed:len[%lu]", len);
}
}
}
list.clear();
}
}
}
return ret;
}
