int TaskManager::finish_task(const bool result, const TaskInfo & task) { int ret = OB_SUCCESS; if (task.get_token() != task_token_) { TBSYS_LOG(ERROR, "check task token failed:token[%ld], task[%ld]", task_token_, task.get_token()); ret = OB_ERROR; } else { map<uint64_t, TaskInfo>::iterator it; int64_t timestamp = tbsys::CTimeUtil::getTime(); tbsys::CThreadGuard lock(&lock_); int64_t task_count = get_server_task_count(task.get_location()[task.get_index()].chunkserver_); if (task_count < 1) { TBSYS_LOG(WARN, "check server task count failed:task[%lu], count[%ld]", task.get_id(), task_count); } else { TBSYS_LOG(DEBUG, "server ip = %ld, task_count = %ld", task.get_location()[task.get_index()].chunkserver_.get_ipv4(), task_count); // wait timeout for next dispatch working_queue_[task.get_location()[task.get_index()].chunkserver_] = --task_count; // print_access_server(); } it = doing_queue_.find(task.get_id()); if (it != doing_queue_.end()) { if (true == result) { ++total_finish_count_; total_finish_time_ += timestamp - task.get_timestamp(); complete_queue_.insert(pair<uint64_t, TaskInfo>(task.get_id(), task)); doing_queue_.erase(it); } // WARN: not insert into wait queue for timeout if result != true } else { it = complete_queue_.find(task.get_id()); if (it != complete_queue_.end()) { if (true == result) { // for compute average finish time ++total_finish_count_; total_finish_time_ += timestamp - task.get_timestamp(); } TBSYS_LOG(WARN, "find the task already finished:task[%lu]", task.get_id()); } else { TBSYS_LOG(ERROR, "not find this task in doing and complete queue:task[%lu]", task.get_id()); ret = OB_ERROR; } } } TBSYS_LOG(INFO, "finish monitor task [id=%lu] stat:wait[%lu], doing[%lu], finish[%lu], avg_time[%ld]", task.get_id(), wait_queue_.size(), doing_queue_.size(), complete_queue_.size(), total_finish_time_ / total_finish_count_); return ret; }
int TaskManager::fetch_task(TaskCounter & counter, TaskInfo & task) { int ret = OB_SUCCESS; int64_t task_count = 0; bool find_task = false; map<uint64_t, TaskInfo>::iterator it; tbsys::CThreadGuard lock(&lock_); // step 1. check waiting task for (it = wait_queue_.begin(); it != wait_queue_.end(); ++it) { for (int64_t i = 0; i < it->second.get_location().size(); ++i) { if (it->second.get_location()[i].tablet_version_ != tablet_version_ && it->second.get_location()[i].tablet_version_ != (tablet_version_ + 1)) { #if 1 it->second.get_location()[i].dump(it->second.get_location()[i]); TBSYS_LOG(DEBUG, "skip task[%ld], due to version compatiablility", it->first); #endif continue; } TBSYS_LOG(DEBUG, "server:%ld is selected, task_count = %ld", it->second.get_location()[i].chunkserver_.get_ipv4(), task_count); task_count = get_server_task_count(it->second.get_location()[i].chunkserver_); if (task_count >= max_count_) { continue; } else { task = it->second; task.set_index(i); find_task = true; task.set_timestamp(tbsys::CTimeUtil::getTime()); // remove this item to doing_queue_ wait_queue_.erase(it); doing_queue_.insert(pair<uint64_t, TaskInfo>(task.get_id(), task)); break; } } if (find_task) { break; } } // step 2. check doing timeout task if ((false == find_task) && (total_finish_count_ != 0)) { int64_t timestamp = tbsys::CTimeUtil::getTime(); int64_t avg_finish_time = total_finish_time_/total_finish_count_; for (it = doing_queue_.begin(); it != doing_queue_.end(); ++it) { if ((timestamp - it->second.get_timestamp()) > (avg_times_ * avg_finish_time)) { int64_t last_index = task.get_index(); /* last used mergeserver index */ // timeout so reset the visit count for (int64_t i = 0; i < it->second.get_location().size(); ++i) { if (i == last_index) { continue; /* do not allocate same task to same server */ } task_count = get_server_task_count(it->second.get_location()[i].chunkserver_); // must > not include equal with if (task_count > max_count_) { continue; } else { TBSYS_LOG(INFO, "check task timeout:task[%lu], avg_time[%ld], timeout_times[%ld], " "total_time[%ld], total_finish[%ld], finish[%ld], now[%ld], add_time[%ld], old_idx=%lu, new_idx=%lu", it->second.get_id(), avg_finish_time, avg_times_, total_finish_time_, total_finish_count_, complete_queue_.size(), timestamp, it->second.get_timestamp(), last_index, i); // update timestamp it->second.set_timestamp(timestamp); task = it->second; task.set_index(i); find_task = true; break; } } if (find_task) { break; } } } } // set task start timestamp task.set_timestamp(tbsys::CTimeUtil::getTime()); counter.total_count_ = total_task_count_; counter.wait_count_ = wait_queue_.size(); counter.doing_count_ = doing_queue_.size(); counter.finish_count_ = complete_queue_.size(); if (false == find_task) { TBSYS_LOG(DEBUG, "not find suitable task"); ret = OB_ERROR; } else { // update working queue for first merge server working_queue_[task.get_location()[task.get_index()].chunkserver_] = ++task_count; //print_access_server(); } return ret; }