std::string
JobQueue::toString()
{
stringstream ss;
IceUtil::Monitor<IceUtil::Mutex>::Lock lock(*this);
if ( activeJobCount_ )
ss<<"ACTIVE JOBS (maybe out of sync):"<<endl;
for ( size_t i=0; i<workerPool_.size(); ++i )
{
string s;
if ( workerPool_[i] )
{
WorkerPtr w = WorkerPtr::dynamicCast( workerPool_[i] );
s = w->toString();
if ( !s.empty() )
ss << " " << w->toString() << endl;
}
}
if ( !pendingJobs_.empty() )
ss<<"PENDING JOBS:"<<endl;
for ( std::list<JobPtr>::const_iterator it=pendingJobs_.begin(); it!=pendingJobs_.end(); ++it ) {
const JobPtr job = *it;
ss << " " << job->toString() << endl;
}
return ss.str();
}
bool StatSystem::growStats(JobPtr job, int multiplier)
{
set<StatPtr> statSet = job->getGrowthStats();
for(StatPtr stat:statSet)
{
auto isStatSame = [&] (StatPtr statPtr)
{
return *statPtr == *stat;
};
auto statItr = find_if(calculatedStats.begin(),
calculatedStats.end(),
isStatSame);
if(statItr == calculatedStats.end())
{
auto statBase = job->getBaseStats();
auto baseIter = statBase.find(stat);
StatPtr foundBase = *baseIter;
calculatedStats.insert(foundBase);
statItr = calculatedStats.find(stat);//Review this> Unnecesary find?
}
StatPtr foundStat = *statItr;
int growth = stat->getDefaultNumber();
foundStat->grow(growth * multiplier);
}
return true;
}
void JobsTabWidget::backupJob(JobPtr job)
{
if(!job)
return;
if(!job->validateUrls())
{
if(job->urls().isEmpty())
{
QMessageBox::warning(this, tr("Job error"),
tr("Job %1 has no backup paths selected. "
"Nothing to back up.")
.arg(job->name()));
return;
}
else
{
QMessageBox::StandardButton confirm = QMessageBox::question(
this, tr("Job warning"),
tr("Some backup paths for Job %1 are not"
" accessible anymore and thus backup may"
" be incomplete."
" Proceed with backup?")
.arg(job->name()));
if(confirm != QMessageBox::Yes)
return;
}
}
emit backupNow(job->createBackupTask());
}
void JobListWidget::execDeleteJob(JobListWidgetItem *jobItem)
{
if(!jobItem)
{
DEBUG << "Null JobListWidgetItem passed.";
return;
}
JobPtr job = jobItem->job();
QMessageBox::StandardButton confirm =
QMessageBox::question(this, tr("Confirm action"),
tr("Are you sure you want to delete job \"%1\" "
"(this cannot be undone)?")
.arg(job->name()));
if(confirm != QMessageBox::Yes)
return;
bool purgeArchives = false;
if(!job->archives().isEmpty())
{
QMessageBox::StandardButton delArchives =
QMessageBox::question(this, tr("Confirm action"),
tr("Also delete %1 archives "
"belonging to this job "
"(this cannot be undone)?")
.arg(job->archives().count()));
if(delArchives == QMessageBox::Yes)
purgeArchives = true;
}
emit deleteJob(job, purgeArchives);
delete jobItem;
emit countChanged(count(), visibleItemsCount());
}
void Peer::HandleTaskExecute(MessagePtr pMsg)
{
bool status = true;
PeerNodePtr pPeer = pMsg->m_conn->GetPeerNode();
JobPtr pJob = this->GetJob(pMsg->m_fromPeerId, pMsg->m_taskId, pPeer);
if(pJob == NULL) {
Log(ERR, L"Peer::HandleTaskExecute- Task Execute request received for Non-Existing Job from Peer: %d, taskid: %d\n",
pMsg->m_fromPeerId, pMsg->m_taskId);
status = false;
}
else {
Log(CONSOLE, L"Handling Task Execute from Peer: %d, taskid: %d\n",
pMsg->m_fromPeerId, pMsg->m_taskId);
// start the job
pJob->SetPeerNode(pMsg->m_conn->GetPeerNode());
int result = pJob->Start();
//[TODO] handle job failure to execute
if(result < 0) {
Log(ERR, L"Peer::HandleTaskExecute- Task Execute request could not be completed from Peer: %d, taskid: %d\n",
pMsg->m_fromPeerId, pMsg->m_taskId);
status = false;
}
}
// send response
PeerNodePtr pNode = pMsg->m_conn->GetPeerNode();
pNode->TaskExecuteResponse(pMsg->m_taskId, status);
}
void
JobQueue::add( const JobPtr& job )
{
IceUtil::Monitor<IceUtil::Mutex>::Lock lock(*this);
if ( isQueueStopping_ )
return;
pendingJobs_.push_back( job );
if ( job->isTracable_ || config_.traceAddEvents )
tracer_.info( "Added "+job->toString()+" to the job queue" );
if ( config_.queueSizeWarn>0 && pendingJobs_.size()>=(size_t)config_.queueSizeWarn ) {
std::stringstream ss;
ss << "Job queue size " << pendingJobs_.size() << " after adding "<<job->toString();
tracer_.warning( ss.str() );
}
if ( config_.traceStalledJobs ) {
std::string jobInfo;
for ( size_t i=0; i<workerPool_.size(); ++i )
{
WorkerPtr w = WorkerPtr::dynamicCast( workerPool_[i] );
if ( w->isJobStalled( jobInfo ) )
tracer_.warning( jobInfo );
}
}
notify();
}
JobPtr Peer::GetJob(int peerId, int taskId, PeerNodePtr pNode)
{
std::map<int, JobPtr>::iterator it;
for(it = m_jobList.begin(); it != m_jobList.end(); it++) {
JobPtr pJob = (*it).second;
if(pJob->GetTaskId() == taskId && pJob->GetRemotePeerId() == peerId)
if(pNode == NULL)
return pJob;
else if(pNode == pJob->GetPeerNode())
return pJob;
}
return JobPtr();
}
Job::JobPtr Job::newJob(const string& name) {
JobPtr tmp;
std::unique_lock<mutex> lock(globalJobMutex);
auto it = getMap().find(boost::to_upper_copy(name));
if(it != getMap().end()) {
tmp.reset(it->second.second());
} else {
#ifdef DEBUG__
cerr << "No job with tag \"" << name << "\" registered." << endl;
#endif
}
return tmp;
}
bool GroupPool::AddJobToPrivilegeQueue(const JobPtr& job, bool push_front) {
WriteLocker locker(m_privilege_job_queue_lock);
// TODO test
if (!FindByName(job->GetGroupName())) {
LOG4CPLUS_ERROR(logger, "No this group, group name: " << job->GetGroupName());
return false;
}
printf("privilege!\n");
if (push_front)
m_privilege_wait_job_queue.push_front(job);
else
m_privilege_wait_job_queue.push_back(job);
return true;
}
msg_t *Scheduler::concurrentHandler(msg_t &request, msg_t &response,
unsigned long sizeBytes) {
auto jCondVar = CondVarPtr(new boost::condition_variable);
boost::mutex jobMutex;
boost::unique_lock<boost::mutex> lock(jobMutex);
auto jInfo = JobInfoPtr(new JobInfo());
jInfo->setFinished(false);
jInfo->setStarted(false);
JobPtr nJob = JobPtr(new Job(&request, getNextId()));
JobTuple t = std::make_tuple(nJob, jInfo, jCondVar);
JobTuplePtr jobPtr = std::make_shared<JobTuple>(t);
std::get<0>(*jobPtr)->setIssueTime(bc::system_clock::now());
readyQ->push_front(jobPtr);
// LOGF(debug, " New conn tid = %1%, Added job to readyQ %2%")
// % boost::this_thread::get_id() % *nJob;
try {
while (!jInfo->isFinished()) {
jCondVar->wait(lock);
}
lock.unlock();
// LOG(debug) << *nJob << " finished";
msg_t *rsp = nJob->getRsp();
if (rsp == NULL) {
// ERROR
} else {
response.msgId = rsp->msgId;
response.dataSize = rsp->dataSize;
response.paramsSize = 0;
response.totalBytes = rsp->totalBytes;
response.dataBytes = rsp->dataBytes;
memcpy(&response.data, rsp->data, rsp->dataBytes);
free(rsp);
}
}
catch (boost::thread_interrupted &) {
// LOG(debug) << "Concurrent handler interrupted";
return &response;
}
#ifdef DEBUG
response.print();
#endif
return &response;
}
//////////////////////////////////////////////////////////////////////
// Jobqueue
//
void Jobqueue::put(JobPtr& j, bool force)
{
j->touch();
cxxtools::MutexLock lock(_mutex);
if (!force && _capacity > 0)
{
while (_jobs.size() >= _capacity)
{
log_warn("Jobqueue full");
_notFull.wait(lock);
}
}
_jobs.push_back(j);
// We have to drop ownership before releasing the lock of the queue.
// Therefore we set the smart pointer to 0.
j = 0;
if (_waitThreads == 0)
noWaitThreads.signal();
_notEmpty.signal();
}
// add job to wait queue
bool GroupPool::AddJobToWaitQueue(const JobPtr& job) {
int priority = job->GetRawPriority();
if (JOB_RAW_PRIO_PRIVILEGE == priority) {
if (!AddJobToPrivilegeQueue(job)) {
LOG4CPLUS_ERROR(logger, "Failed to AddJobToPrivilegeQueue.");
return false;
}
} else {
if (!AddJobToWaitQueueByGroup(job)) {
// test
printf("No this group, %s\n", job->GetGroupName().c_str());
LOG4CPLUS_ERROR(logger, "No this group, group name: " << job->GetGroupName());
return false;
}
}
return true;
}
void JobListWidget::restoreItem()
{
if(sender())
{
JobPtr job = qobject_cast<JobListWidgetItem *>(sender())->job();
if(!job->archives().isEmpty())
{
ArchivePtr archive = job->archives().first();
RestoreDialog *restoreDialog = new RestoreDialog(this, archive);
restoreDialog->show();
connect(restoreDialog, &RestoreDialog::accepted, [=] {
emit restoreArchive(restoreDialog->archive(),
restoreDialog->getOptions());
});
}
}
}
void Peer::HandleTaskCancel(MessagePtr pMsg)
{
bool status = true;
JobPtr pJob = this->GetJob(pMsg->m_fromPeerId, pMsg->m_taskId);
if(pJob == NULL) {
Log(ERR, L"Peer::HandleTaskCancel- Task Cancel request received for Non-Existing Job from Peer: %d, taskid: %d\n",
pMsg->m_fromPeerId, pMsg->m_taskId);
status = false;
}
else {
Log(CONSOLE, L"Handling Task Cancel from Peer: %d, taskid: %d\n", pMsg->m_fromPeerId, pMsg->m_taskId);
// cancel the job
pJob->Stop();
m_jobList.erase(pJob->GetJobId());
}
}
bool JobQueueWorker::operator()()
{
logger_ = &log4cpp::Category::getInstance("JobQueueWorker(" + boost::lexical_cast<std::string>(boost::this_thread::get_id()) + ")");
while (true)
{
/*
* Get the next work item to do
*/
JobPtr job = queue_.dequeue();
if (!job)
return true;
logger_->debug("beginning job");
job->run();
logger_->debug("finished job");
}
}
bc::duration<double> Scheduler::estimateExecutionTime(JobPtr j) {
// TODO[mtottenh]:Add code to estimate a job finish time
double milliseconds = (double)j->getReq()->predictedTimeMillis;
bc::duration<double, boost::ratio<1, 1000> > milli_sec(milliseconds);
auto micro_sec = bc::duration_cast<bc::microseconds>(milli_sec);
// LOG(debug) << milli_sec.count() << "ms"
// << ", " << micro_sec.count() << "us";
return milli_sec;
}
bool GroupPool::AddJobToWaitQueueByGroup(const JobPtr& job) {
string group_name = job->GetGroupName();
GroupPtr group_ptr = GetGroupPtr(group_name);
if (NULL == group_ptr) {
return false;
}
// group_ptr->AddJobToWaitQueueAutoLock(job);
group_ptr->AddJobToWaitQueue(job);
return true;
}
int32_t DefaultScheduler::ScheduleOneJob(JobPtr& job_ptr) {
printf("Default Scheduler\n");
bool has_success = false;
bool has_no_success = false;
list<TaskPtr> task_list = job_ptr->GetTaskList();
for (list<TaskPtr>::const_iterator it = task_list.begin();
it != task_list.end(); ++it)
{
// only schedule waiting task
if((*it)->GetTaskState() != TASK_WAITING) {
continue;
}
if (ScheduleOneTask(*it) != 0) {
has_no_success = true; // has task no success
} else {
has_success = true; // has task success
}
}
if (!has_no_success) {
// all tasks success
job_ptr->SetState(JOB_RUNNING);
return 0;
} else if (has_success) {
// partly tasks success
job_ptr->SetState(JOB_SCHEDULING);
return -1;
} else {
// all tasks no success
return -2;
}
return 0;
}
void JobListWidget::selectJob(JobPtr job)
{
if(!job)
{
DEBUG << "Null JobPtr passed.";
return;
}
for(int i = 0; i < count(); ++i)
{
JobListWidgetItem *jobItem = static_cast<JobListWidgetItem *>(item(i));
if(jobItem && (jobItem->job()->objectKey() == job->objectKey()))
{
clearSelection();
setCurrentItem(jobItem);
scrollToItem(currentItem(), QAbstractItemView::EnsureVisible);
break;
}
}
}
void Peer::HandleJobExecution(WMessagePtr pMsg)
{
bool bJobResult = false;
SYS_MSG sysMsg = pMsg->GetSysMsg();
if(sysMsg == TASK_EXECUTION_SUCCESS) {
bJobResult = true;
int jobId = pMsg->GetJobId();
if(jobId <= 0) {
Log(ERR, L"Peer::HandleJobExecution Message from Invalid Job.. Ignoring it\n");
return;
}
// find the job
if(m_jobList.find(jobId) == m_jobList.end()) {
Log(ERR, L"Peer::HandleJobExecution Message from Invalid Job.. Ignoring it\n");
return;
}
JobPtr pJob = m_jobList[jobId];
// send the response to remote peer
RemotePeers rpeer = this->GetRemotePeerById(pJob->GetRemotePeerId());
// get the stats and put in workhistory
double execTime = pJob->GetExecutionTime();
WorkHistoryPtr pWH = WorkHistory::GetWorkHistory();
pWH->SetNewFinishTaskInfo(execTime, true);
// not found in db
// [TODO] handle when peer not found in db
if(rpeer.peer_id < 0) {
Log(ERR, L"Peer::HandleJobExecution Peer: %d, Not found in DB for job: %d, still sending the result\n",
pJob->GetRemotePeerId(), jobId);
}
//PeerNodePtr pNode = PeerNode::CreatePeerNode(rpeer);
PeerNodePtr pNode = pJob->GetPeerNode();
pNode->TaskExecuteResult(pJob->GetTaskId(), bJobResult, pJob->GetResult(), pJob->GetParam());
}
}
void JobListWidget::addJob(JobPtr job)
{
if(!job)
{
DEBUG << "Null JobPtr passed.";
return;
}
JobListWidgetItem *item = new JobListWidgetItem(job);
connect(item, &JobListWidgetItem::requestBackup, this,
&JobListWidget::backupItem);
connect(item, &JobListWidgetItem::requestInspect, this,
&JobListWidget::inspectItem);
connect(item, &JobListWidgetItem::requestRestore, this,
&JobListWidget::restoreItem);
connect(item, &JobListWidgetItem::requestDelete, this,
&JobListWidget::deleteItem);
insertItem(count(), item);
setItemWidget(item, item->widget());
item->setHidden(!job->name().contains(_filter));
emit countChanged(count(), visibleItemsCount());
}
void JobQueue::enqueue(JobPtr job)
{
if (!job)
throw std::runtime_error("Illegal state: Tried to insert NULL job into JobQueue");
boost::unique_lock<boost::mutex> lock(queue_mutex_);
/* Ensure a duplicate job isn't already present. If one is
* already there, don't add the new one.
* FIXME: This currently doesn't handle the case when the
* duplicate job is currently being executed. */
for (Queue::iterator itr = queue_.begin(); itr != queue_.end(); ++itr)
{
JobPtr j = *itr;
if (!job->equals(j))
continue;
/* Found a duplicate job */
if (JobPriorityComparator()(job, j))
{
/* New job comes before existing job; replace
* the existing job */
queue_.erase(itr);
break;
}
else
{
/* New job comes after existing job; drop
* the new job */
return;
}
}
queue_.insert(job);
queue_cond_.notify_one();
}
void JobInfo::PrintJobInfo( std::string &info, Scheduler &scheduler, int64_t jobId )
{
std::ostringstream ss;
ScheduledJobs &jobs = scheduler.GetScheduledJobs();
JobPtr job;
if ( !jobs.FindJobByJobId( jobId, job ) )
{
ss << "job isn't executing now, jobId = " << jobId;
info = ss.str();
return;
}
ss << "================" << std::endl <<
"Job info, jobId = " << job->GetJobId() << std::endl;
if ( job->GetGroupId() >= 0 )
{
ss << "group id = " << job->GetGroupId() << std::endl;
}
if ( job->GetJobGroup() )
{
const std::string &metaJobName = job->GetJobGroup()->GetName();
if ( !metaJobName.empty() )
{
ss << "meta job name = '" << metaJobName << '\'' << std::endl;
}
}
if ( !job->GetAlias().empty() )
{
ss << "job alias = '" << job->GetAlias() << '\'' << std::endl;
}
else
{
ss << "job path = '" << job->GetFilePath() << '\'' << std::endl;
}
if ( !job->GetName().empty() )
{
ss << "job name = '" << job->GetName() << '\'' << std::endl;
}
ss << "----------------" << std::endl;
{
int totalExec = job->GetNumPlannedExec();
int numExec = totalExec - jobs.GetNumExec( jobId );
ss << "job executions = " << numExec << std::endl <<
"total planned executions = " << totalExec << std::endl;
}
{
int numWorkers = 0;
int numCPU = 0;
const Scheduler::IPToNodeState &nodeState = scheduler.GetNodeState();
for( auto it = nodeState.cbegin(); it != nodeState.cend(); ++it )
{
const NodeState &nodeState = it->second;
const WorkerPtr &worker = nodeState.GetWorker();
if ( !worker )
continue;
const WorkerJob &workerJob = worker->GetJob();
if ( workerJob.HasJob( jobId ) )
{
++numWorkers;
numCPU += workerJob.GetNumTasks( jobId );
}
}
ss << "busy workers = " << numWorkers << std::endl;
ss << "busy cpu's = " << numCPU << std::endl;
}
ss << "================";
info = ss.str();
}
void JobScheduler::wait( JobPtr const &job )
{
while( !job->isCompleted() ) {
executeNextJob();
}
}
void Scheduler::runJob(JobResPairPtr j) {
ResourceListPtr resourceList;
JobTuplePtr jobTuplePtr;
std::tie(jobTuplePtr, resourceList) = *j;
JobPtr jobPtr = std::get<0>(*jobTuplePtr);
// LOG(debug) << "- Scheduler::runJob(" << *jobPtr << ")";
// LOG(debug) << "- Allocated[" << resourceList->size() << "]";
char packed_rids = 0x0;
std::stringstream ss;
for (auto &r : *resourceList) {
ss << r.getId() << ", ";
switch (r.getId()) {
case 1:
packed_rids |= RID1;
break;
case 2:
packed_rids |= RID2;
break;
case 3:
packed_rids |= RID3;
break;
case 4:
packed_rids |= RID4;
break;
}
}
// LOG(debug) << "\t* Rids: { " << ss.str() << "}";
const Resource& r = resourceList->front();
const string &name = r.getName().c_str();
int portNumber = r.getPort();
msg_t *req = jobPtr->getReq();
// LOG(debug) << "\t* Opening connection to: " << name << ":"
// << portNumber << "";
Client c(portNumber, name);
auto start = bc::system_clock::now();
c.start();
boost::this_thread::interruption_point();
if (req != NULL) {
req->rids = packed_rids;
c.send(req);
}
int sizeBytes = sizeof(msg_t) + req->expDataSizeBytes;
char *buff = (char *)calloc(sizeBytes, 1);
if (buff == NULL) {
// LOG(debug) << "\tUnable to allocate result buffer";
c.stop();
returnResources(resourceList);
resourceList = nullptr;
}
msg_t *rsp = (msg_t *)buff;
do {
// LOG(debug) << "\t Reading reply, size: " << sizeBytes;
c.read(buff, sizeBytes);
} while (rsp->msgId != MSG_RESULT);
// LOG(debug) << "\t* Got Result";
c.stop();
auto end = bc::system_clock::now();
jobPtr->setMeasuredExecutionTime(end - start);
jobPtr->setActualExecutionTime(bc::milliseconds(rsp->predictedTimeMillis));
jobPtr->setRsp((msg_t *)buff);
returnResources(resourceList);
resourceList = nullptr;
std::get<0>(*jobTuplePtr)->setFinishTime(bc::system_clock::now());
finishedQ->push_front(jobTuplePtr);
// LOG(debug) << "Removing job from runQ";
removeJobFromRunQ(jobPtr->getId());
}
void JobScheduler::execute( JobPtr const &job )
{
job->execute();
}