static void req_stat_job_step2(
struct stat_cntl *cntl) /* I/O (free'd on return) */
{
svrattrl *pal;
job *pjob = NULL;
struct batch_request *preq;
struct batch_reply *preply;
int rc = 0;
enum TJobStatTypeEnum type;
pbs_queue *pque = NULL;
int exec_only = 0;
int bad = 0;
long DTime; /* delta time - only report full pbs_attribute list if J->MTime > DTime */
static svrattrl *dpal = NULL;
int job_array_index = 0;
job_array *pa = NULL;
char log_buf[LOCAL_LOG_BUF_SIZE];
int iter;
time_t time_now = time(NULL);
long poll_jobs = 0;
char job_id[PBS_MAXSVRJOBID+1];
int job_substate = -1;
time_t job_momstattime = -1;
preq = cntl->sc_origrq;
type = (enum TJobStatTypeEnum)cntl->sc_type;
preply = &preq->rq_reply;
/* See pbs_server_attributes(1B) for details on "poll_jobs" behaviour */
if (dpal == NULL)
{
/* build 'delta' pbs_attribute list */
svrattrl *tpal;
tlist_head dalist;
int aindex;
int atrlist[] =
{
JOB_ATR_jobname,
JOB_ATR_resc_used,
JOB_ATR_LAST
};
CLEAR_LINK(dalist);
for (aindex = 0;atrlist[aindex] != JOB_ATR_LAST;aindex++)
{
if ((tpal = attrlist_create("", "", 23)) == NULL)
{
return;
}
tpal->al_valln = atrlist[aindex];
if (dpal == NULL)
dpal = tpal;
append_link(&dalist, &tpal->al_link, tpal);
}
} /* END if (dpal == NULL) */
if (type == tjstArray)
{
pa = get_array(preq->rq_ind.rq_status.rq_id);
if (pa == NULL)
{
req_reject(PBSE_UNKARRAYID, 0, preq, NULL, "unable to find array");
return;
}
}
iter = -1;
get_svr_attr_l(SRV_ATR_PollJobs, &poll_jobs);
if (!poll_jobs)
{
/* polljobs not set - indicates we may need to obtain fresh data from
MOM */
if (cntl->sc_jobid[0] == '\0')
pjob = NULL;
else
pjob = svr_find_job(cntl->sc_jobid, FALSE);
while (1)
{
if (pjob == NULL)
{
/* start from the first job */
if (type == tjstJob)
{
pjob = svr_find_job(preq->rq_ind.rq_status.rq_id, FALSE);
}
else if (type == tjstQueue)
{
pjob = next_job(cntl->sc_pque->qu_jobs,&iter);
}
else if (type == tjstArray)
{
job_array_index = 0;
/* increment job_array_index until we find a non-null pointer or hit the end */
while (job_array_index < pa->ai_qs.array_size)
{
if (pa->job_ids[job_array_index] != NULL)
{
if ((pjob = svr_find_job(pa->job_ids[job_array_index], FALSE)) != NULL)
{
unlock_ji_mutex(pjob, __func__, "2", LOGLEVEL);
break;
}
}
job_array_index++;
}
}
else
{
pjob = next_job(&alljobs,&iter);
}
} /* END if (pjob == NULL) */
else
{
strcpy(job_id, pjob->ji_qs.ji_jobid);
unlock_ji_mutex(pjob, __func__, "3", LOGLEVEL);
if (type == tjstJob)
break;
if (type == tjstQueue)
pjob = next_job(cntl->sc_pque->qu_jobs,&iter);
else if (type == tjstArray)
{
pjob = NULL;
/* increment job_array_index until we find a non-null pointer or hit the end */
while (++job_array_index < pa->ai_qs.array_size)
{
if (pa->job_ids[job_array_index] != NULL)
{
if ((pjob = svr_find_job(pa->job_ids[job_array_index], FALSE)) != NULL)
{
unlock_ji_mutex(pjob, __func__, "3", LOGLEVEL);
break;
}
}
}
}
else
pjob = next_job(&alljobs,&iter);
}
if (pjob == NULL)
break;
strcpy(job_id, pjob->ji_qs.ji_jobid);
job_substate = pjob->ji_qs.ji_substate;
job_momstattime = pjob->ji_momstat;
strcpy(cntl->sc_jobid, job_id);
unlock_ji_mutex(pjob, __func__, "4", LOGLEVEL);
pjob = NULL;
/* PBS_RESTAT_JOB defaults to 30 seconds */
if ((job_substate == JOB_SUBSTATE_RUNNING) &&
((time_now - job_momstattime) > JobStatRate))
{
/* go to MOM for status */
if ((rc = stat_to_mom(job_id, cntl)) == PBSE_MEM_MALLOC)
break;
if (rc != 0)
{
pjob = svr_find_job(job_id, FALSE);
rc = 0;
continue;
}
if (pa != NULL)
unlock_ai_mutex(pa, __func__, "1", LOGLEVEL);
return; /* will pick up after mom replies */
}
} /* END while(1) */
if (rc != 0)
{
if (pa != NULL)
unlock_ai_mutex(pa, __func__, "2", LOGLEVEL);
reply_free(preply);
req_reject(rc, 0, preq, NULL, "cannot get update from mom");
return;
}
} /* END if (!server.sv_attr[SRV_ATR_PollJobs].at_val.at_long) */
/*
* now ready for part 3, building the status reply,
* loop through again
*/
if ((type == tjstSummarizeArraysQueue) ||
(type == tjstSummarizeArraysServer))
{
/* No array can be owned for these options */
update_array_statuses();
}
if (type == tjstJob)
pjob = svr_find_job(preq->rq_ind.rq_status.rq_id, FALSE);
else if (type == tjstQueue)
pjob = next_job(cntl->sc_pque->qu_jobs,&iter);
else if (type == tjstSummarizeArraysQueue)
pjob = next_job(cntl->sc_pque->qu_jobs_array_sum,&iter);
else if (type == tjstSummarizeArraysServer)
pjob = next_job(&array_summary,&iter);
else if (type == tjstArray)
{
job_array_index = -1;
pjob = NULL;
/* increment job_array_index until we find a non-null pointer or hit the end */
while (++job_array_index < pa->ai_qs.array_size)
{
if (pa->job_ids[job_array_index] != NULL)
{
if ((pjob = svr_find_job(pa->job_ids[job_array_index], FALSE)) != NULL)
{
break;
}
}
}
}
else
pjob = next_job(&alljobs,&iter);
DTime = 0;
if (preq->rq_extend != NULL)
{
char *ptr;
/* FORMAT: { EXECQONLY | DELTA:<EPOCHTIME> } */
if (strstr(preq->rq_extend, EXECQUEONLY))
exec_only = 1;
ptr = strstr(preq->rq_extend, "DELTA:");
if (ptr != NULL)
{
ptr += strlen("delta:");
DTime = strtol(ptr, NULL, 10);
}
}
if ((type == tjstTruncatedServer) ||
(type == tjstTruncatedQueue))
{
long sentJobCounter;
long qjcounter;
long qmaxreport;
int iter = -1;
/* loop through all queues */
while ((pque = next_queue(&svr_queues,&iter)) != NULL)
{
qjcounter = 0;
if ((exec_only == 1) &&
(pque->qu_qs.qu_type != QTYPE_Execution))
{
/* ignore routing queues */
unlock_queue(pque, __func__, "ignore queue", LOGLEVEL);
continue;
}
if (((pque->qu_attr[QA_ATR_MaxReport].at_flags & ATR_VFLAG_SET) != 0) &&
(pque->qu_attr[QA_ATR_MaxReport].at_val.at_long >= 0))
{
qmaxreport = pque->qu_attr[QA_ATR_MaxReport].at_val.at_long;
}
else
{
qmaxreport = TMAX_JOB;
}
if (LOGLEVEL >= 5)
{
sprintf(log_buf,"giving scheduler up to %ld idle jobs in queue %s\n",
qmaxreport,
pque->qu_qs.qu_name);
log_event(PBSEVENT_SYSTEM,PBS_EVENTCLASS_QUEUE,pque->qu_qs.qu_name,log_buf);
}
sentJobCounter = 0;
/* loop through jobs in queue */
if (pjob != NULL)
unlock_ji_mutex(pjob, __func__, "5", LOGLEVEL);
iter = -1;
while ((pjob = next_job(pque->qu_jobs,&iter)) != NULL)
{
if ((qjcounter >= qmaxreport) &&
(pjob->ji_qs.ji_state == JOB_STATE_QUEUED))
{
/* max_report of queued jobs reached for queue */
unlock_ji_mutex(pjob, __func__, "6", LOGLEVEL);
continue;
}
pal = (svrattrl *)GET_NEXT(preq->rq_ind.rq_status.rq_attr);
rc = status_job(
pjob,
preq,
(pjob->ji_wattr[JOB_ATR_mtime].at_val.at_long >= DTime) ? pal : dpal,
&preply->brp_un.brp_status,
&bad);
if ((rc != 0) && (rc != PBSE_PERM))
{
req_reject(rc, bad, preq, NULL, NULL);
if (pa != NULL)
{
unlock_ai_mutex(pa, __func__, "1", LOGLEVEL);
}
unlock_ji_mutex(pjob, __func__, "7", LOGLEVEL);
unlock_queue(pque, __func__, "perm", LOGLEVEL);
return;
}
sentJobCounter++;
if (pjob->ji_qs.ji_state == JOB_STATE_QUEUED)
qjcounter++;
unlock_ji_mutex(pjob, __func__, "8", LOGLEVEL);
} /* END foreach (pjob from pque) */
if (LOGLEVEL >= 5)
{
sprintf(log_buf,"sent scheduler %ld total jobs for queue %s\n",
sentJobCounter,
pque->qu_qs.qu_name);
log_event(PBSEVENT_SYSTEM,PBS_EVENTCLASS_QUEUE,pque->qu_qs.qu_name,log_buf);
}
unlock_queue(pque, __func__, "end while", LOGLEVEL);
} /* END for (pque) */
if (pa != NULL)
unlock_ai_mutex(pa, __func__, "1", LOGLEVEL);
reply_send_svr(preq);
return;
} /* END if ((type == tjstTruncatedServer) || ...) */
while (pjob != NULL)
{
/* go ahead and build the status reply for this job */
if (exec_only)
{
if (cntl->sc_pque != NULL)
{
if (cntl->sc_pque->qu_qs.qu_type != QTYPE_Execution)
goto nextjob;
}
else
{
if (pa != NULL)
pthread_mutex_unlock(pa->ai_mutex);
pque = get_jobs_queue(&pjob);
if (pa != NULL)
pthread_mutex_lock(pa->ai_mutex);
if ((pjob == NULL) ||
(pque == NULL))
goto nextjob;
mutex_mgr pque_mutex = mutex_mgr(pque->qu_mutex, true);
if (pque->qu_qs.qu_type != QTYPE_Execution)
{
goto nextjob;
}
}
}
pal = (svrattrl *)GET_NEXT(preq->rq_ind.rq_status.rq_attr);
rc = status_job(
pjob,
preq,
pal,
&preply->brp_un.brp_status,
&bad);
if ((rc != 0) &&
(rc != PBSE_PERM))
{
if (pa != NULL)
{
unlock_ai_mutex(pa, __func__, "1", LOGLEVEL);
}
unlock_ji_mutex(pjob, __func__, "9", LOGLEVEL);
req_reject(rc, bad, preq, NULL, NULL);
return;
}
/* get next job */
nextjob:
if (pjob != NULL)
unlock_ji_mutex(pjob, __func__, "10", LOGLEVEL);
if (type == tjstJob)
break;
if (type == tjstQueue)
pjob = next_job(cntl->sc_pque->qu_jobs,&iter);
else if (type == tjstSummarizeArraysQueue)
pjob = next_job(cntl->sc_pque->qu_jobs_array_sum,&iter);
else if (type == tjstSummarizeArraysServer)
pjob = next_job(&array_summary,&iter);
else if (type == tjstArray)
{
pjob = NULL;
/* increment job_array_index until we find a non-null pointer or hit the end */
while (++job_array_index < pa->ai_qs.array_size)
{
if (pa->job_ids[job_array_index] != NULL)
{
if ((pjob = svr_find_job(pa->job_ids[job_array_index], FALSE)) != NULL)
{
break;
}
}
}
}
else
pjob = next_job(&alljobs,&iter);
rc = 0;
} /* END while (pjob != NULL) */
if (pa != NULL)
{
unlock_ai_mutex(pa, __func__, "1", LOGLEVEL);
}
reply_send_svr(preq);
if (LOGLEVEL >= 7)
{
log_event(PBSEVENT_SYSTEM,
PBS_EVENTCLASS_JOB,
"req_statjob",
"Successfully returned the status of queued jobs\n");
}
return;
} /* END req_stat_job_step2() */
int acct_job(
job *pjob, /* I */
dynamic_string *ds) /* O */
{
int rc;
long cray_enabled = FALSE;
int resc_access_perm = READ_ONLY;
char local_buf[MAXLINE*4];
pbs_queue *pque;
tlist_head attrlist;
svrattrl *pal;
if (pjob == NULL)
{
return(PBSE_NONE);
}
CLEAR_HEAD(attrlist);
/* user */
/* acct_job is only called from account_jobstr and account_jobend. BufSize should be
PBS_ACCT_MAX_RCD + 1 in size. */
sprintf(local_buf, "user=%s ",
pjob->ji_wattr[JOB_ATR_euser].at_val.at_str);
if ((rc = append_dynamic_string(ds, local_buf)) != PBSE_NONE)
return(rc);
/* group */
sprintf(local_buf, "group=%s ",
pjob->ji_wattr[JOB_ATR_egroup].at_val.at_str);
if ((rc = append_dynamic_string(ds, local_buf)) != PBSE_NONE)
return(rc);
/* account */
if (pjob->ji_wattr[JOB_ATR_account].at_flags & ATR_VFLAG_SET)
{
sprintf(local_buf, "account=%s ",
pjob->ji_wattr[JOB_ATR_account].at_val.at_str);
if ((rc = append_dynamic_string(ds, local_buf)) != PBSE_NONE)
return(rc);
}
/* job name */
sprintf(local_buf, "jobname=%s ",
pjob->ji_wattr[JOB_ATR_jobname].at_val.at_str);
if ((rc = append_dynamic_string(ds, local_buf)) != PBSE_NONE)
return(rc);
if ((pque = get_jobs_queue(&pjob)) != NULL)
{
/* queue name */
sprintf(local_buf, "queue=%s ", pque->qu_qs.qu_name);
unlock_queue(pque, __func__, NULL, LOGLEVEL);
if ((rc = append_dynamic_string(ds, local_buf)) != PBSE_NONE)
return(rc);
}
else if (pjob == NULL)
{
log_err(PBSE_JOBNOTFOUND, __func__, "Job lost while acquiring queue 1");
return(PBSE_JOBNOTFOUND);
}
/* create time */
sprintf(local_buf, "ctime=%ld ",
pjob->ji_wattr[JOB_ATR_ctime].at_val.at_long);
if ((rc = append_dynamic_string(ds, local_buf)) != PBSE_NONE)
return(rc);
/* queued time */
sprintf(local_buf, "qtime=%ld ",
pjob->ji_wattr[JOB_ATR_qtime].at_val.at_long);
if ((rc = append_dynamic_string(ds, local_buf)) != PBSE_NONE)
return(rc);
/* eligible time, how long ready to run */
sprintf(local_buf, "etime=%ld ",
pjob->ji_wattr[JOB_ATR_etime].at_val.at_long);
if ((rc = append_dynamic_string(ds, local_buf)) != PBSE_NONE)
return(rc);
/* execution start time */
sprintf(local_buf, "start=%ld ",
(long)pjob->ji_qs.ji_stime);
if ((rc = append_dynamic_string(ds, local_buf)) != PBSE_NONE)
return(rc);
/* user */
sprintf(local_buf, "owner=%s ",
pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str);
if ((rc = append_dynamic_string(ds, local_buf)) != PBSE_NONE)
return(rc);
/* For large clusters strings can get pretty long. We need to see if there
is a need to allocate a bigger buffer */
/* execution host name */
if (pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str != NULL)
{
append_dynamic_string(ds, "exec_host=");
append_dynamic_string(ds, pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str);
if ((rc = append_dynamic_string(ds, " ")) != PBSE_NONE)
return(rc);
}
get_svr_attr_l(SRV_ATR_CrayEnabled, &cray_enabled);
if ((cray_enabled == TRUE) &&
(pjob->ji_wattr[JOB_ATR_login_node_id].at_flags & ATR_VFLAG_SET))
{
append_dynamic_string(ds, "login_node=");
append_dynamic_string(ds, pjob->ji_wattr[JOB_ATR_login_node_id].at_val.at_str);
if ((rc = append_dynamic_string(ds, " ")) != PBSE_NONE)
return(rc);
}
/* now encode the job's resource_list pbs_attribute */
job_attr_def[JOB_ATR_resource].at_encode(
&pjob->ji_wattr[JOB_ATR_resource],
&attrlist,
job_attr_def[JOB_ATR_resource].at_name,
NULL,
ATR_ENCODE_CLIENT,
resc_access_perm);
while ((pal = GET_NEXT(attrlist)) != NULL)
{
/* exec_host can use a lot of buffer space. Use a dynamic string */
append_dynamic_string(ds, pal->al_name);
if (pal->al_resc != NULL)
{
append_dynamic_string(ds, ".");
append_dynamic_string(ds, pal->al_resc);
}
append_dynamic_string(ds, "=");
append_dynamic_string(ds, pal->al_value);
if ((rc = append_dynamic_string(ds, " ")) != PBSE_NONE)
return(rc);
delete_link(&pal->al_link);
free(pal);
} /* END while (pal != NULL) */
#ifdef ATTR_X_ACCT
/* x attributes */
if (pjob->ji_wattr[JOB_SITE_ATR_x].at_flags & ATR_VFLAG_SET)
{
sprintf(local_buf, "x=%s ",
pjob->ji_wattr[JOB_SITE_ATR_x].at_val.at_str);
if ((rc = append_dynamic_string(ds, local_buf)) != PBSE_NONE)
return(rc);
}
#endif
/* SUCCESS */
return(PBSE_NONE);
} /* END acct_job() */
int modify_job_attr(
job *pjob, /* I (modified) */
svrattrl *plist, /* I */
int perm,
int *bad) /* O */
{
int allow_unkn = -1;
long i;
pbs_attribute newattr[JOB_ATR_LAST];
pbs_attribute *pattr;
int rc;
char log_buf[LOCAL_LOG_BUF_SIZE];
pbs_queue *pque;
if ((pque = get_jobs_queue(&pjob)) != NULL)
{
mutex_mgr pque_mutex = mutex_mgr(pque->qu_mutex, true);
if (pque->qu_qs.qu_type != QTYPE_Execution)
allow_unkn = JOB_ATR_UNKN;
}
else if (pjob->ji_parent_job != NULL)
{
allow_unkn = JOB_ATR_UNKN;
}
else
{
log_err(PBSE_JOBNOTFOUND, __func__, "Job lost while acquiring queue 5");
return(PBSE_JOBNOTFOUND);
}
pattr = pjob->ji_wattr;
/* call attr_atomic_set to decode and set a copy of the attributes */
rc = attr_atomic_set(
plist, /* I */
pattr, /* I */
newattr, /* O */
job_attr_def, /* I */
JOB_ATR_LAST,
allow_unkn, /* I */
perm, /* I */
bad); /* O */
/* if resource limits are being changed ... */
if ((rc == 0) &&
(newattr[JOB_ATR_resource].at_flags & ATR_VFLAG_SET))
{
if ((perm & (ATR_DFLAG_MGWR | ATR_DFLAG_OPWR)) == 0)
{
/* If job is running, only manager/operator can raise limits */
if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING)
{
long lim = TRUE;
int comp_resc_lt;
get_svr_attr_l(SRV_ATR_QCQLimits, &lim);
comp_resc_lt = comp_resc2(&pjob->ji_wattr[JOB_ATR_resource],
&newattr[JOB_ATR_resource],
lim,
NULL,
LESS);
if (comp_resc_lt != 0)
{
rc = PBSE_PERM;
}
}
/* Also check against queue and system limits */
if (rc == 0)
{
if ((pque = get_jobs_queue(&pjob)) != NULL)
{
mutex_mgr pque_mutex = mutex_mgr(pque->qu_mutex, true);
rc = chk_resc_limits( &newattr[JOB_ATR_resource], pque, NULL);
}
else if (pjob == NULL)
{
log_err(PBSE_JOBNOTFOUND, __func__, "Job lost while acquiring queue 6");
return(PBSE_JOBNOTFOUND);
}
else
rc = PBSE_QUENOTAVAILABLE;
}
}
} /* END if ((rc == 0) && ...) */
/* special check on permissions for hold */
if ((rc == 0) &&
(newattr[JOB_ATR_hold].at_flags & ATR_VFLAG_MODIFY))
{
i = newattr[JOB_ATR_hold].at_val.at_long ^
(pattr + JOB_ATR_hold)->at_val.at_long;
rc = chk_hold_priv(i, perm);
}
if (rc == 0)
{
for (i = 0;i < JOB_ATR_LAST;i++)
{
if (newattr[i].at_flags & ATR_VFLAG_MODIFY)
{
if (job_attr_def[i].at_action)
{
rc = job_attr_def[i].at_action(
&newattr[i],
pjob,
ATR_ACTION_ALTER);
if (rc)
break;
}
}
} /* END for (i) */
if ((rc == 0) &&
((newattr[JOB_ATR_userlst].at_flags & ATR_VFLAG_MODIFY) ||
(newattr[JOB_ATR_grouplst].at_flags & ATR_VFLAG_MODIFY)))
{
/* need to reset execution uid and gid */
rc = set_jobexid(pjob, newattr, NULL);
}
if ((rc == 0) &&
(newattr[JOB_ATR_outpath].at_flags & ATR_VFLAG_MODIFY))
{
/* need to recheck if JOB_ATR_outpath is a special case of host only */
if (newattr[JOB_ATR_outpath].at_val.at_str[strlen(newattr[JOB_ATR_outpath].at_val.at_str) - 1] == ':')
{
std::string ds = "";
newattr[JOB_ATR_outpath].at_val.at_str = strdup(prefix_std_file(pjob, ds, (int)'o'));
}
/*
* if the output path was specified and ends with a '/'
* then append the standard file name
*/
else if (newattr[JOB_ATR_outpath].at_val.at_str[strlen(newattr[JOB_ATR_outpath].at_val.at_str) - 1] == '/')
{
std::string ds = "";
newattr[JOB_ATR_outpath].at_val.at_str[strlen(newattr[JOB_ATR_outpath].at_val.at_str) - 1] = '\0';
replace_attr_string(&newattr[JOB_ATR_outpath],
(strdup(add_std_filename(pjob, newattr[JOB_ATR_outpath].at_val.at_str, (int)'o', ds))));
}
}
if ((rc == 0) &&
(newattr[JOB_ATR_errpath].at_flags & ATR_VFLAG_MODIFY))
{
/* need to recheck if JOB_ATR_errpath is a special case of host only */
if (newattr[JOB_ATR_errpath].at_val.at_str[strlen(newattr[JOB_ATR_errpath].at_val.at_str) - 1] == ':')
{
std::string ds = "";
newattr[JOB_ATR_errpath].at_val.at_str = strdup(prefix_std_file(pjob, ds, (int)'e'));
}
/*
* if the error path was specified and ends with a '/'
* then append the standard file name
*/
else if (newattr[JOB_ATR_errpath].at_val.at_str[strlen(newattr[JOB_ATR_errpath].at_val.at_str) - 1] == '/')
{
std::string ds = "";
newattr[JOB_ATR_errpath].at_val.at_str[strlen(newattr[JOB_ATR_errpath].at_val.at_str) - 1] = '\0';
replace_attr_string(&newattr[JOB_ATR_errpath],
(strdup(add_std_filename(pjob, newattr[JOB_ATR_errpath].at_val.at_str,(int)'e', ds))));
}
}
} /* END if (rc == 0) */
if (rc != 0)
{
for (i = 0;i < JOB_ATR_LAST;i++)
job_attr_def[i].at_free(newattr + i);
/* FAILURE */
return(rc);
} /* END if (rc != 0) */
/* OK, now copy the new values into the job attribute array */
for (i = 0;i < JOB_ATR_LAST;i++)
{
if (newattr[i].at_flags & ATR_VFLAG_MODIFY)
{
if (LOGLEVEL >= 7)
{
sprintf(log_buf, "attr %s modified", job_attr_def[i].at_name);
log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,pjob->ji_qs.ji_jobid,log_buf);
}
job_attr_def[i].at_free(pattr + i);
if ((newattr[i].at_type == ATR_TYPE_LIST) ||
(newattr[i].at_type == ATR_TYPE_RESC))
{
list_move(
&newattr[i].at_val.at_list,
&(pattr + i)->at_val.at_list);
}
else
{
*(pattr + i) = newattr[i];
}
(pattr + i)->at_flags = newattr[i].at_flags;
}
} /* END for (i) */
/* note, the newattr[] attributes are on the stack, they go away automatically */
pjob->ji_modified = 1;
return(PBSE_NONE);
} /* END modify_job_attr() */
int execute_job_delete(
job *pjob, /* M */
char *Msg, /* I */
struct batch_request *preq) /* I */
{
struct work_task *pwtnew;
int rc;
const char *sigt = "SIGTERM";
char log_buf[LOCAL_LOG_BUF_SIZE];
time_t time_now = time(NULL);
long force_cancel = FALSE;
long array_compatible = FALSE;
chk_job_req_permissions(&pjob,preq);
if (pjob == NULL)
{
/* preq is rejected in chk_job_req_permissions here */
return(-1);
}
mutex_mgr job_mutex(pjob->ji_mutex, true);
if (LOGLEVEL >= 10)
log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_QUEUE, __func__, pjob->ji_qs.ji_jobid);
if (pjob->ji_qs.ji_state == JOB_STATE_TRANSIT)
{
/* see note in req_delete - not sure this is possible still,
* but the deleted code is irrelevant now. I will leave this
* part --dbeer */
return(-1);
}
if (pjob->ji_qs.ji_substate == JOB_SUBSTATE_PRERUN ||
pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN ||
pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN1 ||
pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN2 ||
pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN3 )
{
/* If JOB_SUBSTATE_PRERUN being sent to MOM, wait till she gets it going */
/* retry in one second */
/* If JOB_SUBSTATE_RERUN, RERUN1, RERUN2 or RERUN3 the
job is being requeued. Wait until finished */
static time_t cycle_check_when = 0;
static char cycle_check_jid[PBS_MAXSVRJOBID + 1];
if (cycle_check_when != 0)
{
if (!strcmp(pjob->ji_qs.ji_jobid, cycle_check_jid) &&
(time_now - cycle_check_when > 10))
{
/* state not updated after 10 seconds */
/* did the mom ever get it? delete it anyways... */
cycle_check_jid[0] = '\0';
cycle_check_when = 0;
goto jump;
}
if (time_now - cycle_check_when > 20)
{
/* give up after 20 seconds */
cycle_check_jid[0] = '\0';
cycle_check_when = 0;
}
} /* END if (cycle_check_when != 0) */
if (cycle_check_when == 0)
{
/* new PRERUN job located */
cycle_check_when = time_now;
strcpy(cycle_check_jid, pjob->ji_qs.ji_jobid);
}
sprintf(log_buf, "job cannot be deleted, state=PRERUN, requeuing delete request");
log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,pjob->ji_qs.ji_jobid,log_buf);
pwtnew = set_task(WORK_Timed,time_now + 1,post_delete_route,preq,FALSE);
if (pwtnew == NULL)
{
req_reject(PBSE_SYSTEM, 0, preq, NULL, NULL);
return(-1);
}
else
{
return(ROUTE_DELETE);
}
} /* END if (pjob->ji_qs.ji_substate == JOB_SUBSTATE_PRERUN) */
jump:
/*
* Log delete and if requesting client is not job owner, send mail.
*/
sprintf(log_buf, "requestor=%s@%s", preq->rq_user, preq->rq_host);
/* NOTE: should annotate accounting record with extend message (NYI) */
account_record(PBS_ACCT_DEL, pjob, log_buf);
sprintf(log_buf, msg_manager, msg_deletejob, preq->rq_user, preq->rq_host);
log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,pjob->ji_qs.ji_jobid,log_buf);
/* NOTE: should incorporate job delete message */
if (Msg != NULL)
{
/* have text message in request extension, add it */
int len = strlen(log_buf);
snprintf(log_buf + len, sizeof(log_buf) - len, "\n%s", Msg);
}
if ((svr_chk_owner(preq, pjob) != 0) &&
(pjob->ji_has_delete_nanny == FALSE))
{
/* only send email if owner did not delete job and job deleted
has not been previously attempted */
svr_mailowner(pjob, MAIL_DEL, MAIL_FORCE, log_buf);
/*
* If we sent mail and already sent the extra message
* then reset message so we don't trigger a redundant email
* in job_abt()
*/
if (Msg != NULL)
{
Msg = NULL;
}
}
if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHECKPOINT_FILE) != 0)
{
/* job has restart file at mom, change restart comment if failed */
change_restart_comment_if_needed(pjob);
}
if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING)
{
/*
* setup a nanny task to make sure the job is actually deleted (see the
* comments at job_delete_nanny()).
*/
if (pjob->ji_has_delete_nanny == TRUE)
{
req_reject(PBSE_IVALREQ, 0, preq, NULL, "job cancel in progress");
return(-1);
}
apply_job_delete_nanny(pjob, time_now + 60);
/*
* Send signal request to MOM. The server will automagically
* pick up and "finish" off the client request when MOM replies.
*/
get_batch_request_id(preq);
if ((rc = issue_signal(&pjob, sigt, post_delete_mom1, strdup(preq->rq_id))))
{
/* cant send to MOM */
req_reject(rc, 0, preq, NULL, NULL);
}
/* normally will ack reply when mom responds */
if (pjob != NULL)
{
sprintf(log_buf, msg_delrunjobsig, sigt);
log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,pjob->ji_qs.ji_jobid,log_buf);
}
else
job_mutex.set_lock_on_exit(false);
return(-1);
} /* END if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING) */
/* make a cleanup task if set */
get_svr_attr_l(SRV_ATR_JobForceCancelTime, &force_cancel);
if (force_cancel > 0)
{
char *dup_jobid = strdup(pjob->ji_qs.ji_jobid);
set_task(WORK_Timed, time_now + force_cancel, ensure_deleted, dup_jobid, FALSE);
}
/* if configured, and this job didn't have a slot limit hold, free a job
* held with the slot limit hold */
get_svr_attr_l(SRV_ATR_MoabArrayCompatible, &array_compatible);
if ((array_compatible != FALSE) &&
((pjob->ji_wattr[JOB_ATR_hold].at_val.at_long & HOLD_l) == FALSE))
{
if ((pjob->ji_arraystructid[0] != '\0') &&
(pjob->ji_is_array_template == FALSE))
{
int i;
int newstate;
int newsub;
job *tmp;
job_array *pa = get_jobs_array(&pjob);
if (pjob == NULL)
{
job_mutex.set_lock_on_exit(false);
return(-1);
}
for (i = 0; i < pa->ai_qs.array_size; i++)
{
if (pa->job_ids[i] == NULL)
continue;
if (!strcmp(pa->job_ids[i], pjob->ji_qs.ji_jobid))
continue;
if ((tmp = svr_find_job(pa->job_ids[i], FALSE)) == NULL)
{
free(pa->job_ids[i]);
pa->job_ids[i] = NULL;
}
else
{
if (tmp->ji_wattr[JOB_ATR_hold].at_val.at_long & HOLD_l)
{
tmp->ji_wattr[JOB_ATR_hold].at_val.at_long &= ~HOLD_l;
if (tmp->ji_wattr[JOB_ATR_hold].at_val.at_long == 0)
{
tmp->ji_wattr[JOB_ATR_hold].at_flags &= ~ATR_VFLAG_SET;
}
svr_evaljobstate(tmp, &newstate, &newsub, 1);
svr_setjobstate(tmp, newstate, newsub, FALSE);
job_save(tmp, SAVEJOB_FULL, 0);
unlock_ji_mutex(tmp, __func__, "5", LOGLEVEL);
break;
}
unlock_ji_mutex(tmp, __func__, "6", LOGLEVEL);
}
}
unlock_ai_mutex(pa, __func__, "1", LOGLEVEL);
}
} /* END MoabArrayCompatible check */
if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHECKPOINT_FILE) != 0)
{
/* job has restart file at mom, do end job processing */
svr_setjobstate(pjob, JOB_STATE_EXITING, JOB_SUBSTATE_EXITING, FALSE);
/* force new connection */
pjob->ji_momhandle = -1;
if (LOGLEVEL >= 7)
{
sprintf(log_buf, "calling on_job_exit from %s", __func__);
log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf);
}
set_task(WORK_Immed, 0, on_job_exit_task, strdup(pjob->ji_qs.ji_jobid), FALSE);
}
else if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_StagedIn) != 0)
{
/* job has staged-in file, should remove them */
remove_stagein(&pjob);
job_mutex.set_lock_on_exit(false);
if (pjob != NULL)
job_abt(&pjob, Msg);
}
else
{
/*
* the job is not transitting (though it may have been) and
* is not running, so put in into a complete state.
*/
struct pbs_queue *pque;
int KeepSeconds = 0;
svr_setjobstate(pjob, JOB_STATE_COMPLETE, JOB_SUBSTATE_COMPLETE, FALSE);
if ((pque = get_jobs_queue(&pjob)) != NULL)
{
pque->qu_numcompleted++;
unlock_queue(pque, __func__, NULL, LOGLEVEL);
if (LOGLEVEL >= 7)
{
sprintf(log_buf, "calling on_job_exit from %s", __func__);
log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf);
}
pthread_mutex_lock(server.sv_attr_mutex);
KeepSeconds = attr_ifelse_long(
&pque->qu_attr[QE_ATR_KeepCompleted],
&server.sv_attr[SRV_ATR_KeepCompleted],
0);
pthread_mutex_unlock(server.sv_attr_mutex);
}
else
KeepSeconds = 0;
if (pjob != NULL)
{
set_task(WORK_Timed, time_now + KeepSeconds, on_job_exit_task, strdup(pjob->ji_qs.ji_jobid), FALSE);
}
else
job_mutex.set_lock_on_exit(false);
} /* END else if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHECKPOINT_FILE) != 0) */
return(PBSE_NONE);
} /* END execute_job_delete() */
int job_route(
job *jobp) /* job to route */
{
int bad_state = 0;
time_t life;
time_t time_now = time(NULL);
char log_buf[LOCAL_LOG_BUF_SIZE];
struct pbs_queue *qp;
long retry_time;
if (LOGLEVEL >= 8)
{
sprintf(log_buf, "%s", jobp->ji_qs.ji_jobid);
log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf);
}
qp = get_jobs_queue(&jobp);
if (jobp == NULL)
{
return(PBSE_JOB_RECYCLED);
}
if (qp == NULL)
{
return(PBSE_BADSTATE);
}
mutex_mgr qp_mutex = mutex_mgr(qp->qu_mutex, true);
/* see if the job is able to be routed */
switch (jobp->ji_qs.ji_state)
{
case JOB_STATE_TRANSIT:
return(0); /* already going, ignore it */
/*NOTREACHED*/
break;
case JOB_STATE_QUEUED:
/* NO-OP */
break; /* ok to try */
case JOB_STATE_HELD:
/* job may be acceptable */
bad_state = !qp->qu_attr[QR_ATR_RouteHeld].at_val.at_long;
break;
case JOB_STATE_WAITING:
/* job may be acceptable */
bad_state = !qp->qu_attr[QR_ATR_RouteWaiting].at_val.at_long;
break;
default:
snprintf(log_buf, sizeof(log_buf), "%s %d %s",
pbse_to_txt(PBSE_BADSTATE), jobp->ji_qs.ji_state, __func__);
log_event(PBSEVENT_DEBUG,PBS_EVENTCLASS_JOB,jobp->ji_qs.ji_jobid,log_buf);
return(PBSE_NONE);
/*NOTREACHED*/
break;
}
/* check the queue limits, can we route any (more) */
if (qp->qu_attr[QA_ATR_Started].at_val.at_long == 0)
{
/* queue not started - no routing */
return(0);
}
if ((qp->qu_attr[QA_ATR_MaxRun].at_flags & ATR_VFLAG_SET) &&
(qp->qu_attr[QA_ATR_MaxRun].at_val.at_long <= qp->qu_njstate[JOB_STATE_TRANSIT]))
{
/* max number of jobs being routed */
return(0);
}
/* what is the retry time and life time of a job in this queue */
if (qp->qu_attr[QR_ATR_RouteRetryTime].at_flags & ATR_VFLAG_SET)
{
retry_time =
(long)time_now +
qp->qu_attr[QR_ATR_RouteRetryTime].at_val.at_long;
}
else
{
retry_time = (long)time_now + PBS_NET_RETRY_TIME;
}
if (qp->qu_attr[QR_ATR_RouteLifeTime].at_flags & ATR_VFLAG_SET)
{
life =
jobp->ji_qs.ji_un.ji_routet.ji_quetime +
qp->qu_attr[QR_ATR_RouteLifeTime].at_val.at_long;
}
else
{
life = 0; /* forever */
}
if (life && (life < time_now))
{
log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,jobp->ji_qs.ji_jobid,msg_routexceed);
/* job too long in queue */
return(PBSE_ROUTEEXPD);
}
if (bad_state)
{
/* not currently routing this job */
return(PBSE_NONE);
}
/* default_router and site_alt_router expect the queue to
be unlocked. */
if (qp->qu_attr[QR_ATR_AltRouter].at_val.at_long == 0)
{
qp_mutex.unlock();
return(default_router(jobp, qp, retry_time));
}
qp_mutex.unlock();
return(site_alt_router(jobp, qp, retry_time));
} /* END job_route() */
int delete_inactive_job(
job **pjob_ptr,
const char *Msg)
{
job *pjob;
char log_buf[LOCAL_LOG_BUF_SIZE];
if (pjob_ptr == NULL)
return(PBSE_BAD_PARAMETER);
pjob = *pjob_ptr;
if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHECKPOINT_FILE) != 0)
{
/* job has restart file at mom, do end job processing */
svr_setjobstate(pjob, JOB_STATE_EXITING, JOB_SUBSTATE_EXITING, FALSE);
/* force new connection */
pjob->ji_momhandle = -1;
if (LOGLEVEL >= 7)
{
sprintf(log_buf, "calling on_job_exit from %s", __func__);
log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf);
}
set_task(WORK_Immed, 0, on_job_exit_task, strdup(pjob->ji_qs.ji_jobid), FALSE);
}
else if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_StagedIn) != 0)
{
/* job has staged-in file, should remove them */
remove_stagein(&pjob);
if (pjob != NULL)
job_abt(&pjob, Msg);
}
else
{
/*
* the job is not transitting (though it may have been) and
* is not running, so put in into a complete state.
*/
struct pbs_queue *pque;
int KeepSeconds = 0;
svr_setjobstate(pjob, JOB_STATE_COMPLETE, JOB_SUBSTATE_COMPLETE, FALSE);
if ((pque = get_jobs_queue(&pjob)) != NULL)
{
unlock_queue(pque, __func__, NULL, LOGLEVEL);
if (LOGLEVEL >= 7)
{
sprintf(log_buf, "calling on_job_exit from %s", __func__);
log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf);
}
pthread_mutex_lock(server.sv_attr_mutex);
KeepSeconds = attr_ifelse_long(
&pque->qu_attr[QE_ATR_KeepCompleted],
&server.sv_attr[SRV_ATR_KeepCompleted],
0);
pthread_mutex_unlock(server.sv_attr_mutex);
}
else
KeepSeconds = 0;
if (pjob != NULL)
set_task(WORK_Timed, time(NULL) + KeepSeconds, on_job_exit_task, strdup(pjob->ji_qs.ji_jobid), FALSE);
} /* END else if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHECKPOINT_FILE) != 0) */
if (pjob == NULL)
*pjob_ptr = NULL;
return(PBSE_NONE);
} /* END delete_inactive_job() */
void post_delete_mom1(
batch_request *preq_sig)
{
int delay = 0;
int dellen = strlen(deldelaystr);
job *pjob;
pbs_queue *pque;
char *preq_clt_id;
struct batch_request *preq_clt = NULL; /* original client request */
int rc;
time_t time_now = time(NULL);
if (preq_sig == NULL)
return;
rc = preq_sig->rq_reply.brp_code;
preq_clt_id = (char *)preq_sig->rq_extra;
free_br(preq_sig);
if (preq_clt_id != NULL)
{
preq_clt = get_remove_batch_request(preq_clt_id);
free(preq_clt_id);
}
/* the client request has been handled another way, nothing left to do */
if (preq_clt == NULL)
return;
pjob = svr_find_job(preq_clt->rq_ind.rq_delete.rq_objname, FALSE);
if (pjob == NULL)
{
/* job has gone away */
req_reject(PBSE_UNKJOBID, 0, preq_clt, NULL, NULL);
return;
}
mutex_mgr job_mutex(pjob->ji_mutex, true);
if (rc)
{
/* mom rejected request */
if (rc == PBSE_UNKJOBID)
{
/* MOM claims no knowledge, so just purge it */
log_event(
PBSEVENT_JOB,
PBS_EVENTCLASS_JOB,
pjob->ji_qs.ji_jobid,
"MOM rejected signal during delete");
/* removed the resources assigned to job */
free_nodes(pjob);
set_resc_assigned(pjob, DECR);
job_mutex.set_lock_on_exit(false);
svr_job_purge(pjob);
reply_ack(preq_clt);
}
else
req_reject(rc, 0, preq_clt, NULL, NULL);
return;
}
if (preq_clt->rq_extend)
{
if (strncmp(preq_clt->rq_extend, deldelaystr, dellen) == 0)
{
delay = atoi(preq_clt->rq_extend + dellen);
}
}
reply_ack(preq_clt); /* dont need it, reply now */
/*
* if no delay specified in original request, see if kill_delay
* queue attribute is set.
*/
if (delay == 0)
{
if ((pque = get_jobs_queue(&pjob)) != NULL)
{
mutex_mgr pque_mutex = mutex_mgr(pque->qu_mutex, true);
pthread_mutex_lock(server.sv_attr_mutex);
delay = attr_ifelse_long(&pque->qu_attr[QE_ATR_KillDelay],
&server.sv_attr[SRV_ATR_KillDelay],
2);
pthread_mutex_unlock(server.sv_attr_mutex);
}
else if (pjob == NULL)
{
job_mutex.set_lock_on_exit(false);
return;
}
}
set_task(WORK_Timed, delay + time_now, post_delete_mom2, strdup(pjob->ji_qs.ji_jobid), FALSE);
/*
* Since the first signal has succeeded, let's reschedule the
* nanny to be 1 minute after the second phase.
*/
apply_job_delete_nanny(pjob, time_now + delay + 60);
} /* END post_delete_mom1() */
int req_rerunjob(
struct batch_request *preq)
{
int rc = PBSE_NONE;
job *pjob;
int MgrRequired = TRUE;
char log_buf[LOCAL_LOG_BUF_SIZE];
/* check if requestor is admin, job owner, etc */
if ((pjob = chk_job_request(preq->rq_ind.rq_rerun, preq)) == 0)
{
/* FAILURE */
/* chk_job_request calls req_reject() */
rc = PBSE_SYSTEM;
return rc; /* This needs to fixed to return an accurate error */
}
/* the job must be running or completed */
if (pjob->ji_qs.ji_state >= JOB_STATE_EXITING)
{
if (pjob->ji_wattr[JOB_ATR_checkpoint_name].at_flags & ATR_VFLAG_SET)
{
/* allow end-users to rerun checkpointed jobs */
MgrRequired = FALSE;
}
}
else if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING)
{
/* job is running */
/* NO-OP */
}
else
{
/* FAILURE - job is in bad state */
rc = PBSE_BADSTATE;
snprintf(log_buf, LOCAL_LOG_BUF_SIZE, "job %s is in a bad state",
preq->rq_ind.rq_rerun);
req_reject(rc, 0, preq, NULL, log_buf);
unlock_ji_mutex(pjob, __func__, "2", LOGLEVEL);
return rc;
}
if ((MgrRequired == TRUE) &&
((preq->rq_perm & (ATR_DFLAG_MGWR | ATR_DFLAG_OPWR)) == 0))
{
/* FAILURE */
rc = PBSE_PERM;
snprintf(log_buf, LOCAL_LOG_BUF_SIZE,
"additional permissions required (ATR_DFLAG_MGWR | ATR_DFLAG_OPWR)");
req_reject(rc, 0, preq, NULL, log_buf);
unlock_ji_mutex(pjob, __func__, "3", LOGLEVEL);
return rc;
}
/* the job must be rerunnable */
if (pjob->ji_wattr[JOB_ATR_rerunable].at_val.at_long == 0)
{
/* NOTE: should force override this constraint? maybe (???) */
/* no, the user is saying that the job will break, and
IEEE Std 1003.1 specifically says rerun is to be rejected
if rerunable==FALSE -garrick */
rc = PBSE_NORERUN;
snprintf(log_buf, LOCAL_LOG_BUF_SIZE, "job %s not rerunnable",
preq->rq_ind.rq_rerun);
req_reject(rc, 0, preq, NULL, log_buf);
unlock_ji_mutex(pjob, __func__, "4", LOGLEVEL);
return rc;
}
if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING)
{
/* ask MOM to kill off the job if it is running */
int delay = 0;
pbs_queue *pque;
if ((pque = get_jobs_queue(&pjob)) != NULL)
{
mutex_mgr pque_mutex = mutex_mgr(pque->qu_mutex, true);
pthread_mutex_lock(server.sv_attr_mutex);
delay = attr_ifelse_long(&pque->qu_attr[QE_ATR_KillDelay],
&server.sv_attr[SRV_ATR_KillDelay],
0);
pthread_mutex_unlock(server.sv_attr_mutex);
}
else if (pjob == NULL)
{
rc = PBSE_NORERUN;
unlock_ji_mutex(pjob, __func__, "4", LOGLEVEL);
return rc;
}
if(delay != 0)
{
static const char *rerun = "rerun";
char *extra = strdup(rerun);
get_batch_request_id(preq);
if ((rc = issue_signal(&pjob, "SIGTERM", delay_and_send_sig_kill, extra, strdup(preq->rq_id))))
{
/* cant send to MOM */
req_reject(rc, 0, preq, NULL, NULL);
}
unlock_ji_mutex(pjob, __func__, "4", LOGLEVEL);
return rc;
}
else
{
static const char *rerun = "rerun";
char *extra = strdup(rerun);
rc = issue_signal(&pjob, "SIGKILL", post_rerun, extra, NULL);
}
}
else
{
if (pjob->ji_wattr[JOB_ATR_hold].at_val.at_long == HOLD_n)
{
svr_setjobstate(pjob, JOB_STATE_QUEUED, JOB_SUBSTATE_QUEUED, FALSE);
}
else
{
svr_setjobstate(pjob, JOB_STATE_HELD, JOB_SUBSTATE_HELD, FALSE);
}
/* reset some job attributes */
pjob->ji_wattr[JOB_ATR_comp_time].at_flags &= ~ATR_VFLAG_SET;
pjob->ji_wattr[JOB_ATR_reported].at_flags &= ~ATR_VFLAG_SET;
set_statechar(pjob);
rc = -1;
}
return finalize_rerunjob(preq,pjob,rc);
}
void delay_and_send_sig_kill(batch_request *preq_sig)
{
int delay = 0;
job *pjob;
pbs_queue *pque;
struct batch_request *preq_clt = NULL; /* original client request */
int rc;
time_t time_now = time(NULL);
if (preq_sig == NULL)
return;
rc = preq_sig->rq_reply.brp_code;
if (preq_sig->rq_extend != NULL)
{
preq_clt = get_remove_batch_request(preq_sig->rq_extend);
}
free_br(preq_sig);
/* the client request has been handled another way, nothing left to do */
if (preq_clt == NULL)
return;
if ((pjob = chk_job_request(preq_clt->rq_ind.rq_rerun, preq_clt)) == NULL)
{
/* job has gone away */
req_reject(PBSE_UNKJOBID, 0, preq_clt, NULL, NULL);
return;
}
if (rc)
{
/* mom rejected request */
if (rc == PBSE_UNKJOBID)
{
/* MOM claims no knowledge, so just purge it */
log_event(
PBSEVENT_JOB,
PBS_EVENTCLASS_JOB,
pjob->ji_qs.ji_jobid,
"MOM rejected signal during rerun");
/* removed the resources assigned to job */
free_nodes(pjob);
set_resc_assigned(pjob, DECR);
unlock_ji_mutex(pjob, __func__, "3", LOGLEVEL);
svr_job_purge(pjob);
reply_ack(preq_clt);
}
else
{
unlock_ji_mutex(pjob, __func__, "3", LOGLEVEL);
req_reject(rc, 0, preq_clt, NULL, NULL);
}
return;
}
if ((pque = get_jobs_queue(&pjob)) != NULL)
{
mutex_mgr pque_mutex = mutex_mgr(pque->qu_mutex, true);
pthread_mutex_lock(server.sv_attr_mutex);
delay = attr_ifelse_long(&pque->qu_attr[QE_ATR_KillDelay],
&server.sv_attr[SRV_ATR_KillDelay],
0);
pthread_mutex_unlock(server.sv_attr_mutex);
}
else if (pjob == NULL)
{
unlock_ji_mutex(pjob, __func__, "3", LOGLEVEL);
return;
}
unlock_ji_mutex(pjob, __func__, "3", LOGLEVEL);
reply_ack(preq_clt);
set_task(WORK_Timed, delay + time_now, send_sig_kill, strdup(pjob->ji_qs.ji_jobid), FALSE);
}
void rerun_or_kill(
job **pjob_ptr, /* I (modified/freed) */
char *text) /* I */
{
long server_state = SV_STATE_DOWN;
char log_buf[LOCAL_LOG_BUF_SIZE];
pbs_queue *pque;
job *pjob = *pjob_ptr;
get_svr_attr_l(SRV_ATR_State, &server_state);
if (pjob->ji_wattr[JOB_ATR_rerunable].at_val.at_long)
{
/* job is rerunable, mark it to be requeued */
issue_signal(&pjob, "SIGKILL", free_br, NULL);
if (pjob != NULL)
{
pjob->ji_qs.ji_substate = JOB_SUBSTATE_RERUN;
if ((pque = get_jobs_queue(&pjob)) != NULL)
{
snprintf(log_buf, sizeof(log_buf), "%s%s%s", msg_init_queued, pque->qu_qs.qu_name, text);
unlock_queue(pque, __func__, NULL, LOGLEVEL);
}
}
}
else if (server_state != SV_STATE_SHUTDEL)
{
/* job not rerunable, immediate shutdown - kill it off */
snprintf(log_buf, sizeof(log_buf), "%s%s", msg_job_abort, text);
/* need to record log message before purging job */
log_event(
PBSEVENT_SYSTEM | PBSEVENT_JOB | PBSEVENT_DEBUG,
PBS_EVENTCLASS_JOB,
pjob->ji_qs.ji_jobid,
log_buf);
job_abt(pjob_ptr, log_buf);
return;
}
else
{
/* delayed shutdown, leave job running */
snprintf(log_buf, sizeof(log_buf), "%s%s", msg_leftrunning, text);
}
if (pjob != NULL)
{
log_event(
PBSEVENT_SYSTEM | PBSEVENT_JOB | PBSEVENT_DEBUG,
PBS_EVENTCLASS_JOB,
pjob->ji_qs.ji_jobid,
log_buf);
}
return;
} /* END rerun_or_kill() */
int req_orderjob(
struct batch_request *vp) /* I */
{
job *pjob;
job *pjob1;
job *pjob2;
int rank;
int rc = 0;
char tmpqn[PBS_MAXQUEUENAME+1];
struct batch_request *req = (struct batch_request *)vp;
char log_buf[LOCAL_LOG_BUF_SIZE];
pbs_queue *pque1;
pbs_queue *pque2;
bool reservation1 = false;
bool reservation2 = false;
if ((pjob1 = chk_job_request(req->rq_ind.rq_move.rq_jid, req)) == NULL)
{
return(PBSE_NONE);
}
mutex_mgr job1_mutex(pjob1->ji_mutex, true);
if ((pjob2 = chk_job_request(req->rq_ind.rq_move.rq_destin, req)) == NULL)
{
return(PBSE_NONE);
}
mutex_mgr job2_mutex(pjob2->ji_mutex, true);
if (((pjob = pjob1)->ji_qs.ji_state == JOB_STATE_RUNNING) ||
((pjob = pjob2)->ji_qs.ji_state == JOB_STATE_RUNNING))
{
#ifndef NDEBUG
sprintf(log_buf, "%s %d",
pbse_to_txt(PBSE_BADSTATE),
pjob->ji_qs.ji_state);
strcat(log_buf, __func__);
log_event(
PBSEVENT_DEBUG,
PBS_EVENTCLASS_JOB,
pjob->ji_qs.ji_jobid,
log_buf);
#endif /* NDEBUG */
req_reject(PBSE_BADSTATE, 0, req, NULL, NULL);
return(PBSE_NONE);
}
else if ((pjob1->ji_qhdr == NULL) || (pjob2->ji_qhdr == NULL))
{
req_reject(PBSE_BADSTATE, 0, req, NULL, "One of the jobs does not have a queue");
return(PBSE_NONE);
}
else if (pjob1->ji_qhdr != pjob2->ji_qhdr)
{
/* jobs are in different queues */
int ok = FALSE;
if ((pque2 = get_jobs_queue(&pjob2)) == NULL)
{
rc = PBSE_BADSTATE;
if (pjob2 == NULL)
job2_mutex.set_unlock_on_exit(false);
}
else
{
mutex_mgr pque2_mutex = mutex_mgr(pque2->qu_mutex, true);
if ((rc = svr_chkque(pjob1, pque2, get_variable(pjob1, pbs_o_host), MOVE_TYPE_Order, NULL)) == PBSE_NONE)
{
reservation1 = have_reservation(pjob1, pque2);
pque2_mutex.unlock();
if ((pque1 = get_jobs_queue(&pjob1)) == NULL)
{
rc = PBSE_BADSTATE;
if (pjob1 == NULL)
job1_mutex.set_unlock_on_exit(false);
}
else if (pjob1 != NULL)
{
mutex_mgr pque1_mutex = mutex_mgr(pque1->qu_mutex, true);
if ((rc = svr_chkque(pjob2, pque1, get_variable(pjob2, pbs_o_host), MOVE_TYPE_Order, NULL)) == PBSE_NONE)
{
reservation2 = have_reservation(pjob2, pque1);
ok = TRUE;
}
}
}
}
if (ok == FALSE)
{
req_reject(rc, 0, req, NULL, NULL);
return(PBSE_NONE);
}
}
/* now swap the order of the two jobs in the queue lists */
rank = pjob1->ji_wattr[JOB_ATR_qrank].at_val.at_long;
pjob1->ji_wattr[JOB_ATR_qrank].at_val.at_long =
pjob2->ji_wattr[JOB_ATR_qrank].at_val.at_long;
pjob2->ji_wattr[JOB_ATR_qrank].at_val.at_long = rank;
if (pjob1->ji_qhdr != pjob2->ji_qhdr)
{
strcpy(tmpqn, pjob1->ji_qs.ji_queue);
strcpy(pjob1->ji_qs.ji_queue, pjob2->ji_qs.ji_queue);
strcpy(pjob2->ji_qs.ji_queue, tmpqn);
svr_dequejob(pjob1, FALSE);
svr_dequejob(pjob2, FALSE);
if (svr_enquejob(pjob1, FALSE, -1, reservation1) == PBSE_JOB_RECYCLED)
{
pjob1 = NULL;
job1_mutex.set_unlock_on_exit(false);
}
if (svr_enquejob(pjob2, FALSE, -1, reservation2) == PBSE_JOB_RECYCLED)
{
pjob2 = NULL;
job2_mutex.set_unlock_on_exit(false);
}
}
else
{
if ((pque1 = get_jobs_queue(&pjob1)) != NULL)
{
mutex_mgr pque1_mutex = mutex_mgr(pque1->qu_mutex, true);
swap_jobs(pque1->qu_jobs,pjob1,pjob2);
swap_jobs(NULL,pjob1,pjob2);
}
}
/* need to update disk copy of both jobs to save new order */
if (pjob1 != NULL)
{
job_save(pjob1, SAVEJOB_FULL, 0);
}
if (pjob2 != NULL)
{
job_save(pjob2, SAVEJOB_FULL, 0);
}
/* SUCCESS */
reply_ack(req);
return(PBSE_NONE);
} /* END req_orderjob() */
void delay_and_send_sig_kill(
batch_request *preq_sig)
{
int delay = 0;
job *pjob;
pbs_queue *pque;
batch_request *preq_clt = NULL; /* original client request */
int rc;
time_t time_now = time(NULL);
char log_buf[LOCAL_LOG_BUF_SIZE];
if (preq_sig == NULL)
return;
rc = preq_sig->rq_reply.brp_code;
if (preq_sig->rq_extend != NULL)
{
preq_clt = get_remove_batch_request(preq_sig->rq_extend);
}
free_br(preq_sig);
/* the client request has been handled another way, nothing left to do */
if (preq_clt == NULL)
return;
if ((pjob = chk_job_request(preq_clt->rq_ind.rq_rerun, preq_clt)) == NULL)
{
/* job has gone away, chk_job_request() calls req_reject() on failure */
return;
}
mutex_mgr pjob_mutex = mutex_mgr(pjob->ji_mutex, true);
if (rc)
{
/* mom rejected request */
if (rc == PBSE_UNKJOBID)
{
/* MOM claims no knowledge, so just purge it */
log_event(
PBSEVENT_JOB,
PBS_EVENTCLASS_JOB,
pjob->ji_qs.ji_jobid,
"MOM rejected signal during rerun");
/* removed the resources assigned to job */
free_nodes(pjob);
set_resc_assigned(pjob, DECR);
unlock_ji_mutex(pjob, __func__, "3", LOGLEVEL);
svr_job_purge(pjob);
reply_ack(preq_clt);
}
else
{
pjob_mutex.unlock();
req_reject(rc, 0, preq_clt, NULL, NULL);
}
return;
}
// Apply the user delay first so it takes precedence.
if (pjob->ji_wattr[JOB_ATR_user_kill_delay].at_flags & ATR_VFLAG_SET)
delay = pjob->ji_wattr[JOB_ATR_user_kill_delay].at_val.at_long;
if ((pque = get_jobs_queue(&pjob)) != NULL)
{
mutex_mgr pque_mutex = mutex_mgr(pque->qu_mutex, true);
mutex_mgr server_mutex = mutex_mgr(server.sv_attr_mutex, false);
if (delay == 0)
{
delay = attr_ifelse_long(&pque->qu_attr[QE_ATR_KillDelay],
&server.sv_attr[SRV_ATR_KillDelay],
0);
}
}
else
{
/* why is the pque null. Something went wrong */
snprintf(log_buf, LOCAL_LOG_BUF_SIZE, "jobid %s returned a null queue", pjob->ji_qs.ji_jobid);
req_reject(PBSE_UNKQUE, 0, preq_clt, NULL, log_buf);
return;
}
pjob_mutex.unlock();
reply_ack(preq_clt);
set_task(WORK_Timed, delay + time_now, send_sig_kill, strdup(pjob->ji_qs.ji_jobid), FALSE);
}
