END_TEST
START_TEST(test_apply_job_delete_nanny)
{
job *pjob = (job *)calloc(1, sizeof(job));
fail_unless(apply_job_delete_nanny(pjob, -1) == -1);
fail_unless(pjob->ji_has_delete_nanny == FALSE);
fail_unless(apply_job_delete_nanny(pjob, 0) == PBSE_NONE);
fail_unless(pjob->ji_has_delete_nanny == TRUE);
pjob->ji_has_delete_nanny = FALSE;
fail_unless(apply_job_delete_nanny(pjob, 10) == PBSE_NONE);
fail_unless(pjob->ji_has_delete_nanny == TRUE);
fail_unless(apply_job_delete_nanny(pjob, 1) == PBSE_NONE);
}
static void job_delete_nanny(
struct work_task *pwt)
{
job *pjob;
char *sigk = "SIGKILL";
char *jobid;
struct batch_request *newreq;
char log_buf[LOCAL_LOG_BUF_SIZE];
time_t time_now = time(NULL);
long nanny = FALSE;
/* short-circuit if nanny isn't enabled */
get_svr_attr_l(SRV_ATR_JobNanny, &nanny);
if (!nanny)
{
jobid = (char *)pwt->wt_parm1;
if (jobid != NULL)
{
pjob = svr_find_job(jobid, FALSE);
if (pjob != NULL)
{
sprintf(log_buf, "exiting job '%s' still exists, sending a SIGKILL", pjob->ji_qs.ji_jobid);
log_err(-1, "job nanny", log_buf);
/* build up a Signal Job batch request */
if ((newreq = alloc_br(PBS_BATCH_SignalJob)) != NULL)
{
strcpy(newreq->rq_ind.rq_signal.rq_jid, pjob->ji_qs.ji_jobid);
snprintf(newreq->rq_ind.rq_signal.rq_signame, sizeof(newreq->rq_ind.rq_signal.rq_signame), "%s", sigk);
}
issue_signal(&pjob, sigk, post_job_delete_nanny, newreq);
if (pjob != NULL)
{
apply_job_delete_nanny(pjob, time_now + 60);
unlock_ji_mutex(pjob, __func__, "1", LOGLEVEL);
}
}
}
else
{
log_err(ENOMEM, __func__, "Cannot allocate memory");
}
}
if (pwt->wt_parm1 != NULL)
free(pwt->wt_parm1);
free(pwt->wt_mutex);
free(pwt);
} /* END job_delete_nanny() */
static void job_delete_nanny(
struct work_task *pwt)
{
job *pjob;
char *sigk = "SIGKILL";
struct batch_request *newreq;
/* short-circuit if nanny isn't enabled */
if (!server.sv_attr[SRV_ATR_JobNanny].at_val.at_long)
{
release_req(pwt);
return;
}
pjob = (job *)pwt->wt_parm1;
sprintf(log_buffer, "exiting job '%s' still exists, sending a SIGKILL",
pjob->ji_qs.ji_jobid);
log_err(-1, "job nanny", log_buffer);
/* build up a Signal Job batch request */
if ((newreq = alloc_br(PBS_BATCH_SignalJob)) != NULL)
{
strcpy(newreq->rq_ind.rq_signal.rq_jid, pjob->ji_qs.ji_jobid);
strncpy(newreq->rq_ind.rq_signal.rq_signame, sigk, PBS_SIGNAMESZ);
}
issue_signal(pjob, sigk, post_job_delete_nanny, newreq);
apply_job_delete_nanny(pjob, time_now + 60);
return;
} /* END job_delete_nanny() */
static void post_delete_mom1(
struct work_task *pwt)
{
int delay = 0;
int dellen = strlen(deldelaystr);
job *pjob;
pbs_queue *pque;
char *preq_clt_id;
struct batch_request *preq_sig; /* signal request to MOM */
struct batch_request *preq_clt = NULL; /* original client request */
int rc;
time_t time_now = time(NULL);
preq_sig = get_remove_batch_request((char *)pwt->wt_parm1);
free(pwt->wt_mutex);
free(pwt);
if (preq_sig == NULL)
return;
rc = preq_sig->rq_reply.brp_code;
preq_clt_id = preq_sig->rq_extra;
free_br(preq_sig);
if (preq_clt_id != NULL)
{
preq_clt = get_remove_batch_request(preq_clt_id);
free(preq_clt_id);
}
/* the client request has been handled another way, nothing left to do */
if (preq_clt == NULL)
return;
pjob = svr_find_job(preq_clt->rq_ind.rq_delete.rq_objname, FALSE);
if (pjob == NULL)
{
/* job has gone away */
req_reject(PBSE_UNKJOBID, 0, preq_clt, NULL, NULL);
return;
}
if (rc)
{
/* mom rejected request */
if (rc == PBSE_UNKJOBID)
{
/* MOM claims no knowledge, so just purge it */
log_event(
PBSEVENT_JOB,
PBS_EVENTCLASS_JOB,
pjob->ji_qs.ji_jobid,
"MOM rejected signal during delete");
/* removed the resources assigned to job */
free_nodes(pjob);
set_resc_assigned(pjob, DECR);
svr_job_purge(pjob);
reply_ack(preq_clt);
}
else
{
req_reject(rc, 0, preq_clt, NULL, NULL);
unlock_ji_mutex(pjob, __func__, "1", LOGLEVEL);
}
return;
}
if (preq_clt->rq_extend)
{
if (strncmp(preq_clt->rq_extend, deldelaystr, dellen) == 0)
{
delay = atoi(preq_clt->rq_extend + dellen);
}
}
reply_ack(preq_clt); /* dont need it, reply now */
/*
* if no delay specified in original request, see if kill_delay
* queue attribute is set.
*/
if (delay == 0)
{
if ((pque = get_jobs_queue(&pjob)) != NULL)
{
pthread_mutex_lock(server.sv_attr_mutex);
delay = attr_ifelse_long(&pque->qu_attr[QE_ATR_KillDelay],
&server.sv_attr[SRV_ATR_KillDelay],
2);
pthread_mutex_unlock(server.sv_attr_mutex);
unlock_queue(pque, __func__, NULL, LOGLEVEL);
}
else if (pjob != NULL)
return;
}
set_task(WORK_Timed, delay + time_now, post_delete_mom2, strdup(pjob->ji_qs.ji_jobid), FALSE);
/*
* Since the first signal has succeeded, let's reschedule the
* nanny to be 1 minute after the second phase.
*/
apply_job_delete_nanny(pjob, time_now + delay + 60);
unlock_ji_mutex(pjob, __func__, "2", LOGLEVEL);
} /* END post_delete_mom1() */
int execute_job_delete(
job *pjob, /* M */
char *Msg, /* I */
struct batch_request *preq) /* I */
{
struct work_task *pwtnew;
int rc;
char *sigt = "SIGTERM";
int has_mutex = TRUE;
char log_buf[LOCAL_LOG_BUF_SIZE];
time_t time_now = time(NULL);
long force_cancel = FALSE;
long array_compatible = FALSE;
chk_job_req_permissions(&pjob,preq);
if (pjob == NULL)
{
/* preq is rejected in chk_job_req_permissions here */
return(-1);
}
if (pjob->ji_qs.ji_state == JOB_STATE_TRANSIT)
{
/* see note in req_delete - not sure this is possible still,
* but the deleted code is irrelevant now. I will leave this
* part --dbeer */
unlock_ji_mutex(pjob, __func__, "1", LOGLEVEL);
return(-1);
}
if (pjob->ji_qs.ji_substate == JOB_SUBSTATE_PRERUN ||
pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN ||
pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN1 ||
pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN2 ||
pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN3 )
{
/* If JOB_SUBSTATE_PRERUN being sent to MOM, wait till she gets it going */
/* retry in one second */
/* If JOB_SUBSTATE_RERUN, RERUN1, RERUN2 or RERUN3 the
job is being requeued. Wait until finished */
static time_t cycle_check_when = 0;
static char cycle_check_jid[PBS_MAXSVRJOBID + 1];
if (cycle_check_when != 0)
{
if (!strcmp(pjob->ji_qs.ji_jobid, cycle_check_jid) &&
(time_now - cycle_check_when > 10))
{
/* state not updated after 10 seconds */
/* did the mom ever get it? delete it anyways... */
cycle_check_jid[0] = '\0';
cycle_check_when = 0;
goto jump;
}
if (time_now - cycle_check_when > 20)
{
/* give up after 20 seconds */
cycle_check_jid[0] = '\0';
cycle_check_when = 0;
}
} /* END if (cycle_check_when != 0) */
if (cycle_check_when == 0)
{
/* new PRERUN job located */
cycle_check_when = time_now;
strcpy(cycle_check_jid, pjob->ji_qs.ji_jobid);
}
sprintf(log_buf, "job cannot be deleted, state=PRERUN, requeuing delete request");
log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,pjob->ji_qs.ji_jobid,log_buf);
pwtnew = set_task(WORK_Timed,time_now + 1,post_delete_route,preq,FALSE);
unlock_ji_mutex(pjob, __func__, "2", LOGLEVEL);
if (pwtnew == NULL)
{
req_reject(PBSE_SYSTEM, 0, preq, NULL, NULL);
return(-1);
}
else
{
return(ROUTE_DELETE);
}
} /* END if (pjob->ji_qs.ji_substate == JOB_SUBSTATE_PRERUN) */
jump:
/*
* Log delete and if requesting client is not job owner, send mail.
*/
sprintf(log_buf, "requestor=%s@%s", preq->rq_user, preq->rq_host);
/* NOTE: should annotate accounting record with extend message (NYI) */
account_record(PBS_ACCT_DEL, pjob, log_buf);
sprintf(log_buf, msg_manager, msg_deletejob, preq->rq_user, preq->rq_host);
log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,pjob->ji_qs.ji_jobid,log_buf);
/* NOTE: should incorporate job delete message */
if (Msg != NULL)
{
/* have text message in request extension, add it */
strcat(log_buf, "\n");
strcat(log_buf, Msg);
}
if ((svr_chk_owner(preq, pjob) != 0) &&
(pjob->ji_has_delete_nanny == FALSE))
{
/* only send email if owner did not delete job and job deleted
has not been previously attempted */
svr_mailowner(pjob, MAIL_DEL, MAIL_FORCE, log_buf);
/*
* If we sent mail and already sent the extra message
* then reset message so we don't trigger a redundant email
* in job_abt()
*/
if (Msg != NULL)
{
Msg = NULL;
}
}
if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHECKPOINT_FILE) != 0)
{
/* job has restart file at mom, change restart comment if failed */
change_restart_comment_if_needed(pjob);
}
if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING)
{
/*
* setup a nanny task to make sure the job is actually deleted (see the
* comments at job_delete_nanny()).
*/
if (pjob->ji_has_delete_nanny == TRUE)
{
unlock_ji_mutex(pjob, __func__, "3", LOGLEVEL);
req_reject(PBSE_IVALREQ, 0, preq, NULL, "job cancel in progress");
return(-1);
}
apply_job_delete_nanny(pjob, time_now + 60);
/*
* Send signal request to MOM. The server will automagically
* pick up and "finish" off the client request when MOM replies.
*/
get_batch_request_id(preq);
if ((rc = issue_signal(&pjob, sigt, post_delete_mom1, strdup(preq->rq_id))))
{
/* cant send to MOM */
req_reject(rc, 0, preq, NULL, NULL);
}
/* normally will ack reply when mom responds */
if (pjob != NULL)
{
sprintf(log_buf, msg_delrunjobsig, sigt);
log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,pjob->ji_qs.ji_jobid,log_buf);
unlock_ji_mutex(pjob, __func__, "4", LOGLEVEL);
}
return(-1);
} /* END if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING) */
/* make a cleanup task if set */
get_svr_attr_l(SRV_ATR_JobForceCancelTime, &force_cancel);
if (force_cancel > 0)
{
char *dup_jobid = strdup(pjob->ji_qs.ji_jobid);
set_task(WORK_Timed, time_now + force_cancel, ensure_deleted, dup_jobid, FALSE);
}
/* if configured, and this job didn't have a slot limit hold, free a job
* held with the slot limit hold */
get_svr_attr_l(SRV_ATR_MoabArrayCompatible, &array_compatible);
if ((array_compatible != FALSE) &&
((pjob->ji_wattr[JOB_ATR_hold].at_val.at_long & HOLD_l) == FALSE))
{
if ((pjob->ji_arraystruct != NULL) &&
(pjob->ji_is_array_template == FALSE))
{
int i;
int newstate;
int newsub;
job *tmp;
job_array *pa = get_jobs_array(&pjob);
if (pjob == NULL)
return(-1);
for (i = 0; i < pa->ai_qs.array_size; i++)
{
if (pa->job_ids[i] == NULL)
continue;
if (!strcmp(pa->job_ids[i], pjob->ji_qs.ji_jobid))
continue;
if ((tmp = svr_find_job(pa->job_ids[i], FALSE)) == NULL)
{
free(pa->job_ids[i]);
pa->job_ids[i] = NULL;
}
else
{
if (tmp->ji_wattr[JOB_ATR_hold].at_val.at_long & HOLD_l)
{
tmp->ji_wattr[JOB_ATR_hold].at_val.at_long &= ~HOLD_l;
if (tmp->ji_wattr[JOB_ATR_hold].at_val.at_long == 0)
{
tmp->ji_wattr[JOB_ATR_hold].at_flags &= ~ATR_VFLAG_SET;
}
svr_evaljobstate(tmp, &newstate, &newsub, 1);
svr_setjobstate(tmp, newstate, newsub, FALSE);
job_save(tmp, SAVEJOB_FULL, 0);
unlock_ji_mutex(tmp, __func__, "5", LOGLEVEL);
break;
}
unlock_ji_mutex(tmp, __func__, "6", LOGLEVEL);
}
}
if (LOGLEVEL >= 7)
{
sprintf(log_buf, "%s: unlocking ai_mutex", __func__);
log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf);
}
pthread_mutex_unlock(pa->ai_mutex);
}
} /* END MoabArrayCompatible check */
if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHECKPOINT_FILE) != 0)
{
/* job has restart file at mom, do end job processing */
svr_setjobstate(pjob, JOB_STATE_EXITING, JOB_SUBSTATE_EXITING, FALSE);
/* force new connection */
pjob->ji_momhandle = -1;
if (LOGLEVEL >= 7)
{
sprintf(log_buf, "calling on_job_exit from %s", __func__);
log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf);
}
set_task(WORK_Immed, 0, on_job_exit, strdup(pjob->ji_qs.ji_jobid), FALSE);
}
else if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_StagedIn) != 0)
{
/* job has staged-in file, should remove them */
remove_stagein(&pjob);
if (pjob != NULL)
job_abt(&pjob, Msg);
has_mutex = FALSE;
}
else
{
/*
* the job is not transitting (though it may have been) and
* is not running, so put in into a complete state.
*/
struct pbs_queue *pque;
int KeepSeconds = 0;
svr_setjobstate(pjob, JOB_STATE_COMPLETE, JOB_SUBSTATE_COMPLETE, FALSE);
if ((pque = get_jobs_queue(&pjob)) != NULL)
{
pque->qu_numcompleted++;
unlock_queue(pque, __func__, NULL, LOGLEVEL);
if (LOGLEVEL >= 7)
{
sprintf(log_buf, "calling on_job_exit from %s", __func__);
log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf);
}
pthread_mutex_lock(server.sv_attr_mutex);
KeepSeconds = attr_ifelse_long(
&pque->qu_attr[QE_ATR_KeepCompleted],
&server.sv_attr[SRV_ATR_KeepCompleted],
0);
pthread_mutex_unlock(server.sv_attr_mutex);
}
else
KeepSeconds = 0;
if (pjob != NULL)
{
set_task(WORK_Timed, time_now + KeepSeconds, on_job_exit, strdup(pjob->ji_qs.ji_jobid), FALSE);
}
else
has_mutex = FALSE;
} /* END else if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHECKPOINT_FILE) != 0) */
if (has_mutex == TRUE)
unlock_ji_mutex(pjob, __func__, "7", LOGLEVEL);
return(PBSE_NONE);
} /* END execute_job_delete() */
/**
* attempt_delete()
* deletes a job differently depending on the job's state
*
* @return TRUE if the job was deleted, FALSE if skipped
* @param pjob - a pointer to the job being handled
*/
int attempt_delete(
void *j) /* I */
{
int skipped = FALSE;
struct work_task *pwtold;
struct work_task *pwtnew;
job *pjob;
/* job considered deleted if null */
if (j == NULL)
return(TRUE);
pjob = (job *)j;
if (pjob->ji_qs.ji_state == JOB_STATE_TRANSIT)
{
/*
* Find pid of router from existing work task entry,
* then establish another work task on same child.
* Next, signal the router and wait for its completion;
*/
pwtold = (struct work_task *)GET_NEXT(pjob->ji_svrtask);
while (pwtold != NULL)
{
if ((pwtold->wt_type == WORK_Deferred_Child) ||
(pwtold->wt_type == WORK_Deferred_Cmp))
{
kill((pid_t)pwtold->wt_event, SIGTERM);
pjob->ji_qs.ji_substate = JOB_SUBSTATE_ABORT;
}
pwtold = (struct work_task *)GET_NEXT(pwtold->wt_linkobj);
}
skipped = TRUE;
return(!skipped);
} /* END if (pjob->ji_qs.ji_state == JOB_SUBSTATE_TRANSIT) */
else if (pjob->ji_qs.ji_substate == JOB_SUBSTATE_PRERUN)
{
/* we'll wait for the mom to get this job, then delete it */
skipped = TRUE;
} /* END if (pjob->ji_qs.ji_substate == JOB_SUBSTATE_PRERUN) */
else if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING)
{
/* set up nanny */
if (!has_job_delete_nanny(pjob))
{
apply_job_delete_nanny(pjob, time_now + 60);
/* need to issue a signal to the mom, but we don't want to sent an ack to the
* client when the mom replies */
issue_signal(pjob, "SIGTERM", post_delete, NULL);
}
if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHECKPOINT_FILE) != 0)
{
/* job has restart file at mom, change restart comment if failed */
change_restart_comment_if_needed(pjob);
}
return(!skipped);
} /* END if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING) */
if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHECKPOINT_FILE) != 0)
{
/* job has restart file at mom, change restart comment if failed */
change_restart_comment_if_needed(pjob);
/* job has restart file at mom, do end job processing */
svr_setjobstate(pjob, JOB_STATE_EXITING, JOB_SUBSTATE_EXITING);
pjob->ji_momhandle = -1;
/* force new connection */
pwtnew = set_task(WORK_Immed, 0, on_job_exit, (void *)pjob);
if (pwtnew)
{
append_link(&pjob->ji_svrtask, &pwtnew->wt_linkobj, pwtnew);
}
}
else if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_StagedIn) != 0)
{
/* job has staged-in file, should remove them */
remove_stagein(pjob);
job_abt(&pjob, NULL);
}
else
{
/*
* the job is not transitting (though it may have been) and
* is not running, so put in into a complete state.
*/
struct work_task *ptask;
struct pbs_queue *pque;
int KeepSeconds = 0;
svr_setjobstate(pjob, JOB_STATE_COMPLETE, JOB_SUBSTATE_COMPLETE);
if ((pque = pjob->ji_qhdr) && (pque != NULL))
{
pque->qu_numcompleted++;
}
KeepSeconds = attr_ifelse_long(
&pque->qu_attr[(int)QE_ATR_KeepCompleted],
&server.sv_attr[(int)SRV_ATR_KeepCompleted],
0);
ptask = set_task(WORK_Timed, time_now + KeepSeconds, on_job_exit, pjob);
if (ptask != NULL)
{
append_link(&pjob->ji_svrtask, &ptask->wt_linkobj, ptask);
}
}
return(!skipped);
} /* END attempt_delete() */
/**
* attempt_delete()
* deletes a job differently depending on the job's state
*
* @return TRUE if the job was deleted, FALSE if skipped
* @param pjob - a pointer to the job being handled
*/
int attempt_delete(
void *j) /* I */
{
int skipped = FALSE;
int release_mutex = TRUE;
job *pjob;
time_t time_now = time(NULL);
char log_buf[LOCAL_LOG_BUF_SIZE];
/* job considered deleted if null */
if (j == NULL)
return(TRUE);
pjob = (job *)j;
if (pjob->ji_qs.ji_state == JOB_STATE_TRANSIT)
{
/* I'm not sure if this is still possible since the thread
* waits on the job to finish transmiting, but I'll leave
* this part here --dbeer */
skipped = TRUE;
return(!skipped);
} /* END if (pjob->ji_qs.ji_state == JOB_SUBSTATE_TRANSIT) */
else if (pjob->ji_qs.ji_substate == JOB_SUBSTATE_PRERUN)
{
/* we'll wait for the mom to get this job, then delete it */
skipped = TRUE;
} /* END if (pjob->ji_qs.ji_substate == JOB_SUBSTATE_PRERUN) */
else if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING)
{
/* set up nanny */
if (pjob->ji_has_delete_nanny == FALSE)
{
apply_job_delete_nanny(pjob, time_now + 60);
/* need to issue a signal to the mom, but we don't want to sent an ack to the
* client when the mom replies */
issue_signal(&pjob, "SIGTERM", post_delete, NULL);
}
if (pjob != NULL)
{
if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHECKPOINT_FILE) != 0)
{
/* job has restart file at mom, change restart comment if failed */
change_restart_comment_if_needed(pjob);
}
unlock_ji_mutex(pjob, __func__, "1", LOGLEVEL);
}
return(!skipped);
} /* END if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING) */
if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHECKPOINT_FILE) != 0)
{
/* job has restart file at mom, change restart comment if failed */
change_restart_comment_if_needed(pjob);
/* job has restart file at mom, do end job processing */
svr_setjobstate(pjob, JOB_STATE_EXITING, JOB_SUBSTATE_EXITING, FALSE);
pjob->ji_momhandle = -1;
/* force new connection */
if (LOGLEVEL >= 7)
{
sprintf(log_buf, "calling on_job_exit from %s", __func__);
log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf);
}
set_task(WORK_Immed, 0, on_job_exit, strdup(pjob->ji_qs.ji_jobid), FALSE);
}
else if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_StagedIn) != 0)
{
/* job has staged-in file, should remove them */
remove_stagein(&pjob);
if (pjob != NULL)
job_abt(&pjob, NULL);
release_mutex = FALSE;
}
else
{
/*
* the job is not transitting (though it may have been) and
* is not running, so put in into a complete state.
*/
struct pbs_queue *pque;
int KeepSeconds = 0;
svr_setjobstate(pjob, JOB_STATE_COMPLETE, JOB_SUBSTATE_COMPLETE, FALSE);
if ((pque = get_jobs_queue(&pjob)) != NULL)
{
pque->qu_numcompleted++;
unlock_queue(pque, __func__, NULL, LOGLEVEL);
}
if (pjob != NULL)
{
pthread_mutex_lock(server.sv_attr_mutex);
KeepSeconds = attr_ifelse_long(
&pque->qu_attr[QE_ATR_KeepCompleted],
&server.sv_attr[SRV_ATR_KeepCompleted],
0);
pthread_mutex_unlock(server.sv_attr_mutex);
if (LOGLEVEL >= 7)
{
sprintf(log_buf, "calling on_job_exit from %s", __func__);
log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf);
}
set_task(WORK_Timed, time_now + KeepSeconds, on_job_exit, strdup(pjob->ji_qs.ji_jobid), FALSE);
}
else
release_mutex = FALSE;
}
if (release_mutex == TRUE)
unlock_ji_mutex(pjob, __func__, "2", LOGLEVEL);
return(!skipped);
} /* END attempt_delete() */
static void post_delete_mom1(
struct work_task *pwt)
{
int delay = 0;
int dellen = strlen(deldelaystr);
job *pjob;
struct work_task *pwtnew;
pbs_queue *pque;
struct batch_request *preq_sig; /* signal request to MOM */
struct batch_request *preq_clt; /* original client request */
int rc;
preq_sig = pwt->wt_parm1;
rc = preq_sig->rq_reply.brp_code;
preq_clt = preq_sig->rq_extra;
release_req(pwt);
pjob = find_job(preq_clt->rq_ind.rq_delete.rq_objname);
if (pjob == NULL)
{
/* job has gone away */
req_reject(PBSE_UNKJOBID, 0, preq_clt, NULL, NULL);
return;
}
if (rc)
{
/* mom rejected request */
if (rc == PBSE_UNKJOBID)
{
/* MOM claims no knowledge, so just purge it */
log_event(
PBSEVENT_JOB,
PBS_EVENTCLASS_JOB,
pjob->ji_qs.ji_jobid,
"MOM rejected signal during delete");
/* removed the resources assigned to job */
free_nodes(pjob);
set_resc_assigned(pjob, DECR);
job_purge(pjob);
reply_ack(preq_clt);
}
else
{
req_reject(rc, 0, preq_clt, NULL, NULL);
}
return;
}
if (preq_clt->rq_extend)
{
if (strncmp(preq_clt->rq_extend, deldelaystr, dellen) == 0)
{
delay = atoi(preq_clt->rq_extend + dellen);
}
}
reply_ack(preq_clt); /* dont need it, reply now */
/*
* if no delay specified in original request, see if kill_delay
* queue attribute is set.
*/
if (delay == 0)
{
pque = pjob->ji_qhdr;
delay = attr_ifelse_long(&pque->qu_attr[QE_ATR_KillDelay],
&server.sv_attr[SRV_ATR_KillDelay],
2);
}
pwtnew = set_task(WORK_Timed, delay + time_now, post_delete_mom2, pjob);
if (pwtnew)
{
/* insure that work task will be removed if job goes away */
append_link(&pjob->ji_svrtask, &pwtnew->wt_linkobj, pwtnew);
}
/*
* Since the first signal has succeeded, let's reschedule the
* nanny to be 1 minute after the second phase.
*/
apply_job_delete_nanny(pjob, time_now + delay + 60);
return;
} /* END post_delete_mom1() */
void req_deletejob(
struct batch_request *preq) /* I */
{
job *pjob;
struct work_task *pwtold;
struct work_task *pwtnew;
struct work_task *pwtcheck;
int rc;
char *sigt = "SIGTERM";
char *Msg = NULL;
/* check if we are getting a purgecomplete from scheduler */
if ((preq->rq_extend != NULL) &&
!strncmp(preq->rq_extend,PURGECOMP,strlen(PURGECOMP)))
{
/*
* purge_completed_jobs will respond with either an ack or reject
*/
purge_completed_jobs(preq);
return;
}
/* The way this is implemented, if the user enters the command "qdel -p <jobid>",
* they can then delete jobs other than their own since the authorization
* checks are made below in chk_job_request. This should probably be fixed.
*/
if (forced_jobpurge(preq) != 0)
{
return;
}
/* NOTE: should support rq_objname={<JOBID>|ALL|<name:<JOBNAME>} */
/* NYI */
pjob = chk_job_request(preq->rq_ind.rq_delete.rq_objname, preq);
if (pjob == NULL)
{
/* NOTE: chk_job_request() will issue req_reject() */
return;
}
if (preq->rq_extend != NULL)
{
if (strncmp(preq->rq_extend, deldelaystr, strlen(deldelaystr)) &&
strncmp(preq->rq_extend, delasyncstr, strlen(delasyncstr)) &&
strncmp(preq->rq_extend, delpurgestr, strlen(delpurgestr)))
{
/* have text message in request extension, add it */
Msg = preq->rq_extend;
/*
* Message capability is only for operators and managers.
* Check if request is authorized
*/
if ((preq->rq_perm & (ATR_DFLAG_OPRD | ATR_DFLAG_OPWR |
ATR_DFLAG_MGRD | ATR_DFLAG_MGWR)) == 0)
{
req_reject(PBSE_PERM, 0, preq, NULL,
"must have operator or manager privilege to use -m parameter");
return;
}
}
}
if (pjob->ji_qs.ji_state == JOB_STATE_TRANSIT)
{
/*
* Find pid of router from existing work task entry,
* then establish another work task on same child.
* Next, signal the router and wait for its completion;
*/
pwtold = (struct work_task *)GET_NEXT(pjob->ji_svrtask);
while (pwtold != NULL)
{
if ((pwtold->wt_type == WORK_Deferred_Child) ||
(pwtold->wt_type == WORK_Deferred_Cmp))
{
pwtnew = set_task(
pwtold->wt_type,
pwtold->wt_event,
post_delete_route,
preq);
if (pwtnew != NULL)
{
/*
* reset type in case the SIGCHLD came
* in during the set_task; it makes
* sure that next_task() will find the
* new entry.
*/
pwtnew->wt_type = pwtold->wt_type;
pwtnew->wt_aux = pwtold->wt_aux;
kill((pid_t)pwtold->wt_event, SIGTERM);
pjob->ji_qs.ji_substate = JOB_SUBSTATE_ABORT;
return; /* all done for now */
}
else
{
req_reject(PBSE_SYSTEM, 0, preq, NULL, NULL);
return;
}
}
pwtold = (struct work_task *)GET_NEXT(pwtold->wt_linkobj);
}
/* should never get here ... */
log_err(-1, "req_delete", "Did not find work task for router");
req_reject(PBSE_INTERNAL, 0, preq, NULL, NULL);
return;
}
if (pjob->ji_qs.ji_substate == JOB_SUBSTATE_PRERUN ||
pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN ||
pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN1 ||
pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN2 ||
pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN3 )
{
/* If JOB_SUBSTATE_PRERUN being sent to MOM, wait till she gets it going */
/* retry in one second */
/* If JOB_SUBSTATE_RERUN, RERUN1, RERUN2 or RERUN3 the
job is being requeued. Wait until finished */
static time_t cycle_check_when = 0;
static char cycle_check_jid[PBS_MAXSVRJOBID + 1];
if (cycle_check_when != 0)
{
if (!strcmp(pjob->ji_qs.ji_jobid, cycle_check_jid) &&
(time_now - cycle_check_when > 10))
{
/* state not updated after 10 seconds */
/* did the mom ever get it? delete it anyways... */
cycle_check_jid[0] = '\0';
cycle_check_when = 0;
goto jump;
}
if (time_now - cycle_check_when > 20)
{
/* give up after 20 seconds */
cycle_check_jid[0] = '\0';
cycle_check_when = 0;
}
} /* END if (cycle_check_when != 0) */
if (cycle_check_when == 0)
{
/* new PRERUN job located */
cycle_check_when = time_now;
strcpy(cycle_check_jid, pjob->ji_qs.ji_jobid);
}
sprintf(log_buffer, "job cannot be deleted, state=PRERUN, requeuing delete request");
log_event(
PBSEVENT_JOB,
PBS_EVENTCLASS_JOB,
pjob->ji_qs.ji_jobid,
log_buffer);
pwtnew = set_task(
WORK_Timed,
time_now + 1,
post_delete_route,
preq);
if (pwtnew == 0)
req_reject(PBSE_SYSTEM, 0, preq, NULL, NULL);
return;
} /* END if (pjob->ji_qs.ji_substate == JOB_SUBSTATE_PRERUN) */
jump:
/*
* Log delete and if requesting client is not job owner, send mail.
*/
sprintf(log_buffer, "requestor=%s@%s",
preq->rq_user,
preq->rq_host);
/* NOTE: should annotate accounting record with extend message (NYI) */
account_record(PBS_ACCT_DEL, pjob, log_buffer);
sprintf(log_buffer, msg_manager,
msg_deletejob,
preq->rq_user,
preq->rq_host);
log_event(
PBSEVENT_JOB,
PBS_EVENTCLASS_JOB,
pjob->ji_qs.ji_jobid,
log_buffer);
/* NOTE: should incorporate job delete message */
if (Msg != NULL)
{
/* have text message in request extension, add it */
strcat(log_buffer, "\n");
strcat(log_buffer, Msg);
}
if ((svr_chk_owner(preq, pjob) != 0) &&
!has_job_delete_nanny(pjob))
{
/* only send email if owner did not delete job and job deleted
has not been previously attempted */
svr_mailowner(pjob, MAIL_DEL, MAIL_FORCE, log_buffer);
/*
* If we sent mail and already sent the extra message
* then reset message so we don't trigger a redundant email
* in job_abt()
*/
if (Msg != NULL)
{
Msg = NULL;
}
}
if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHECKPOINT_FILE) != 0)
{
/* job has restart file at mom, change restart comment if failed */
change_restart_comment_if_needed(pjob);
}
if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING)
{
/*
* setup a nanny task to make sure the job is actually deleted (see the
* comments at job_delete_nanny()).
*/
if (has_job_delete_nanny(pjob))
{
req_reject(PBSE_IVALREQ, 0, preq, NULL, "job cancel in progress");
return;
}
apply_job_delete_nanny(pjob, time_now + 60);
/* check if we are getting a asynchronous delete */
if ((preq->rq_extend != NULL) &&
!strncmp(preq->rq_extend,DELASYNC,strlen(DELASYNC)))
{
struct batch_request *preq_tmp = NULL;
/*
* Respond with an ack now instead of after MOM processing
* Create a new batch request and fill it in. It will be freed by reply_ack
*/
snprintf(log_buffer,sizeof(log_buffer), "Deleting job asynchronously");
log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,pjob->ji_qs.ji_jobid,log_buffer);
preq_tmp = alloc_br(PBS_BATCH_DeleteJob);
preq_tmp->rq_perm = preq->rq_perm;
preq_tmp->rq_ind.rq_manager.rq_cmd = preq->rq_ind.rq_manager.rq_cmd;
preq_tmp->rq_ind.rq_manager.rq_objtype = preq->rq_ind.rq_manager.rq_objtype;
preq_tmp->rq_fromsvr = preq->rq_fromsvr;
preq_tmp->rq_extsz = preq->rq_extsz;
preq_tmp->rq_conn = preq->rq_conn;
memcpy(preq_tmp->rq_ind.rq_manager.rq_objname,
preq->rq_ind.rq_manager.rq_objname, PBS_MAXSVRJOBID + 1);
memcpy(preq_tmp->rq_user, preq->rq_user, PBS_MAXUSER + 1);
memcpy(preq_tmp->rq_host, preq->rq_host, PBS_MAXHOSTNAME + 1);
reply_ack(preq_tmp);
preq->rq_noreply = TRUE; /* set for no more replies */
}
/* make a cleanup task if set */
if ((server.sv_attr[SRV_ATR_JobForceCancelTime].at_flags & ATR_VFLAG_SET) &&
(server.sv_attr[SRV_ATR_JobForceCancelTime].at_val.at_long > 0))
{
pwtcheck = set_task(
WORK_Timed,
time_now + server.sv_attr[SRV_ATR_JobForceCancelTime].at_val.at_long,
ensure_deleted,
preq);
if (pwtcheck != NULL)
append_link(&pjob->ji_svrtask, &pwtcheck->wt_linkobj, pwtcheck);
}
/*
* Send signal request to MOM. The server will automagically
* pick up and "finish" off the client request when MOM replies.
*/
if ((rc = issue_signal(pjob, sigt, post_delete_mom1, preq)))
{
/* cant send to MOM */
req_reject(rc, 0, preq, NULL, NULL);
}
/* normally will ack reply when mom responds */
sprintf(log_buffer, msg_delrunjobsig,
sigt);
LOG_EVENT(
PBSEVENT_JOB,
PBS_EVENTCLASS_JOB,
pjob->ji_qs.ji_jobid,
log_buffer);
return;
} /* END if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING) */
/* make a cleanup task if set */
if ((server.sv_attr[SRV_ATR_JobForceCancelTime].at_flags & ATR_VFLAG_SET) &&
(server.sv_attr[SRV_ATR_JobForceCancelTime].at_val.at_long > 0))
{
pwtcheck = set_task(
WORK_Timed,
time_now + server.sv_attr[SRV_ATR_JobForceCancelTime].at_val.at_long,
ensure_deleted,
preq);
if (pwtcheck != NULL)
append_link(&pjob->ji_svrtask, &pwtcheck->wt_linkobj, pwtcheck);
}
/* if configured, and this job didn't have a slot limit hold, free a job
* held with the slot limit hold */
if ((server.sv_attr[SRV_ATR_MoabArrayCompatible].at_val.at_long != FALSE) &&
((pjob->ji_wattr[JOB_ATR_hold].at_val.at_long & HOLD_l) == FALSE))
{
if ((pjob->ji_arraystruct != NULL) &&
(pjob->ji_is_array_template == FALSE))
{
int i;
int newstate;
int newsub;
job *tmp;
job_array *pa = pjob->ji_arraystruct;
for (i = 0; i < pa->ai_qs.array_size; i++)
{
if (pa->jobs[i] == NULL)
continue;
tmp = (job *)pa->jobs[i];
if (tmp->ji_wattr[JOB_ATR_hold].at_val.at_long & HOLD_l)
{
tmp->ji_wattr[JOB_ATR_hold].at_val.at_long &= ~HOLD_l;
if (tmp->ji_wattr[JOB_ATR_hold].at_val.at_long == 0)
{
tmp->ji_wattr[JOB_ATR_hold].at_flags &= ~ATR_VFLAG_SET;
}
svr_evaljobstate(tmp, &newstate, &newsub, 1);
svr_setjobstate(tmp, newstate, newsub);
job_save(tmp, SAVEJOB_FULL, 0);
break;
}
}
}
} /* END MoabArrayCompatible check */
if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHECKPOINT_FILE) != 0)
{
/* job has restart file at mom, do end job processing */
svr_setjobstate(pjob, JOB_STATE_EXITING, JOB_SUBSTATE_EXITING);
pjob->ji_momhandle = -1;
/* force new connection */
pwtnew = set_task(WORK_Immed, 0, on_job_exit, (void *)pjob);
if (pwtnew)
{
append_link(&pjob->ji_svrtask, &pwtnew->wt_linkobj, pwtnew);
}
}
else if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_StagedIn) != 0)
{
/* job has staged-in file, should remove them */
remove_stagein(pjob);
job_abt(&pjob, Msg);
}
else
{
/*
* the job is not transitting (though it may have been) and
* is not running, so put in into a complete state.
*/
struct work_task *ptask;
struct pbs_queue *pque;
int KeepSeconds = 0;
svr_setjobstate(pjob, JOB_STATE_COMPLETE, JOB_SUBSTATE_COMPLETE);
if ((pque = pjob->ji_qhdr) && (pque != NULL))
{
pque->qu_numcompleted++;
}
KeepSeconds = attr_ifelse_long(
&pque->qu_attr[QE_ATR_KeepCompleted],
&server.sv_attr[SRV_ATR_KeepCompleted],
0);
ptask = set_task(WORK_Timed, time_now + KeepSeconds, on_job_exit, pjob);
if (ptask != NULL)
{
append_link(&pjob->ji_svrtask, &ptask->wt_linkobj, ptask);
}
} /* END else if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_CHECKPOINT_FILE) != 0) */
reply_ack(preq);
return;
} /* END req_deletejob() */
