job *job_recov(
char *filename) /* I */ /* pathname to job save file */
{
int fds;
job *pj;
char *pn;
char namebuf[MAXPATHLEN];
char log_buf[LOCAL_LOG_BUF_SIZE];
#ifndef PBS_MOM
char parent_id[PBS_MAXSVRJOBID + 1];
job_array *pa;
#endif
pj = job_alloc(); /* allocate & initialize job structure space */
if (pj == NULL)
{
/* FAILURE - cannot alloc memory */
return(NULL);
}
snprintf(namebuf, MAXPATHLEN, "%s%s", path_jobs, filename); /* job directory path, filename */
fds = open(namebuf, O_RDONLY, 0);
if (fds < 0)
{
snprintf(log_buf, LOCAL_LOG_BUF_SIZE, "unable to open %s", namebuf);
log_err(errno, __func__, log_buf);
#ifndef PBS_MOM
unlock_ji_mutex(pj, __func__, "1", LOGLEVEL);
free(pj->ji_mutex);
#endif
free((char *)pj);
/* FAILURE - cannot open job file */
return(NULL);
}
/* read in job quick save sub-structure */
if (read_ac_socket(fds, (char *)&pj->ji_qs, sizeof(pj->ji_qs)) != sizeof(pj->ji_qs) &&
pj->ji_qs.qs_version == PBS_QS_VERSION)
{
snprintf(log_buf, LOCAL_LOG_BUF_SIZE, "Unable to read %s", namebuf);
log_err(errno, __func__, log_buf);
#ifndef PBS_MOM
unlock_ji_mutex(pj, __func__, "2", LOGLEVEL);
free(pj->ji_mutex);
#endif
free((char *)pj);
close(fds);
return(NULL);
}
/* is ji_qs the version we expect? */
if (pj->ji_qs.qs_version != PBS_QS_VERSION)
{
/* ji_qs is older version */
snprintf(log_buf, LOCAL_LOG_BUF_SIZE,
"%s appears to be from an old version. Attempting to convert.\n",
namebuf);
log_err(-1, __func__, log_buf);
if (job_qs_upgrade(pj, fds, namebuf, pj->ji_qs.qs_version) != 0)
{
snprintf(log_buf, LOCAL_LOG_BUF_SIZE, "unable to upgrade %s\n", namebuf);
log_err(-1, __func__, log_buf);
#ifndef PBS_MOM
unlock_ji_mutex(pj, __func__, "3", LOGLEVEL);
free(pj->ji_mutex);
#endif
free((char *)pj);
close(fds);
return(NULL);
}
} /* END if (pj->ji_qs.qs_version != PBS_QS_VERSION) */
/* Does file name match the internal name? */
/* This detects ghost files */
pn = strrchr(namebuf, (int)'/') + 1;
if (strncmp(pn, pj->ji_qs.ji_fileprefix, strlen(pj->ji_qs.ji_fileprefix)) != 0)
{
/* mismatch, discard job */
snprintf(log_buf, LOCAL_LOG_BUF_SIZE, "Job Id %s does not match file name for %s",
pj->ji_qs.ji_jobid,
namebuf);
log_err(-1, __func__, log_buf);
#ifndef PBS_MOM
unlock_ji_mutex(pj, __func__, "4", LOGLEVEL);
free(pj->ji_mutex);
#endif
free((char *)pj);
close(fds);
return(NULL);
}
/* read in working attributes */
if (recov_attr(
fds,
pj,
job_attr_def,
pj->ji_wattr,
JOB_ATR_LAST,
JOB_ATR_UNKN,
TRUE) != 0)
{
snprintf(log_buf, LOCAL_LOG_BUF_SIZE, "unable to recover %s (file is likely corrupted)", namebuf);
log_err(-1, __func__, log_buf);
#ifndef PBS_MOM
unlock_ji_mutex(pj, __func__, "5", LOGLEVEL);
job_free(pj, FALSE);
#else
mom_job_free(pj);
#endif
close(fds);
return(NULL);
}
#ifndef PBS_MOM
/* Comment out the mother superior tracking. Will be debugged later
if (pj->ji_wattr[JOB_ATR_exec_host].at_val.at_str != NULL)
{*/
/* add job to the mother superior list for it's node */
/* char *ms = strdup(pj->ji_wattr[JOB_ATR_exec_host].at_val.at_str);
char *end = strchr(ms, '/');
if (end != NULL)
*end = '\0';
if ((end = strchr(ms, '+')) != NULL)
*end = '\0';
add_to_ms_list(ms, pj);
free(ms);
}*/
#endif
#ifdef PBS_MOM
/* read in tm sockets and ips */
if (recov_tmsock(fds, pj) != 0)
{
snprintf(log_buf, LOCAL_LOG_BUF_SIZE,
"warning: tmsockets not recovered from %s (written by an older pbs_mom?)",
namebuf);
log_err(-1, __func__, log_buf);
}
#else /* not PBS_MOM */
if (strchr(pj->ji_qs.ji_jobid, '[') != NULL)
{
/* job is part of an array. We need to put a link back to the server
job array struct for this array. We also have to link this job into
the linked list of jobs belonging to the array. */
array_get_parent_id(pj->ji_qs.ji_jobid, parent_id);
pa = get_array(parent_id);
if (pa == NULL)
{
job_abt(&pj, (char *)"Array job missing array struct, aborting job");
close(fds);
return NULL;
}
strcpy(pj->ji_arraystructid, parent_id);
if (strcmp(parent_id, pj->ji_qs.ji_jobid) == 0)
{
pj->ji_is_array_template = TRUE;
}
else
{
pa->job_ids[(int)pj->ji_wattr[JOB_ATR_job_array_id].at_val.at_long] = strdup(pj->ji_qs.ji_jobid);
pa->jobs_recovered++;
/* This is a bit of a kluge, but for some reason if an array job was
on hold when the server went down the ji_wattr[JOB_ATR_hold].at_val.at_long
value is 0 on recovery even though pj->ji_qs.ji_state is JOB_STATE_HELD and
the substate is JOB_SUBSTATE_HELD
*/
if ((pj->ji_qs.ji_state == JOB_STATE_HELD) &&
(pj->ji_qs.ji_substate == JOB_SUBSTATE_HELD))
{
pj->ji_wattr[JOB_ATR_hold].at_val.at_long = HOLD_l;
pj->ji_wattr[JOB_ATR_hold].at_flags = ATR_VFLAG_SET;
}
}
if (pa != NULL)
{
unlock_ai_mutex(pa, __func__, "1", LOGLEVEL);
}
}
#endif
close(fds);
pj->ji_commit_done = 1;
/* all done recovering the job */
job_save(pj, SAVEJOB_FULL, 0);
return(pj);
} /* END job_recov() */
void mom_job_purge(
job *pjob) /* I (modified) */
{
job_file_delete_info *jfdi;
jfdi = (job_file_delete_info *)calloc(1, sizeof(job_file_delete_info));
if (jfdi == NULL)
{
log_err(ENOMEM,__func__, (char *)"No space to allocate info for job file deletion");
return;
}
#ifdef NVIDIA_GPUS
/*
* Did this job have a gpuid assigned?
* if so, then update gpu status
*/
if (((pjob->ji_wattr[JOB_ATR_exec_gpus].at_flags & ATR_VFLAG_SET) != 0) &&
(pjob->ji_wattr[JOB_ATR_exec_gpus].at_val.at_str != NULL))
{
send_update_soon();
}
#endif /* NVIDIA_GPUS */
/* initialize struct information */
if (pjob->ji_flags & MOM_HAS_TMPDIR)
{
jfdi->has_temp_dir = TRUE;
pjob->ji_flags &= ~MOM_HAS_TMPDIR;
}
else
jfdi->has_temp_dir = FALSE;
strcpy(jfdi->jobid,pjob->ji_qs.ji_jobid);
strcpy(jfdi->prefix,pjob->ji_qs.ji_fileprefix);
if ((pjob->ji_wattr[JOB_ATR_checkpoint_dir].at_flags & ATR_VFLAG_SET) &&
(pjob->ji_wattr[JOB_ATR_checkpoint_name].at_flags & ATR_VFLAG_SET))
jfdi->checkpoint_dir = strdup(pjob->ji_wattr[JOB_ATR_checkpoint_dir].at_val.at_str);
jfdi->gid = pjob->ji_qs.ji_un.ji_momt.ji_exgid;
jfdi->uid = pjob->ji_qs.ji_un.ji_momt.ji_exuid;
if (thread_unlink_calls == TRUE)
enqueue_threadpool_request(delete_job_files,jfdi);
else
delete_job_files(jfdi);
/* remove this job from the global queue */
delete_link(&pjob->ji_jobque);
delete_link(&pjob->ji_alljobs);
if (LOGLEVEL >= 6)
{
sprintf(log_buffer,"removing job");
log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buffer);
}
#if IBM_SP2==2 /* IBM SP PSSP 3.1 */
unload_sp_switch(pjob);
#endif /* IBM SP */
mom_job_free(pjob);
/* if no jobs are left, check if MOM should be restarted */
if (((job *)GET_NEXT(svr_alljobs)) == NULL)
MOMCheckRestart();
return;
} /* END mom_job_purge() */
void mom_job_purge(
job *pjob) /* I (modified) */
{
job_file_delete_info *jfdi;
jfdi = (job_file_delete_info *)calloc(1, sizeof(job_file_delete_info));
if (jfdi == NULL)
{
log_err(ENOMEM,__func__, (char *)"No space to allocate info for job file deletion");
return;
}
#ifdef NVIDIA_GPUS
/*
* Did this job have a gpuid assigned?
* if so, then update gpu status
*/
if (((pjob->ji_wattr[JOB_ATR_exec_gpus].at_flags & ATR_VFLAG_SET) != 0) &&
(pjob->ji_wattr[JOB_ATR_exec_gpus].at_val.at_str != NULL))
{
send_update_soon();
}
#endif /* NVIDIA_GPUS */
/* initialize struct information */
if (pjob->ji_flags & MOM_HAS_TMPDIR)
{
jfdi->has_temp_dir = TRUE;
pjob->ji_flags &= ~MOM_HAS_TMPDIR;
}
else
jfdi->has_temp_dir = FALSE;
strcpy(jfdi->jobid,pjob->ji_qs.ji_jobid);
strcpy(jfdi->prefix,pjob->ji_qs.ji_fileprefix);
if ((pjob->ji_wattr[JOB_ATR_checkpoint_dir].at_flags & ATR_VFLAG_SET) &&
(pjob->ji_wattr[JOB_ATR_checkpoint_name].at_flags & ATR_VFLAG_SET))
jfdi->checkpoint_dir = strdup(pjob->ji_wattr[JOB_ATR_checkpoint_dir].at_val.at_str);
jfdi->gid = pjob->ji_qs.ji_un.ji_momt.ji_exgid;
jfdi->uid = pjob->ji_qs.ji_un.ji_momt.ji_exuid;
/* remove each pid in ji_job_pid_set from the global_job_sid_set */
for (job_pid_set_t::const_iterator job_pid_set_iter = pjob->ji_job_pid_set->begin();
job_pid_set_iter != pjob->ji_job_pid_set->end();
job_pid_set_iter++)
{
/* get pid entry from ji_job_pid_set */
pid_t job_pid = *job_pid_set_iter;
/* see if job_pid exists in job_sid set */
job_pid_set_t::const_iterator it = global_job_sid_set.find(job_pid);
if (it != global_job_sid_set.end())
{
/* remove job_pid from the set */
global_job_sid_set.erase(it);
}
}
if (thread_unlink_calls == TRUE)
enqueue_threadpool_request(delete_job_files, jfdi, request_pool);
else
delete_job_files(jfdi);
/* remove this job from the global queue */
delete_link(&pjob->ji_jobque);
delete_link(&pjob->ji_alljobs);
remove_from_exiting_list(pjob);
if (LOGLEVEL >= 6)
{
sprintf(log_buffer,"removing job");
log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buffer);
}
#if IBM_SP2==2 /* IBM SP PSSP 3.1 */
unload_sp_switch(pjob);
#endif /* IBM SP */
//We had a request to change the frequency for the job and now that the job is done
//we want to change the frequency back.
resource *presc = find_resc_entry(&pjob->ji_wattr[JOB_ATR_resource],
find_resc_def(svr_resc_def, "cpuclock", svr_resc_size));
if (presc != NULL)
{
std::string beforeFreq;
nd_frequency.get_frequency_string(beforeFreq);
if(!nd_frequency.restore_frequency())
{
std::string msg = "Failed to restore frequency.";
log_ext(nd_frequency.get_last_error(),__func__,msg.c_str(),LOG_ERR);
}
else
{
std::string afterFreq;
nd_frequency.get_frequency_string(afterFreq);
std::string msg = "Restored frequency from " + beforeFreq + " to " + afterFreq;
log_ext(PBSE_CHANGED_CPU_FREQUENCY,__func__, msg.c_str(),LOG_NOTICE);
}
}
mom_job_free(pjob);
/* if no jobs are left, check if MOM should be restarted */
if (((job *)GET_NEXT(svr_alljobs)) == NULL)
MOMCheckRestart();
return;
} /* END mom_job_purge() */
