/* Notify the gang scheduler that a job has completed */ extern int gs_job_fini(struct job_record *job_ptr) { struct gs_part *p_ptr; char *part_name; if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG) info("gang: entering gs_job_fini for job %u", job_ptr->job_id); if (job_ptr->part_ptr && job_ptr->part_ptr->name) part_name = job_ptr->part_ptr->name; else part_name = job_ptr->partition; pthread_mutex_lock(&data_mutex); p_ptr = list_find_first(gs_part_list, _find_gs_part, part_name); if (!p_ptr) { pthread_mutex_unlock(&data_mutex); if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG) info("gang: leaving gs_job_fini"); return SLURM_SUCCESS; } /* remove job from the partition */ _remove_job_from_part(job_ptr->job_id, p_ptr, true); /* this job may have preempted other jobs, so * check by updating all active rows */ _update_all_active_rows(); pthread_mutex_unlock(&data_mutex); if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG) info("gang: leaving gs_job_fini"); return SLURM_SUCCESS; }
/* Notify the gang scheduler that a job has been started */ extern int gs_job_start(struct job_record *job_ptr) { struct gs_part *p_ptr; uint16_t job_state; if (gs_debug_flags & DEBUG_FLAG_GANG) info("gang: entering gs_job_start for job %u", job_ptr->job_id); /* add job to partition */ pthread_mutex_lock(&data_mutex); p_ptr = list_find_first(gs_part_list, _find_gs_part, job_ptr->partition); if (p_ptr) { job_state = _add_job_to_part(p_ptr, job_ptr); /* if this job is running then check for preemption */ if (job_state == GS_RESUME) _update_all_active_rows(); } pthread_mutex_unlock(&data_mutex); if (!p_ptr) { /* No partition was found for this job, so let it run * uninterupted (what else can we do?) */ error("gang: could not find partition %s for job %u", job_ptr->partition, job_ptr->job_id); } _preempt_job_dequeue(); /* MUST BE OUTSIDE OF data_mutex lock */ if (gs_debug_flags & DEBUG_FLAG_GANG) info("gang: leaving gs_job_start"); return SLURM_SUCCESS; }
/* Notify the gang scheduler that a job has been resumed or started. * In either case, add the job to gang scheduling. */ extern void gs_job_start(struct job_record *job_ptr) { struct gs_part *p_ptr; uint16_t job_sig_state; char *part_name; if (!(slurmctld_conf.preempt_mode & PREEMPT_MODE_GANG)) return; if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG) info("gang: entering gs_job_start for job %u", job_ptr->job_id); /* add job to partition */ if (job_ptr->part_ptr && job_ptr->part_ptr->name) part_name = job_ptr->part_ptr->name; else part_name = job_ptr->partition; slurm_mutex_lock(&data_mutex); p_ptr = list_find_first(gs_part_list, _find_gs_part, part_name); if (p_ptr) { job_sig_state = _add_job_to_part(p_ptr, job_ptr); /* if this job is running then check for preemption */ if (job_sig_state == GS_RESUME) _update_all_active_rows(); } slurm_mutex_unlock(&data_mutex); if (!p_ptr) { /* No partition was found for this job, so let it run * uninterupted (what else can we do?) */ error("gang: could not find partition %s for job %u", part_name, job_ptr->job_id); } _preempt_job_dequeue(); /* MUST BE OUTSIDE OF data_mutex lock */ if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG) info("gang: leaving gs_job_start"); }
/* ensure that all jobs running in SLURM are accounted for. * this procedure assumes that the gs data has already been * locked by the caller! */ static void _scan_slurm_job_list(void) { struct job_record *job_ptr; struct gs_part *p_ptr; int i; ListIterator job_iterator; char *part_name; if (!job_list) { /* no jobs */ if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG) info("gang: _scan_slurm_job_list: job_list NULL"); return; } if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG) info("gang: _scan_slurm_job_list: job_list exists..."); job_iterator = list_iterator_create(job_list); while ((job_ptr = (struct job_record *) list_next(job_iterator))) { if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG) { info("gang: _scan_slurm_job_list: checking job %u", job_ptr->job_id); } if (IS_JOB_PENDING(job_ptr)) continue; if (IS_JOB_SUSPENDED(job_ptr) && (job_ptr->priority == 0)) continue; /* not suspended by us */ if (job_ptr->part_ptr && job_ptr->part_ptr->name) part_name = job_ptr->part_ptr->name; else part_name = job_ptr->partition; if (IS_JOB_SUSPENDED(job_ptr) || IS_JOB_RUNNING(job_ptr)) { /* are we tracking this job already? */ p_ptr = list_find_first(gs_part_list, _find_gs_part, part_name); if (!p_ptr) /* no partition */ continue; i = _find_job_index(p_ptr, job_ptr->job_id); if (i >= 0) /* we're tracking it, so continue */ continue; /* We're not tracking this job. Resume it if it's * suspended, and then add it to the job list. */ if (IS_JOB_SUSPENDED(job_ptr) && job_ptr->priority) { /* The likely scenario here is that the * failed over, and this is a job that gang * had previously suspended. It's not possible * to determine the previous order of jobs * without preserving gang state, which is not * worth the extra infrastructure. Just resume * the job and then add it to the job list. */ _resume_job(job_ptr->job_id); } _add_job_to_part(p_ptr, job_ptr); continue; } /* if the job is not pending, suspended, or running, then * it's completing or completed. Make sure we've released * this job */ p_ptr = list_find_first(gs_part_list, _find_gs_part, part_name); if (!p_ptr) /* no partition */ continue; _remove_job_from_part(job_ptr->job_id, p_ptr, false); } list_iterator_destroy(job_iterator); /* now that all of the old jobs have been flushed out, * update the active row of all partitions */ _update_all_active_rows(); return; }