/* Make claim against resource limit for a user * user_id IN - Owner of burst buffer * bb_size IN - Size of burst buffer * pool IN - Pool containing the burst buffer * state_ptr IN - Global state to update * update_pool_unfree IN - If true, update the pool's unfree space */ extern void bb_limit_add(uint32_t user_id, uint64_t bb_size, char *pool, bb_state_t *state_ptr, bool update_pool_unfree) { burst_buffer_pool_t *pool_ptr; bb_user_t *bb_user; int i; /* Update the pool's used_space, plus unfree_space if needed */ if (!pool || !xstrcmp(pool, state_ptr->bb_config.default_pool)) { state_ptr->used_space += bb_size; if (update_pool_unfree) state_ptr->unfree_space += bb_size; } else { pool_ptr = state_ptr->bb_config.pool_ptr; for (i = 0; i < state_ptr->bb_config.pool_cnt; i++, pool_ptr++){ if (xstrcmp(pool, pool_ptr->name)) continue; pool_ptr->used_space += bb_size; if (update_pool_unfree) pool_ptr->unfree_space += bb_size; break; } if (i >= state_ptr->bb_config.pool_cnt) error("%s: Unable to located pool %s", __func__, pool); } /* Update user space used */ bb_user = bb_find_user_rec(user_id, state_ptr); xassert(bb_user); bb_user->size += bb_size; }
/* Release claim against resource limit for a user */ extern void bb_limit_rem(uint32_t user_id, uint64_t bb_size, char *pool, bb_state_t *state_ptr) { burst_buffer_pool_t *pool_ptr; bb_user_t *bb_user; int i; if (!pool || !xstrcmp(pool, state_ptr->bb_config.default_pool)) { if (state_ptr->used_space >= bb_size) { state_ptr->used_space -= bb_size; } else { error("%s: used_space underflow", __func__); state_ptr->used_space = 0; } if (state_ptr->unfree_space >= bb_size) { state_ptr->unfree_space -= bb_size; } else { /* This will happen if we reload burst buffer state * after making a claim against resources, but before * the buffer actually gets created */ debug2("%s: unfree_space underflow (%"PRIu64" < %"PRIu64")", __func__, state_ptr->unfree_space, bb_size); state_ptr->unfree_space = 0; } } else { pool_ptr = state_ptr->bb_config.pool_ptr; for (i = 0; i < state_ptr->bb_config.pool_cnt; i++, pool_ptr++){ if (xstrcmp(pool, pool_ptr->name)) continue; if (pool_ptr->used_space >= bb_size) { pool_ptr->used_space -= bb_size; } else { error("%s: used_space underflow for pool %s", __func__, pool); pool_ptr->used_space = 0; } if (pool_ptr->unfree_space >= bb_size) { pool_ptr->unfree_space -= bb_size; } else { error("%s: unfree_space underflow for pool %s", __func__, pool); pool_ptr->unfree_space = 0; } break; } if (i >= state_ptr->bb_config.pool_cnt) error("%s: Unable to located pool %s", __func__, pool); } bb_user = bb_find_user_rec(user_id, state_ptr); xassert(bb_user); if (bb_user->size >= bb_size) bb_user->size -= bb_size; else { bb_user->size = 0; error("%s: user limit underflow for uid %u", __func__, user_id); } }
/* Release claim against resource limit for a user */ extern void bb_limit_rem(uint32_t user_id, uint64_t bb_size, char *pool, bb_state_t *state_ptr) { burst_buffer_pool_t *pool_ptr; bb_user_t *bb_user; int i; if (!pool || !xstrcmp(pool, state_ptr->bb_config.default_pool)) { if (state_ptr->used_space >= bb_size) { state_ptr->used_space -= bb_size; } else { error("%s: used_space underflow", __func__); state_ptr->used_space = 0; } if (state_ptr->unfree_space >= bb_size) { state_ptr->unfree_space -= bb_size; } else { error("%s: unfree_space underflow", __func__); state_ptr->unfree_space = 0; } } else { pool_ptr = state_ptr->bb_config.pool_ptr; for (i = 0; i < state_ptr->bb_config.pool_cnt; i++, pool_ptr++){ if (xstrcmp(pool, pool_ptr->name)) continue; if (pool_ptr->used_space >= bb_size) { pool_ptr->used_space -= bb_size; } else { error("%s: used_space underflow for pool %s", __func__, pool); pool_ptr->used_space = 0; } if (pool_ptr->unfree_space >= bb_size) { pool_ptr->unfree_space -= bb_size; } else { error("%s: unfree_space underflow for pool %s", __func__, pool); pool_ptr->unfree_space = 0; } break; } if (i >= state_ptr->bb_config.pool_cnt) error("%s: Unable to located pool %s", __func__, pool); } bb_user = bb_find_user_rec(user_id, state_ptr); xassert(bb_user); if (bb_user->size >= bb_size) bb_user->size -= bb_size; else { bb_user->size = 0; error("%s: user limit underflow for uid %u", __func__, user_id); } }
/* Test if a job can be allocated a burst buffer. * This may preempt currently active stage-in for higher priority jobs. * * RET 0: Job can be started now * 1: Job exceeds configured limits, continue testing with next job * 2: Job needs more resources than currently available can not start, * skip all remaining jobs */ static int _test_size_limit(struct job_record *job_ptr, uint64_t add_space) { burst_buffer_info_msg_t *resv_bb; struct preempt_bb_recs *preempt_ptr = NULL; List preempt_list; ListIterator preempt_iter; bb_user_t *user_ptr; uint64_t tmp_u, tmp_j, lim_u, resv_space = 0; int add_total_space_needed = 0, add_user_space_needed = 0; int add_total_space_avail = 0, add_user_space_avail = 0; time_t now = time(NULL), when; bb_alloc_t *bb_ptr = NULL; int i; char jobid_buf[32]; /* Determine if burst buffer can be allocated now for the job. * If not, determine how much space must be free. */ if (((bb_state.bb_config.job_size_limit != NO_VAL64) && (add_space > bb_state.bb_config.job_size_limit)) || ((bb_state.bb_config.user_size_limit != NO_VAL64) && (add_space > bb_state.bb_config.user_size_limit))) { debug("%s: %s requested space above limit", __func__, jobid2fmt(job_ptr, jobid_buf, sizeof(jobid_buf))); return 1; } if (job_ptr->start_time <= now) when = now; else when = job_ptr->start_time; resv_bb = job_test_bb_resv(job_ptr, when); if (resv_bb) { burst_buffer_info_t *resv_bb_ptr; for (i = 0, resv_bb_ptr = resv_bb->burst_buffer_array; i < resv_bb->record_count; i++, resv_bb_ptr++) { if (resv_bb_ptr->name && strcmp(resv_bb_ptr->name, bb_state.name)) continue; resv_bb_ptr->used_space = bb_granularity(resv_bb_ptr->used_space, bb_state.bb_config.granularity); resv_space += resv_bb_ptr->used_space; } slurm_free_burst_buffer_info_msg(resv_bb); } if (bb_state.bb_config.user_size_limit != NO_VAL64) { user_ptr = bb_find_user_rec(job_ptr->user_id, bb_state.bb_uhash); tmp_u = user_ptr->size; tmp_j = add_space; lim_u = bb_state.bb_config.user_size_limit; add_user_space_needed = tmp_u + tmp_j - lim_u; } add_total_space_needed = bb_state.used_space + add_space + resv_space - bb_state.total_space; if ((add_total_space_needed <= 0) && (add_user_space_needed <= 0)) return 0; /* Identify candidate burst buffers to revoke for higher priority job */ preempt_list = list_create(bb_job_queue_del); for (i = 0; i < BB_HASH_SIZE; i++) { bb_ptr = bb_state.bb_hash[i]; while (bb_ptr) { if (bb_ptr->job_id && (bb_ptr->use_time > now) && (bb_ptr->use_time > job_ptr->start_time)) { preempt_ptr = xmalloc(sizeof( struct preempt_bb_recs)); preempt_ptr->bb_ptr = bb_ptr; preempt_ptr->job_id = bb_ptr->job_id; preempt_ptr->size = bb_ptr->size; preempt_ptr->use_time = bb_ptr->use_time; preempt_ptr->user_id = bb_ptr->user_id; list_push(preempt_list, preempt_ptr); add_total_space_avail += bb_ptr->size; if (bb_ptr->user_id == job_ptr->user_id) add_user_space_avail += bb_ptr->size; } bb_ptr = bb_ptr->next; } } if ((add_total_space_avail >= add_total_space_needed) && (add_user_space_avail >= add_user_space_needed)) { list_sort(preempt_list, bb_preempt_queue_sort); preempt_iter = list_iterator_create(preempt_list); while ((preempt_ptr = list_next(preempt_iter)) && (add_total_space_needed || add_user_space_needed)) { if (add_user_space_needed && (preempt_ptr->user_id == job_ptr->user_id)) { _stop_stage_in(preempt_ptr->job_id); preempt_ptr->bb_ptr->cancelled = true; preempt_ptr->bb_ptr->end_time = 0; if (bb_state.bb_config.debug_flag) { info("%s: %s: Preempting stage-in of " "job %u for %s", plugin_type, __func__, preempt_ptr->job_id, jobid2fmt(job_ptr, jobid_buf, sizeof(jobid_buf))); } add_user_space_needed -= preempt_ptr->size; add_total_space_needed -= preempt_ptr->size; } if ((add_total_space_needed > add_user_space_needed) && (preempt_ptr->user_id != job_ptr->user_id)) { _stop_stage_in(preempt_ptr->job_id); preempt_ptr->bb_ptr->cancelled = true; preempt_ptr->bb_ptr->end_time = 0; if (bb_state.bb_config.debug_flag) { info("%s: %s: Preempting stage-in of " "job %u for %s", plugin_type, __func__, preempt_ptr->job_id, jobid2fmt(job_ptr, jobid_buf, sizeof(jobid_buf))); } add_total_space_needed -= preempt_ptr->size; } } list_iterator_destroy(preempt_iter); } FREE_NULL_LIST(preempt_list); return 2; }