/* Make claim against resource limit for a user
* user_id IN - Owner of burst buffer
* bb_size IN - Size of burst buffer
* pool IN - Pool containing the burst buffer
* state_ptr IN - Global state to update
* update_pool_unfree IN - If true, update the pool's unfree space */
extern void bb_limit_add(uint32_t user_id, uint64_t bb_size, char *pool,
bb_state_t *state_ptr, bool update_pool_unfree)
{
burst_buffer_pool_t *pool_ptr;
bb_user_t *bb_user;
int i;
/* Update the pool's used_space, plus unfree_space if needed */
if (!pool || !xstrcmp(pool, state_ptr->bb_config.default_pool)) {
state_ptr->used_space += bb_size;
if (update_pool_unfree)
state_ptr->unfree_space += bb_size;
} else {
pool_ptr = state_ptr->bb_config.pool_ptr;
for (i = 0; i < state_ptr->bb_config.pool_cnt; i++, pool_ptr++){
if (xstrcmp(pool, pool_ptr->name))
continue;
pool_ptr->used_space += bb_size;
if (update_pool_unfree)
pool_ptr->unfree_space += bb_size;
break;
}
if (i >= state_ptr->bb_config.pool_cnt)
error("%s: Unable to located pool %s", __func__, pool);
}
/* Update user space used */
bb_user = bb_find_user_rec(user_id, state_ptr);
xassert(bb_user);
bb_user->size += bb_size;
}
/* Release claim against resource limit for a user */
extern void bb_limit_rem(uint32_t user_id, uint64_t bb_size, char *pool,
bb_state_t *state_ptr)
{
burst_buffer_pool_t *pool_ptr;
bb_user_t *bb_user;
int i;
if (!pool || !xstrcmp(pool, state_ptr->bb_config.default_pool)) {
if (state_ptr->used_space >= bb_size) {
state_ptr->used_space -= bb_size;
} else {
error("%s: used_space underflow", __func__);
state_ptr->used_space = 0;
}
if (state_ptr->unfree_space >= bb_size) {
state_ptr->unfree_space -= bb_size;
} else {
/* This will happen if we reload burst buffer state
* after making a claim against resources, but before
* the buffer actually gets created */
debug2("%s: unfree_space underflow (%"PRIu64" < %"PRIu64")",
__func__, state_ptr->unfree_space, bb_size);
state_ptr->unfree_space = 0;
}
} else {
pool_ptr = state_ptr->bb_config.pool_ptr;
for (i = 0; i < state_ptr->bb_config.pool_cnt; i++, pool_ptr++){
if (xstrcmp(pool, pool_ptr->name))
continue;
if (pool_ptr->used_space >= bb_size) {
pool_ptr->used_space -= bb_size;
} else {
error("%s: used_space underflow for pool %s",
__func__, pool);
pool_ptr->used_space = 0;
}
if (pool_ptr->unfree_space >= bb_size) {
pool_ptr->unfree_space -= bb_size;
} else {
error("%s: unfree_space underflow for pool %s",
__func__, pool);
pool_ptr->unfree_space = 0;
}
break;
}
if (i >= state_ptr->bb_config.pool_cnt)
error("%s: Unable to located pool %s", __func__, pool);
}
bb_user = bb_find_user_rec(user_id, state_ptr);
xassert(bb_user);
if (bb_user->size >= bb_size)
bb_user->size -= bb_size;
else {
bb_user->size = 0;
error("%s: user limit underflow for uid %u", __func__, user_id);
}
}
/* Release claim against resource limit for a user */
extern void bb_limit_rem(uint32_t user_id, uint64_t bb_size, char *pool,
bb_state_t *state_ptr)
{
burst_buffer_pool_t *pool_ptr;
bb_user_t *bb_user;
int i;
if (!pool || !xstrcmp(pool, state_ptr->bb_config.default_pool)) {
if (state_ptr->used_space >= bb_size) {
state_ptr->used_space -= bb_size;
} else {
error("%s: used_space underflow", __func__);
state_ptr->used_space = 0;
}
if (state_ptr->unfree_space >= bb_size) {
state_ptr->unfree_space -= bb_size;
} else {
error("%s: unfree_space underflow", __func__);
state_ptr->unfree_space = 0;
}
} else {
pool_ptr = state_ptr->bb_config.pool_ptr;
for (i = 0; i < state_ptr->bb_config.pool_cnt; i++, pool_ptr++){
if (xstrcmp(pool, pool_ptr->name))
continue;
if (pool_ptr->used_space >= bb_size) {
pool_ptr->used_space -= bb_size;
} else {
error("%s: used_space underflow for pool %s",
__func__, pool);
pool_ptr->used_space = 0;
}
if (pool_ptr->unfree_space >= bb_size) {
pool_ptr->unfree_space -= bb_size;
} else {
error("%s: unfree_space underflow for pool %s",
__func__, pool);
pool_ptr->unfree_space = 0;
}
break;
}
if (i >= state_ptr->bb_config.pool_cnt)
error("%s: Unable to located pool %s", __func__, pool);
}
bb_user = bb_find_user_rec(user_id, state_ptr);
xassert(bb_user);
if (bb_user->size >= bb_size)
bb_user->size -= bb_size;
else {
bb_user->size = 0;
error("%s: user limit underflow for uid %u", __func__, user_id);
}
}
/* Test if a job can be allocated a burst buffer.
* This may preempt currently active stage-in for higher priority jobs.
*
* RET 0: Job can be started now
* 1: Job exceeds configured limits, continue testing with next job
* 2: Job needs more resources than currently available can not start,
* skip all remaining jobs
*/
static int _test_size_limit(struct job_record *job_ptr, uint64_t add_space)
{
burst_buffer_info_msg_t *resv_bb;
struct preempt_bb_recs *preempt_ptr = NULL;
List preempt_list;
ListIterator preempt_iter;
bb_user_t *user_ptr;
uint64_t tmp_u, tmp_j, lim_u, resv_space = 0;
int add_total_space_needed = 0, add_user_space_needed = 0;
int add_total_space_avail = 0, add_user_space_avail = 0;
time_t now = time(NULL), when;
bb_alloc_t *bb_ptr = NULL;
int i;
char jobid_buf[32];
/* Determine if burst buffer can be allocated now for the job.
* If not, determine how much space must be free. */
if (((bb_state.bb_config.job_size_limit != NO_VAL64) &&
(add_space > bb_state.bb_config.job_size_limit)) ||
((bb_state.bb_config.user_size_limit != NO_VAL64) &&
(add_space > bb_state.bb_config.user_size_limit))) {
debug("%s: %s requested space above limit", __func__,
jobid2fmt(job_ptr, jobid_buf, sizeof(jobid_buf)));
return 1;
}
if (job_ptr->start_time <= now)
when = now;
else
when = job_ptr->start_time;
resv_bb = job_test_bb_resv(job_ptr, when);
if (resv_bb) {
burst_buffer_info_t *resv_bb_ptr;
for (i = 0, resv_bb_ptr = resv_bb->burst_buffer_array;
i < resv_bb->record_count; i++, resv_bb_ptr++) {
if (resv_bb_ptr->name &&
strcmp(resv_bb_ptr->name, bb_state.name))
continue;
resv_bb_ptr->used_space =
bb_granularity(resv_bb_ptr->used_space,
bb_state.bb_config.granularity);
resv_space += resv_bb_ptr->used_space;
}
slurm_free_burst_buffer_info_msg(resv_bb);
}
if (bb_state.bb_config.user_size_limit != NO_VAL64) {
user_ptr = bb_find_user_rec(job_ptr->user_id,
bb_state.bb_uhash);
tmp_u = user_ptr->size;
tmp_j = add_space;
lim_u = bb_state.bb_config.user_size_limit;
add_user_space_needed = tmp_u + tmp_j - lim_u;
}
add_total_space_needed = bb_state.used_space + add_space + resv_space -
bb_state.total_space;
if ((add_total_space_needed <= 0) &&
(add_user_space_needed <= 0))
return 0;
/* Identify candidate burst buffers to revoke for higher priority job */
preempt_list = list_create(bb_job_queue_del);
for (i = 0; i < BB_HASH_SIZE; i++) {
bb_ptr = bb_state.bb_hash[i];
while (bb_ptr) {
if (bb_ptr->job_id &&
(bb_ptr->use_time > now) &&
(bb_ptr->use_time > job_ptr->start_time)) {
preempt_ptr = xmalloc(sizeof(
struct preempt_bb_recs));
preempt_ptr->bb_ptr = bb_ptr;
preempt_ptr->job_id = bb_ptr->job_id;
preempt_ptr->size = bb_ptr->size;
preempt_ptr->use_time = bb_ptr->use_time;
preempt_ptr->user_id = bb_ptr->user_id;
list_push(preempt_list, preempt_ptr);
add_total_space_avail += bb_ptr->size;
if (bb_ptr->user_id == job_ptr->user_id)
add_user_space_avail += bb_ptr->size;
}
bb_ptr = bb_ptr->next;
}
}
if ((add_total_space_avail >= add_total_space_needed) &&
(add_user_space_avail >= add_user_space_needed)) {
list_sort(preempt_list, bb_preempt_queue_sort);
preempt_iter = list_iterator_create(preempt_list);
while ((preempt_ptr = list_next(preempt_iter)) &&
(add_total_space_needed || add_user_space_needed)) {
if (add_user_space_needed &&
(preempt_ptr->user_id == job_ptr->user_id)) {
_stop_stage_in(preempt_ptr->job_id);
preempt_ptr->bb_ptr->cancelled = true;
preempt_ptr->bb_ptr->end_time = 0;
if (bb_state.bb_config.debug_flag) {
info("%s: %s: Preempting stage-in of "
"job %u for %s", plugin_type,
__func__, preempt_ptr->job_id,
jobid2fmt(job_ptr, jobid_buf,
sizeof(jobid_buf)));
}
add_user_space_needed -= preempt_ptr->size;
add_total_space_needed -= preempt_ptr->size;
}
if ((add_total_space_needed > add_user_space_needed) &&
(preempt_ptr->user_id != job_ptr->user_id)) {
_stop_stage_in(preempt_ptr->job_id);
preempt_ptr->bb_ptr->cancelled = true;
preempt_ptr->bb_ptr->end_time = 0;
if (bb_state.bb_config.debug_flag) {
info("%s: %s: Preempting stage-in of "
"job %u for %s", plugin_type,
__func__, preempt_ptr->job_id,
jobid2fmt(job_ptr, jobid_buf,
sizeof(jobid_buf)));
}
add_total_space_needed -= preempt_ptr->size;
}
}
list_iterator_destroy(preempt_iter);
}
FREE_NULL_LIST(preempt_list);
return 2;
}
