/* Compute resource usage for the given job on all available resources
*
* IN: job_ptr - pointer to the job requesting resources
* IN: node_map - bitmap of available nodes
* IN/OUT: core_map - bitmap of available cores
* IN: cr_node_cnt - total number of nodes in the cluster
* IN: cr_type - resource type
* OUT: cpu_cnt - number of cpus that can be used by this job
* IN: test_only - ignore allocated memory check
* RET SLURM_SUCCESS index of selected node or -1 if none
*/
static int _get_res_usage(struct job_record *job_ptr, bitstr_t *node_map,
bitstr_t *core_map, uint32_t cr_node_cnt,
struct node_use_record *node_usage,
uint16_t cr_type, uint16_t **cpu_cnt_ptr,
bool test_only)
{
uint16_t *cpu_cnt, max_cpu_cnt = 0, part_lln_flag = 0;
int i, i_first, i_last;
int node_inx = -1;
if (cr_node_cnt != node_record_count) {
error("select/serial: node count inconsistent with slurmctld");
return SLURM_ERROR;
}
if (!job_ptr) {
error("select/serial: NULL job pointer");
return SLURM_ERROR;
}
if (job_ptr->part_ptr && (job_ptr->part_ptr->flags & PART_FLAG_LLN))
part_lln_flag = 1;
if (job_ptr->details && job_ptr->details->req_node_bitmap)
bit_and(node_map, job_ptr->details->req_node_bitmap);
cpu_cnt = xmalloc(cr_node_cnt * sizeof(uint16_t));
i_first = bit_ffs(node_map);
if (i_first >= 0)
i_last = bit_fls(node_map);
else
i_last = -2;
for (i = i_first; i <= i_last; i++) {
if (!bit_test(node_map, i))
continue;
cpu_cnt[i] = _can_job_run_on_node(job_ptr, core_map, i,
node_usage, cr_type,
test_only);
if (!(cr_type & CR_LLN) && !part_lln_flag && cpu_cnt[i]) {
bit_nclear(node_map, 0, (node_record_count - 1));
bit_set(node_map, i);
node_inx = i;
break; /* select/serial: only need one node */
}
}
if ((cr_type & CR_LLN) || part_lln_flag) {
for (i = i_first; i <= i_last; i++) {
if (cpu_cnt[i] > max_cpu_cnt) {
max_cpu_cnt = cpu_cnt[i];
node_inx = i;
}
}
if (node_inx >= 0) {
bit_nclear(node_map, 0, (node_record_count - 1));
bit_set(node_map, node_inx);
}
}
*cpu_cnt_ptr = cpu_cnt;
return node_inx;
}
/* Select the best set of resources for the given job
* IN: job_ptr - pointer to the job requesting resources
* IN/OUT: node_map - bitmap of available nodes / bitmap of selected nodes
* IN: cr_node_cnt - total number of nodes in the cluster
* IN/OUT: core_map - bitmap of available cores / bitmap of selected cores
* IN: cr_type - resource type
* IN: test_only - ignore allocated memory check
* RET - array with number of CPUs available per node or NULL if not runnable
*/
static uint16_t *_select_nodes(struct job_record *job_ptr,
bitstr_t *node_map, uint32_t cr_node_cnt,
bitstr_t *core_map,
struct node_use_record *node_usage,
uint16_t cr_type, bool test_only)
{
int node_inx;
uint16_t *cpu_cnt, *cpus = NULL;
if (bit_set_count(node_map) == 0)
return NULL;
/* get resource usage for this job from first available node */
node_inx = _get_res_usage(job_ptr, node_map, core_map, cr_node_cnt,
node_usage, cr_type, &cpu_cnt, test_only);
/* if successful, sync up the core_map with the node_map, and
* create a cpus array */
if (node_inx >= 0) {
cpus = xmalloc(sizeof(uint16_t));
cpus[0] = cpu_cnt[node_inx];
if (node_inx != 0) {
bit_nclear(core_map, 0,
(cr_get_coremap_offset(node_inx))-1);
}
if (node_inx < (cr_node_cnt - 1)) {
bit_nclear(core_map,
(cr_get_coremap_offset(node_inx + 1)),
(cr_get_coremap_offset(cr_node_cnt) - 1));
}
}
xfree(cpu_cnt);
return cpus;
}
int get_list(bitstr_t *bits, int low, int high, const char **names, const char **pp)
{
/* clear the bit string, since the default is 'off'.
*/
bit_nclear(bits, 0, (high-low+1));
/* list = range {"," range}
*/
/* process all ranges
*/
while(1)
{
if(get_range(bits, low, high, names, pp) == ERR)
return ERR;
if(**pp == ',') ++(*pp);
else break;
}
/* exiting. skip to some blanks, then skip over the blanks.
*/
while(**pp != 0 && !isspace(**pp)) ++(*pp);
while(isspace(**pp)) ++(*pp);
return OK;
}
void
taskset_cycle( taskset_t *ts )
{
hb_lock( ts->task_cond_lock );
/*
* Signal all threads that their work is available.
*/
bit_nset( ts->task_begin_bitmap, 0, ts->thread_count - 1 );
hb_cond_broadcast( ts->task_begin );
/*
* Wait until all threads have completed. Note that we must
* loop here as hb_cond_wait() on some platforms (e.g pthead_cond_wait)
* may unblock prematurely.
*/
do
{
hb_cond_wait( ts->task_complete, ts->task_cond_lock );
} while ( !allbits_set( ts->task_complete_bitmap, ts->bitmap_elements ) );
/*
* Clear completion indications for next time.
*/
bit_nclear( ts->task_complete_bitmap, 0, ts->thread_count - 1 );
hb_unlock( ts->task_cond_lock );
}
static void cachedev_clearbits (bitstr_t *bitmap, ioreq_event *req)
{
#ifdef DEBUG_CACHEDEV
fprintf(outputfile, "*** %f: Entered cachedev::cachedev_clearbits from block %d to %d\n", simtime, req->blkno, req->blkno+req->bcount-1 );
fflush(outputfile);
#endif
bit_nclear (bitmap, req->blkno, (req->blkno+req->bcount-1));
}
/*
* Initialize our volume bitmap data structures
*/
int BitMapCheckBegin(SGlobPtr g)
{
Boolean isHFSPlus;
if (gBitMapInited)
return (0);
isHFSPlus = VolumeObjectIsHFSPlus( );
gFullBitmapSegment = (UInt32 *)malloc(kBytesPerSegment);
memset((void *)gFullBitmapSegment, 0xff, kBytesPerSegment);
gEmptyBitmapSegment = (UInt32 *)malloc(kBytesPerSegment);
memset((void *)gEmptyBitmapSegment, 0x00, kBytesPerSegment);
gTotalBits = g->calculatedVCB->vcbTotalBlocks;
gTotalSegments = (gTotalBits / kBitsPerSegment);
if (gTotalBits % kBitsPerSegment)
++gTotalSegments;
gFullSegmentList = bit_alloc(gTotalSegments);
bit_nclear(gFullSegmentList, 0, gTotalSegments - 1);
BMS_InitTree();
gBitMapInited = 1;
gBitsMarked = 0;
if (isHFSPlus) {
UInt16 alignBits;
/*
* Allocate the VolumeHeader in the volume bitmap.
* Since the VH is the 3rd sector in we may need to
* add some alignment allocation blocks before it.
*/
if (g->calculatedVCB->vcbBlockSize == 512)
alignBits = 2;
else if (g->calculatedVCB->vcbBlockSize == 1024)
alignBits = 1;
else
alignBits = 0;
(void) CaptureBitmapBits(0, 1 + alignBits);
if (g->calculatedVCB->vcbBlockSize == 512)
alignBits = 1;
else
alignBits = 0;
(void) CaptureBitmapBits(gTotalBits - 1 - alignBits, 1 + alignBits);
}
return (0);
}
int switch_p_libstate_clear(void)
{
#ifdef HAVE_NATIVE_CRAY
pthread_mutex_lock(&port_mutex);
bit_nclear(port_resv, 0, PORT_CNT - 1);
last_alloc_port = 0;
pthread_mutex_unlock(&port_mutex);
#endif
return SLURM_SUCCESS;
}
static TEE_Result maybe_grow_files(struct tee_fs_dirfile_dirh *dirh, int idx)
{
void *p;
if (idx < dirh->nbits)
return TEE_SUCCESS;
p = realloc(dirh->files, bitstr_size(idx + 1));
if (!p)
return TEE_ERROR_OUT_OF_MEMORY;
dirh->files = p;
bit_nclear(dirh->files, dirh->nbits, idx);
dirh->nbits = idx + 1;
return TEE_SUCCESS;
}
static int
get_list(bitstr_t *bits, int low, int high, const char * const names[],
int ch, FILE *file)
{
int done;
/* we know that we point to a non-blank character here;
* must do a Skip_Blanks before we exit, so that the
* next call (or the code that picks up the cmd) can
* assume the same thing.
*/
Debug(DPARS|DEXT, ("get_list()...entered\n"));
/* list = range {"," range}
*/
/* clear the bit string, since the default is 'off'.
*/
bit_nclear(bits, 0, (size_t)(high-low+1));
/* process all ranges
*/
done = FALSE;
while (!done) {
if (EOF == (ch = get_range(bits, low, high, names, ch, file)))
return (EOF);
if (ch == ',')
ch = get_char(file);
else
done = TRUE;
}
/* exiting. skip to some blanks, then skip over the blanks.
*/
Skip_Nonblanks(ch, file);
Skip_Blanks(ch, file);
Debug(DPARS|DEXT, ("get_list()...exiting w/ %02x\n", ch));
return (ch);
}
/* Compute resource usage for the given job on all available resources
*
* IN: job_ptr - pointer to the job requesting resources
* IN: node_map - bitmap of available nodes
* IN/OUT: core_map - bitmap of available cores
* IN: cr_node_cnt - total number of nodes in the cluster
* IN: cr_type - resource type
* OUT: cpu_cnt - number of cpus that can be used by this job
* IN: test_only - ignore allocated memory check
* RET SLURM_SUCCESS index of selected node or -1 if none
*/
static int _get_res_usage(struct job_record *job_ptr, bitstr_t *node_map,
bitstr_t *core_map, uint32_t cr_node_cnt,
struct node_use_record *node_usage,
uint16_t cr_type, uint16_t **cpu_cnt_ptr,
bool test_only)
{
uint16_t *cpu_cnt;
uint32_t n;
int i_first, i_last;
int node_inx = -1;
if (cr_node_cnt != node_record_count) {
error("select/serial: node count inconsistent with slurmctld");
return SLURM_ERROR;
}
if (job_ptr->details && job_ptr->details->req_node_bitmap)
bit_and(node_map, job_ptr->details->req_node_bitmap);
cpu_cnt = xmalloc(cr_node_cnt * sizeof(uint16_t));
i_first = bit_ffs(node_map);
if (i_first >= 0)
i_last = bit_fls(node_map);
else
i_last = -2;
for (n = i_first; n <= i_last; n++) {
if (!bit_test(node_map, n))
continue;
cpu_cnt[n] = _can_job_run_on_node(job_ptr, core_map, n,
node_usage, cr_type,
test_only);
if (cpu_cnt[n]) {
bit_nclear(node_map, 0, (node_record_count - 1));
bit_set(node_map, n);
node_inx = n;
break; /* select/serial: only need one node */
}
}
*cpu_cnt_ptr = cpu_cnt;
return node_inx;
}
/* Execute control sequence. */
int
input_csi_dispatch(struct input_ctx *ictx)
{
struct screen_write_ctx *sctx = &ictx->ctx;
struct window_pane *wp = ictx->wp;
struct screen *s = sctx->s;
struct input_table_entry *entry;
int n, m;
if (ictx->flags & INPUT_DISCARD)
return (0);
if (input_split(ictx) != 0)
return (0);
log_debug("%s: '%c' \"%s\" \"%s\"",
__func__, ictx->ch, ictx->interm_buf, ictx->param_buf);
entry = bsearch(ictx, input_csi_table, nitems(input_csi_table),
sizeof input_csi_table[0], input_table_compare);
if (entry == NULL) {
log_debug("%s: unknown '%c'", __func__, ictx->ch);
return (0);
}
switch (entry->type) {
case INPUT_CSI_CBT:
/* Find the previous tab point, n times. */
n = input_get(ictx, 0, 1, 1);
while (s->cx > 0 && n-- > 0) {
do
s->cx--;
while (s->cx > 0 && !bit_test(s->tabs, s->cx));
}
break;
case INPUT_CSI_CUB:
screen_write_cursorleft(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_CUD:
screen_write_cursordown(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_CUF:
screen_write_cursorright(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_CUP:
n = input_get(ictx, 0, 1, 1);
m = input_get(ictx, 1, 1, 1);
screen_write_cursormove(sctx, m - 1, n - 1);
break;
case INPUT_CSI_CUU:
screen_write_cursorup(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_DA:
switch (input_get(ictx, 0, 0, 0)) {
case 0:
input_reply(ictx, "\033[?1;2c");
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_DCH:
screen_write_deletecharacter(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_DECSTBM:
n = input_get(ictx, 0, 1, 1);
m = input_get(ictx, 1, 1, screen_size_y(s));
screen_write_scrollregion(sctx, n - 1, m - 1);
break;
case INPUT_CSI_DL:
screen_write_deleteline(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_DSR:
switch (input_get(ictx, 0, 0, 0)) {
case 5:
input_reply(ictx, "\033[0n");
break;
case 6:
input_reply(ictx, "\033[%u;%uR", s->cy + 1, s->cx + 1);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_ED:
switch (input_get(ictx, 0, 0, 0)) {
case 0:
screen_write_clearendofscreen(sctx);
break;
case 1:
screen_write_clearstartofscreen(sctx);
break;
case 2:
screen_write_clearscreen(sctx);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_EL:
switch (input_get(ictx, 0, 0, 0)) {
case 0:
screen_write_clearendofline(sctx);
break;
case 1:
screen_write_clearstartofline(sctx);
break;
case 2:
screen_write_clearline(sctx);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_HPA:
n = input_get(ictx, 0, 1, 1);
screen_write_cursormove(sctx, n - 1, s->cy);
break;
case INPUT_CSI_ICH:
screen_write_insertcharacter(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_IL:
screen_write_insertline(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_RM:
switch (input_get(ictx, 0, 0, -1)) {
case 4: /* IRM */
screen_write_insertmode(&ictx->ctx, 0);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_RM_PRIVATE:
switch (input_get(ictx, 0, 0, -1)) {
case 1: /* GATM */
screen_write_kcursormode(&ictx->ctx, 0);
break;
case 3: /* DECCOLM */
screen_write_cursormove(&ictx->ctx, 0, 0);
screen_write_clearscreen(&ictx->ctx);
break;
case 25: /* TCEM */
screen_write_cursormode(&ictx->ctx, 0);
break;
case 1000:
screen_write_mousemode(&ictx->ctx, 0);
break;
case 1049:
window_pane_alternate_off(wp, &ictx->cell);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_SGR:
input_csi_dispatch_sgr(ictx);
break;
case INPUT_CSI_SM:
switch (input_get(ictx, 0, 0, -1)) {
case 4: /* IRM */
screen_write_insertmode(&ictx->ctx, 1);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_SM_PRIVATE:
switch (input_get(ictx, 0, 0, -1)) {
case 1: /* GATM */
screen_write_kcursormode(&ictx->ctx, 1);
break;
case 3: /* DECCOLM */
screen_write_cursormove(&ictx->ctx, 0, 0);
screen_write_clearscreen(&ictx->ctx);
break;
case 25: /* TCEM */
screen_write_cursormode(&ictx->ctx, 1);
break;
case 1000:
screen_write_mousemode(&ictx->ctx, 1);
break;
case 1049:
window_pane_alternate_on(wp, &ictx->cell);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_TBC:
switch (input_get(ictx, 0, 0, 0)) {
case 0:
if (s->cx < screen_size_x(s))
bit_clear(s->tabs, s->cx);
break;
case 3:
bit_nclear(s->tabs, 0, screen_size_x(s) - 1);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_VPA:
n = input_get(ictx, 0, 1, 1);
screen_write_cursormove(sctx, s->cx, n - 1);
break;
}
return (0);
}
/* Sync up the core_bitmap with the CPU array using cyclic distribution
*
* The CPU array contains the distribution of CPUs, which can include
* virtual CPUs (hyperthreads)
*/
static int _cyclic_sync_core_bitmap(struct job_record *job_ptr,
const uint16_t cr_type)
{
uint32_t c, i, j, s, n, *sock_start, *sock_end, size, csize, core_cnt;
uint16_t cps = 0, cpus, vpus, sockets, sock_size;
job_resources_t *job_res = job_ptr->job_resrcs;
bitstr_t *core_map;
bool *sock_used, *sock_avoid;
bool alloc_cores = false, alloc_sockets = false;
uint16_t ntasks_per_core = 0xffff, ntasks_per_socket = 0xffff;
int error_code = SLURM_SUCCESS;
if ((job_res == NULL) || (job_res->core_bitmap == NULL))
return error_code;
if (cr_type & CR_CORE)
alloc_cores = true;
if (slurmctld_conf.select_type_param & CR_ALLOCATE_FULL_SOCKET) {
if (cr_type & CR_SOCKET)
alloc_sockets = true;
} else {
if (cr_type & CR_SOCKET)
alloc_cores = true;
}
core_map = job_res->core_bitmap;
if (job_ptr->details && job_ptr->details->mc_ptr) {
multi_core_data_t *mc_ptr = job_ptr->details->mc_ptr;
if (mc_ptr->ntasks_per_core) {
ntasks_per_core = mc_ptr->ntasks_per_core;
}
if ((mc_ptr->threads_per_core != (uint16_t) NO_VAL) &&
(mc_ptr->threads_per_core < ntasks_per_core)) {
ntasks_per_core = mc_ptr->threads_per_core;
}
if (mc_ptr->ntasks_per_socket)
ntasks_per_socket = mc_ptr->ntasks_per_socket;
}
sock_size = select_node_record[0].sockets;
sock_avoid = xmalloc(sock_size * sizeof(bool));
sock_start = xmalloc(sock_size * sizeof(uint32_t));
sock_end = xmalloc(sock_size * sizeof(uint32_t));
sock_used = xmalloc(sock_size * sizeof(bool));
size = bit_size(job_res->node_bitmap);
csize = bit_size(core_map);
for (c = 0, i = 0, n = 0; n < size; n++) {
if (bit_test(job_res->node_bitmap, n) == 0)
continue;
sockets = select_node_record[n].sockets;
cps = select_node_record[n].cores;
vpus = MIN(select_node_record[n].vpus, ntasks_per_core);
if (select_debug_flags & DEBUG_FLAG_CPU_BIND) {
info("DEBUG: job %u node %s vpus %u cpus %u",
job_ptr->job_id,
select_node_record[n].node_ptr->name,
vpus, job_res->cpus[i]);
}
if ((c + (sockets * cps)) > csize)
fatal("cons_res: _cyclic_sync_core_bitmap index error");
if (sockets > sock_size) {
sock_size = sockets;
xrealloc(sock_avoid, sock_size * sizeof(bool));
xrealloc(sock_start, sock_size * sizeof(uint32_t));
xrealloc(sock_end, sock_size * sizeof(uint32_t));
xrealloc(sock_used, sock_size * sizeof(bool));
}
for (s = 0; s < sockets; s++) {
sock_start[s] = c + (s * cps);
sock_end[s] = sock_start[s] + cps;
sock_avoid[s] = false;
sock_used[s] = false;
}
core_cnt = 0;
cpus = job_res->cpus[i];
if (ntasks_per_socket != 0xffff) {
int x_cpus, cpus_per_socket;
uint32_t total_cpus = 0;
uint32_t *cpus_cnt;
cpus_per_socket = ntasks_per_socket *
job_ptr->details->cpus_per_task;
cpus_cnt = xmalloc(sizeof(uint32_t) * sockets);
for (s = 0; s < sockets; s++) {
for (j = sock_start[s]; j < sock_end[s]; j++) {
if (bit_test(core_map, j))
cpus_cnt[s] += vpus;
}
total_cpus += cpus_cnt[s];
}
for (s = 0; s < sockets && total_cpus > cpus; s++) {
if (cpus_cnt[s] > cpus_per_socket) {
x_cpus = cpus_cnt[s] - cpus_per_socket;
cpus_cnt[s] = cpus_per_socket;
total_cpus -= x_cpus;
}
}
for (s = 0; s < sockets && total_cpus > cpus; s++) {
if ((cpus_cnt[s] <= cpus_per_socket) &&
(total_cpus - cpus_cnt[s] >= cpus)) {
sock_avoid[s] = true;
total_cpus -= cpus_cnt[s];
}
}
xfree(cpus_cnt);
} else if (job_ptr->details->cpus_per_task > 1) {
/* Try to pack all CPUs of each tasks on one socket. */
uint32_t *cpus_cnt, cpus_per_task;
cpus_per_task = job_ptr->details->cpus_per_task;
cpus_cnt = xmalloc(sizeof(uint32_t) * sockets);
for (s = 0; s < sockets; s++) {
for (j = sock_start[s]; j < sock_end[s]; j++) {
if (bit_test(core_map, j))
cpus_cnt[s] += vpus;
}
cpus_cnt[s] -= (cpus_cnt[s] % cpus_per_task);
}
for (s = 0; ((s < sockets) && (cpus > 0)); s++) {
while ((sock_start[s] < sock_end[s]) &&
(cpus_cnt[s] > 0) && (cpus > 0)) {
if (bit_test(core_map, sock_start[s])) {
sock_used[s] = true;
core_cnt++;
if (cpus_cnt[s] < vpus)
cpus_cnt[s] = 0;
else
cpus_cnt[s] -= vpus;
if (cpus < vpus)
cpus = 0;
else
cpus -= vpus;
}
sock_start[s]++;
}
}
xfree(cpus_cnt);
}
while (cpus > 0) {
uint16_t prev_cpus = cpus;
for (s = 0; s < sockets && cpus > 0; s++) {
if (sock_avoid[s])
continue;
while (sock_start[s] < sock_end[s]) {
if (bit_test(core_map, sock_start[s])) {
sock_used[s] = true;
core_cnt++;
break;
} else
sock_start[s]++;
}
if (sock_start[s] == sock_end[s])
/* this socket is unusable */
continue;
if (cpus < vpus)
cpus = 0;
else
cpus -= vpus;
sock_start[s]++;
}
if (prev_cpus == cpus) {
/* we're stuck! */
job_ptr->priority = 0;
job_ptr->state_reason = WAIT_HELD;
error("cons_res: sync loop not progressing, "
"holding job %u", job_ptr->job_id);
error_code = SLURM_ERROR;
goto fini;
}
}
/* clear the rest of the cores in each socket
* FIXME: do we need min_core/min_socket checks here? */
for (s = 0; s < sockets; s++) {
if (sock_start[s] == sock_end[s])
continue;
if (!alloc_sockets || !sock_used[s]) {
bit_nclear(core_map, sock_start[s],
sock_end[s]-1);
}
if ((select_node_record[n].vpus >= 1) &&
(alloc_sockets || alloc_cores) && sock_used[s]) {
for (j=sock_start[s]; j<sock_end[s]; j++) {
/* Mark all cores as used */
if (alloc_sockets)
bit_set(core_map, j);
if (bit_test(core_map, j))
core_cnt++;
}
}
}
if ((alloc_cores || alloc_sockets) &&
(select_node_record[n].vpus >= 1)) {
job_res->cpus[i] = core_cnt *
select_node_record[n].vpus;
}
i++;
/* advance 'c' to the beginning of the next node */
c += sockets * cps;
}
fini: xfree(sock_avoid);
xfree(sock_start);
xfree(sock_end);
xfree(sock_used);
return error_code;
}
/*
* _build_part_bitmap - update the total_cpus, total_nodes, and node_bitmap
* for the specified partition, also reset the partition pointers in
* the node back to this partition.
* IN part_ptr - pointer to the partition
* RET 0 if no error, errno otherwise
* global: node_record_table_ptr - pointer to global node table
* NOTE: this does not report nodes defined in more than one partition. this
* is checked only upon reading the configuration file, not on an update
*/
static int _build_part_bitmap(struct part_record *part_ptr)
{
char *this_node_name;
bitstr_t *old_bitmap;
struct node_record *node_ptr; /* pointer to node_record */
hostlist_t host_list;
part_ptr->total_cpus = 0;
part_ptr->total_nodes = 0;
if (part_ptr->node_bitmap == NULL) {
part_ptr->node_bitmap = bit_alloc(node_record_count);
old_bitmap = NULL;
} else {
old_bitmap = bit_copy(part_ptr->node_bitmap);
bit_nclear(part_ptr->node_bitmap, 0,
node_record_count - 1);
}
if (part_ptr->nodes == NULL) { /* no nodes in partition */
_unlink_free_nodes(old_bitmap, part_ptr);
FREE_NULL_BITMAP(old_bitmap);
return 0;
}
if ((host_list = hostlist_create(part_ptr->nodes)) == NULL) {
FREE_NULL_BITMAP(old_bitmap);
error("hostlist_create error on %s, %m",
part_ptr->nodes);
return ESLURM_INVALID_NODE_NAME;
}
while ((this_node_name = hostlist_shift(host_list))) {
node_ptr = find_node_record(this_node_name);
if (node_ptr == NULL) {
error("_build_part_bitmap: invalid node name %s",
this_node_name);
free(this_node_name);
FREE_NULL_BITMAP(old_bitmap);
hostlist_destroy(host_list);
return ESLURM_INVALID_NODE_NAME;
}
part_ptr->total_nodes++;
if (slurmctld_conf.fast_schedule)
part_ptr->total_cpus += node_ptr->config_ptr->cpus;
else
part_ptr->total_cpus += node_ptr->cpus;
node_ptr->part_cnt++;
xrealloc(node_ptr->part_pptr, (node_ptr->part_cnt *
sizeof(struct part_record *)));
node_ptr->part_pptr[node_ptr->part_cnt-1] = part_ptr;
if (old_bitmap)
bit_clear(old_bitmap,
(int) (node_ptr -
node_record_table_ptr));
bit_set(part_ptr->node_bitmap,
(int) (node_ptr - node_record_table_ptr));
free(this_node_name);
}
hostlist_destroy(host_list);
_unlink_free_nodes(old_bitmap, part_ptr);
last_node_update = time(NULL);
FREE_NULL_BITMAP(old_bitmap);
return 0;
}
/* Perform any power change work to nodes */
static void _do_power_work(time_t now)
{
static time_t last_log = 0, last_work_scan = 0;
int i, wake_cnt = 0, sleep_cnt = 0, susp_total = 0;
time_t delta_t;
uint32_t susp_state;
bitstr_t *wake_node_bitmap = NULL, *sleep_node_bitmap = NULL;
struct node_record *node_ptr;
bool run_suspend = false;
/* Set limit on counts of nodes to have state changed */
delta_t = now - last_work_scan;
if (delta_t >= 60) {
suspend_cnt_f = 0.0;
resume_cnt_f = 0.0;
} else {
float rate = (60 - delta_t) / 60.0;
suspend_cnt_f *= rate;
resume_cnt_f *= rate;
}
suspend_cnt = (suspend_cnt_f + 0.5);
resume_cnt = (resume_cnt_f + 0.5);
if (now > (last_suspend + suspend_timeout)) {
/* ready to start another round of node suspends */
run_suspend = true;
if (last_suspend) {
bit_nclear(suspend_node_bitmap, 0,
(node_record_count - 1));
bit_nclear(resume_node_bitmap, 0,
(node_record_count - 1));
last_suspend = (time_t) 0;
}
}
last_work_scan = now;
/* Build bitmaps identifying each node which should change state */
for (i = 0, node_ptr = node_record_table_ptr;
i < node_record_count; i++, node_ptr++) {
susp_state = IS_NODE_POWER_SAVE(node_ptr);
if (susp_state)
susp_total++;
/* Resume nodes as appropriate */
if (susp_state &&
((resume_rate == 0) || (resume_cnt < resume_rate)) &&
(bit_test(suspend_node_bitmap, i) == 0) &&
(IS_NODE_ALLOCATED(node_ptr) ||
(node_ptr->last_idle > (now - idle_time)))) {
if (wake_node_bitmap == NULL) {
wake_node_bitmap =
bit_alloc(node_record_count);
}
wake_cnt++;
resume_cnt++;
resume_cnt_f++;
node_ptr->node_state &= (~NODE_STATE_POWER_SAVE);
node_ptr->node_state |= NODE_STATE_POWER_UP;
node_ptr->node_state |= NODE_STATE_NO_RESPOND;
bit_clear(power_node_bitmap, i);
bit_clear(avail_node_bitmap, i);
node_ptr->last_response = now + resume_timeout;
bit_set(wake_node_bitmap, i);
bit_set(resume_node_bitmap, i);
}
/* Suspend nodes as appropriate */
if (run_suspend &&
(susp_state == 0) &&
((suspend_rate == 0) || (suspend_cnt < suspend_rate)) &&
(IS_NODE_IDLE(node_ptr) || IS_NODE_DOWN(node_ptr)) &&
(node_ptr->sus_job_cnt == 0) &&
(!IS_NODE_COMPLETING(node_ptr)) &&
(!IS_NODE_POWER_UP(node_ptr)) &&
(node_ptr->last_idle != 0) &&
(node_ptr->last_idle < (now - idle_time)) &&
((exc_node_bitmap == NULL) ||
(bit_test(exc_node_bitmap, i) == 0))) {
if (sleep_node_bitmap == NULL) {
sleep_node_bitmap =
bit_alloc(node_record_count);
}
sleep_cnt++;
suspend_cnt++;
suspend_cnt_f++;
node_ptr->node_state |= NODE_STATE_POWER_SAVE;
node_ptr->node_state &= (~NODE_STATE_NO_RESPOND);
if (!IS_NODE_DOWN(node_ptr) &&
!IS_NODE_DRAIN(node_ptr))
bit_set(avail_node_bitmap, i);
bit_set(power_node_bitmap, i);
bit_set(sleep_node_bitmap, i);
bit_set(suspend_node_bitmap, i);
last_suspend = now;
}
}
if (((now - last_log) > 600) && (susp_total > 0)) {
info("Power save mode: %d nodes", susp_total);
last_log = now;
}
if (sleep_node_bitmap) {
char *nodes;
nodes = bitmap2node_name(sleep_node_bitmap);
if (nodes)
_do_suspend(nodes);
else
error("power_save: bitmap2nodename");
xfree(nodes);
FREE_NULL_BITMAP(sleep_node_bitmap);
/* last_node_update could be changed already by another thread!
last_node_update = now; */
}
if (wake_node_bitmap) {
char *nodes;
nodes = bitmap2node_name(wake_node_bitmap);
if (nodes)
_do_resume(nodes);
else
error("power_save: bitmap2nodename");
xfree(nodes);
FREE_NULL_BITMAP(wake_node_bitmap);
/* last_node_update could be changed already by another thread!
last_node_update = now; */
}
}
extern List as_mysql_modify_qos(mysql_conn_t *mysql_conn, uint32_t uid,
slurmdb_qos_cond_t *qos_cond,
slurmdb_qos_rec_t *qos)
{
ListIterator itr = NULL;
List ret_list = NULL;
int rc = SLURM_SUCCESS;
char *object = NULL;
char *vals = NULL, *extra = NULL, *query = NULL, *name_char = NULL;
time_t now = time(NULL);
char *user_name = NULL;
int set = 0;
MYSQL_RES *result = NULL;
MYSQL_ROW row;
char *tmp_char1=NULL, *tmp_char2=NULL;
bitstr_t *preempt_bitstr = NULL;
char *added_preempt = NULL;
if (!qos_cond || !qos) {
error("we need something to change");
return NULL;
}
if (check_connection(mysql_conn) != SLURM_SUCCESS)
return NULL;
xstrcat(extra, "where deleted=0");
if (qos_cond->description_list
&& list_count(qos_cond->description_list)) {
set = 0;
xstrcat(extra, " && (");
itr = list_iterator_create(qos_cond->description_list);
while ((object = list_next(itr))) {
if (set)
xstrcat(extra, " || ");
xstrfmtcat(extra, "description='%s'", object);
set = 1;
}
list_iterator_destroy(itr);
xstrcat(extra, ")");
}
if (qos_cond->id_list
&& list_count(qos_cond->id_list)) {
set = 0;
xstrcat(extra, " && (");
itr = list_iterator_create(qos_cond->id_list);
while ((object = list_next(itr))) {
if (set)
xstrcat(extra, " || ");
xstrfmtcat(extra, "id='%s'", object);
set = 1;
}
list_iterator_destroy(itr);
xstrcat(extra, ")");
}
if (qos_cond->name_list
&& list_count(qos_cond->name_list)) {
set = 0;
xstrcat(extra, " && (");
itr = list_iterator_create(qos_cond->name_list);
while ((object = list_next(itr))) {
if (set)
xstrcat(extra, " || ");
xstrfmtcat(extra, "name='%s'", object);
set = 1;
}
list_iterator_destroy(itr);
xstrcat(extra, ")");
}
_setup_qos_limits(qos, &tmp_char1, &tmp_char2,
&vals, &added_preempt, 0);
if (added_preempt) {
preempt_bitstr = bit_alloc(g_qos_count);
bit_unfmt(preempt_bitstr, added_preempt+1);
xfree(added_preempt);
}
xfree(tmp_char1);
xfree(tmp_char2);
if (!extra || !vals) {
errno = SLURM_NO_CHANGE_IN_DATA;
FREE_NULL_BITMAP(preempt_bitstr);
error("Nothing to change");
return NULL;
}
query = xstrdup_printf("select name, preempt, id from %s %s;",
qos_table, extra);
xfree(extra);
if (!(result = mysql_db_query_ret(mysql_conn, query, 0))) {
xfree(query);
FREE_NULL_BITMAP(preempt_bitstr);
return NULL;
}
rc = 0;
ret_list = list_create(slurm_destroy_char);
while ((row = mysql_fetch_row(result))) {
slurmdb_qos_rec_t *qos_rec = NULL;
uint32_t id = slurm_atoul(row[2]);
if (preempt_bitstr) {
if (_preemption_loop(mysql_conn, id, preempt_bitstr))
break;
}
object = xstrdup(row[0]);
list_append(ret_list, object);
if (!rc) {
xstrfmtcat(name_char, "(name='%s'", object);
rc = 1;
} else {
xstrfmtcat(name_char, " || name='%s'", object);
}
qos_rec = xmalloc(sizeof(slurmdb_qos_rec_t));
qos_rec->name = xstrdup(object);
qos_rec->id = id;
qos_rec->flags = qos->flags;
qos_rec->grp_cpus = qos->grp_cpus;
qos_rec->grace_time = qos->grace_time;
qos_rec->grp_cpu_mins = qos->grp_cpu_mins;
qos_rec->grp_cpu_run_mins = qos->grp_cpu_run_mins;
qos_rec->grp_jobs = qos->grp_jobs;
qos_rec->grp_nodes = qos->grp_nodes;
qos_rec->grp_submit_jobs = qos->grp_submit_jobs;
qos_rec->grp_wall = qos->grp_wall;
qos_rec->max_cpus_pj = qos->max_cpus_pj;
qos_rec->max_cpu_mins_pj = qos->max_cpu_mins_pj;
qos_rec->max_cpu_run_mins_pu = qos->max_cpu_run_mins_pu;
qos_rec->max_jobs_pu = qos->max_jobs_pu;
qos_rec->max_nodes_pj = qos->max_nodes_pj;
qos_rec->max_submit_jobs_pu = qos->max_submit_jobs_pu;
qos_rec->max_wall_pj = qos->max_wall_pj;
qos_rec->preempt_mode = qos->preempt_mode;
qos_rec->priority = qos->priority;
if (qos->preempt_list) {
ListIterator new_preempt_itr =
list_iterator_create(qos->preempt_list);
char *new_preempt = NULL;
qos_rec->preempt_bitstr = bit_alloc(g_qos_count);
if (row[1] && row[1][0])
bit_unfmt(qos_rec->preempt_bitstr, row[1]+1);
while ((new_preempt = list_next(new_preempt_itr))) {
bool cleared = 0;
if (new_preempt[0] == '-') {
bit_clear(qos_rec->preempt_bitstr,
atol(new_preempt+1));
} else if (new_preempt[0] == '+') {
bit_set(qos_rec->preempt_bitstr,
atol(new_preempt+1));
} else {
if (!cleared) {
cleared = 1;
bit_nclear(
qos_rec->preempt_bitstr,
0,
g_qos_count-1);
}
bit_set(qos_rec->preempt_bitstr,
atol(new_preempt));
}
}
list_iterator_destroy(new_preempt_itr);
}
qos_rec->usage_factor = qos->usage_factor;
qos_rec->usage_thres = qos->usage_thres;
if (addto_update_list(mysql_conn->update_list,
SLURMDB_MODIFY_QOS, qos_rec)
!= SLURM_SUCCESS)
slurmdb_destroy_qos_rec(qos_rec);
}
mysql_free_result(result);
FREE_NULL_BITMAP(preempt_bitstr);
if (row) {
xfree(vals);
xfree(name_char);
xfree(query);
list_destroy(ret_list);
ret_list = NULL;
errno = ESLURM_QOS_PREEMPTION_LOOP;
return ret_list;
}
if (!list_count(ret_list)) {
errno = SLURM_NO_CHANGE_IN_DATA;
debug3("didn't effect anything\n%s", query);
xfree(vals);
xfree(query);
return ret_list;
}
xfree(query);
xstrcat(name_char, ")");
user_name = uid_to_string((uid_t) uid);
rc = modify_common(mysql_conn, DBD_MODIFY_QOS, now,
user_name, qos_table, name_char, vals, NULL);
xfree(user_name);
xfree(name_char);
xfree(vals);
if (rc == SLURM_ERROR) {
error("Couldn't modify qos");
list_destroy(ret_list);
ret_list = NULL;
}
return ret_list;
}
/* sync up core bitmap with new CPU count using a best-fit approach
* on the available resources on each node
*
* "Best-fit" means:
* 1st priority: Use smallest number of boards with sufficient
* available CPUs
* 2nd priority: Use smallest number of sockets with sufficient
* available CPUs
* 3rd priority: Use board combination with the smallest number
* of available CPUs
* 4th priority: Use higher-numbered boards/sockets/cores first
*
* The CPU array contains the distribution of CPUs, which can include
* virtual CPUs (hyperthreads)
*/
static void _block_sync_core_bitmap(struct job_record *job_ptr,
const uint16_t cr_type)
{
uint32_t c, s, i, j, n, b, z, size, csize, core_cnt;
uint16_t cpus, num_bits, vpus = 1;
uint16_t cpus_per_task = job_ptr->details->cpus_per_task;
job_resources_t *job_res = job_ptr->job_resrcs;
bool alloc_cores = false, alloc_sockets = false;
uint16_t ntasks_per_core = 0xffff;
int tmp_cpt = 0;
int count, cpu_min, b_min, elig, s_min, comb_idx, sock_idx;
int elig_idx, comb_brd_idx, sock_list_idx, comb_min, board_num;
int sock_per_comb;
int* boards_core_cnt;
int* sort_brds_core_cnt;
int* board_combs;
int* socket_list;
int* elig_brd_combs;
int* elig_core_cnt;
bool* sockets_used;
uint16_t boards_nb;
uint16_t nboards_nb;
uint16_t sockets_nb;
uint16_t ncores_nb;
uint16_t nsockets_nb;
uint16_t sock_per_brd;
uint16_t req_cores,best_fit_cores = 0;
uint32_t best_fit_location = 0;
uint64_t ncomb_brd;
bool sufficient, best_fit_sufficient;
if (!job_res)
return;
if (!job_res->core_bitmap) {
error("%s: core_bitmap for %pJ is NULL",
__func__, job_ptr);
return;
}
if (bit_ffs(job_res->core_bitmap) == -1) {
error("%s: core_bitmap for job %pJ has no bits set",
__func__, job_ptr);
return;
}
if (cr_type & CR_SOCKET)
alloc_sockets = true;
else if (cr_type & CR_CORE)
alloc_cores = true;
if (job_ptr->details && job_ptr->details->mc_ptr) {
multi_core_data_t *mc_ptr = job_ptr->details->mc_ptr;
if ((mc_ptr->ntasks_per_core != INFINITE16) &&
(mc_ptr->ntasks_per_core)) {
ntasks_per_core = mc_ptr->ntasks_per_core;
}
}
size = bit_size(job_res->node_bitmap);
csize = bit_size(job_res->core_bitmap);
sockets_nb = select_node_record[0].sockets;
sockets_core_cnt = xmalloc(sockets_nb * sizeof(int));
sockets_used = xmalloc(sockets_nb * sizeof(bool));
boards_nb = select_node_record[0].boards;
boards_core_cnt = xmalloc(boards_nb * sizeof(int));
sort_brds_core_cnt = xmalloc(boards_nb * sizeof(int));
for (c = 0, i = 0, n = 0; n < size; n++) {
if (bit_test(job_res->node_bitmap, n) == 0)
continue;
core_cnt = 0;
ncores_nb = select_node_record[n].cores;
nsockets_nb = select_node_record[n].sockets;
nboards_nb = select_node_record[n].boards;
num_bits = nsockets_nb * ncores_nb;
if ((c + num_bits) > csize)
fatal("cons_res: _block_sync_core_bitmap index error");
cpus = job_res->cpus[i];
vpus = cr_cpus_per_core(job_ptr->details, n);
/* compute still required cores on the node */
req_cores = cpus / vpus;
if ( cpus % vpus )
req_cores++;
/* figure out core cnt if task requires more than one core and
* tasks_per_core is 1 */
if ((ntasks_per_core == 1) &&
(cpus_per_task > vpus)) {
/* how many cores a task will consume */
int cores_per_task = (cpus_per_task + vpus - 1) / vpus;
int tasks = cpus / cpus_per_task;
req_cores = tasks * cores_per_task;
}
if (nboards_nb > MAX_BOARDS) {
debug3("cons_res: node[%u]: exceeds max boards; "
"doing best-fit across sockets only", n);
nboards_nb = 1;
}
if ( nsockets_nb > sockets_nb) {
sockets_nb = nsockets_nb;
xrealloc(sockets_core_cnt, sockets_nb * sizeof(int));
xrealloc(sockets_used,sockets_nb * sizeof(bool));
}
if ( nboards_nb > boards_nb) {
boards_nb = nboards_nb;
xrealloc(boards_core_cnt, boards_nb * sizeof(int));
xrealloc(sort_brds_core_cnt, boards_nb * sizeof(int));
}
/* Count available cores on each socket and board */
if (nsockets_nb >= nboards_nb) {
sock_per_brd = nsockets_nb / nboards_nb;
} else {
error("Node socket count lower than board count (%u < %u), %pJ node %s",
nsockets_nb, nboards_nb, job_ptr,
node_record_table_ptr[n].name);
sock_per_brd = 1;
}
for (b = 0; b < nboards_nb; b++) {
boards_core_cnt[b] = 0;
sort_brds_core_cnt[b] = 0;
}
for (s = 0; s < nsockets_nb; s++) {
sockets_core_cnt[s]=0;
sockets_used[s]=false;
b = s/sock_per_brd;
for ( j = c + (s * ncores_nb) ;
j < c + ((s+1) * ncores_nb) ;
j++ ) {
if ( bit_test(job_res->core_bitmap,j) ) {
sockets_core_cnt[s]++;
boards_core_cnt[b]++;
sort_brds_core_cnt[b]++;
}
}
}
/* Sort boards in descending order of available core count */
qsort(sort_brds_core_cnt, nboards_nb, sizeof (int),
_cmp_int_descend);
/* Determine minimum number of boards required for the
* allocation (b_min) */
count = 0;
for (b = 0; b < nboards_nb; b++) {
count+=sort_brds_core_cnt[b];
if (count >= req_cores)
break;
}
b_min = b+1;
sock_per_comb = b_min * sock_per_brd;
/* Allocate space for list of board combinations */
ncomb_brd = comb_counts[nboards_nb-1][b_min-1];
board_combs = xmalloc(ncomb_brd * b_min * sizeof(int));
/* Generate all combinations of b_min boards on the node */
_gen_combs(board_combs, nboards_nb, b_min);
/* Determine which combinations have enough available cores
* for the allocation (eligible board combinations)
*/
elig_brd_combs = xmalloc(ncomb_brd * sizeof(int));
elig_core_cnt = xmalloc(ncomb_brd * sizeof(int));
elig = 0;
for (comb_idx = 0; comb_idx < ncomb_brd; comb_idx++) {
count = 0;
for (comb_brd_idx = 0; comb_brd_idx < b_min;
comb_brd_idx++) {
board_num = board_combs[(comb_idx * b_min)
+ comb_brd_idx];
count += boards_core_cnt[board_num];
}
if (count >= req_cores) {
elig_brd_combs[elig] = comb_idx;
elig_core_cnt[elig] = count;
elig++;
}
}
/* Allocate space for list of sockets for each eligible board
* combination */
socket_list = xmalloc(elig * sock_per_comb * sizeof(int));
/* Generate sorted list of sockets for each eligible board
* combination, and find combination with minimum number
* of sockets and minimum number of cpus required for the
* allocation
*/
s_min = sock_per_comb;
comb_min = 0;
cpu_min = sock_per_comb * ncores_nb;
for (elig_idx = 0; elig_idx < elig; elig_idx++) {
comb_idx = elig_brd_combs[elig_idx];
for (comb_brd_idx = 0; comb_brd_idx < b_min;
comb_brd_idx++) {
board_num = board_combs[(comb_idx * b_min)
+ comb_brd_idx];
sock_list_idx = (elig_idx * sock_per_comb) +
(comb_brd_idx * sock_per_brd);
for (sock_idx = 0; sock_idx < sock_per_brd;
sock_idx++) {
socket_list[sock_list_idx + sock_idx]
= (board_num * sock_per_brd)
+ sock_idx;
}
}
/* Sort this socket list in descending order of
* available core count */
qsort(&socket_list[elig_idx*sock_per_comb],
sock_per_comb, sizeof (int), _cmp_sock);
/* Determine minimum number of sockets required for
* the allocation from this socket list */
count = 0;
for (b = 0; b < sock_per_comb; b++) {
sock_idx =
socket_list[(int)((elig_idx*sock_per_comb)+b)];
count+=sockets_core_cnt[sock_idx];
if (count >= req_cores)
break;
}
b++;
/* Use board combination with minimum number
* of required sockets and minimum number of CPUs
*/
if ((b < s_min) ||
(b == s_min && elig_core_cnt[elig_idx]
<= cpu_min)) {
s_min = b;
comb_min = elig_idx;
cpu_min = elig_core_cnt[elig_idx];
}
}
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("cons_res: best_fit: node[%u]: "
"required cpus: %u, min req boards: %u,",
n, cpus, b_min);
info("cons_res: best_fit: node[%u]: "
"min req sockets: %u, min avail cores: %u",
n, s_min, cpu_min);
}
/* Re-sort socket list for best-fit board combination in
* ascending order of socket number */
qsort(&socket_list[comb_min * sock_per_comb], sock_per_comb,
sizeof (int), _cmp_int_ascend);
xfree(board_combs);
xfree(elig_brd_combs);
xfree(elig_core_cnt);
/* select cores from the sockets of the best-fit board
* combination using a best-fit approach */
tmp_cpt = cpus_per_task;
while ( cpus > 0 ) {
best_fit_cores = 0;
best_fit_sufficient = false;
/* search for the socket with best fit */
for ( z = 0; z < sock_per_comb; z++ ) {
s = socket_list[(comb_min*sock_per_comb)+z];
sufficient = sockets_core_cnt[s] >= req_cores;
if ( (best_fit_cores == 0) ||
(sufficient && !best_fit_sufficient ) ||
(sufficient && (sockets_core_cnt[s] <
best_fit_cores)) ||
(!sufficient && (sockets_core_cnt[s] >
best_fit_cores)) ) {
best_fit_cores = sockets_core_cnt[s];
best_fit_location = s;
best_fit_sufficient = sufficient;
}
}
/* check that we have found a usable socket */
if ( best_fit_cores == 0 )
break;
j = best_fit_location;
if (sock_per_brd)
j /= sock_per_brd;
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("cons_res: best_fit: using node[%u]: "
"board[%u]: socket[%u]: %u cores "
"available", n, j, best_fit_location,
sockets_core_cnt[best_fit_location]);
}
sockets_used[best_fit_location] = true;
for ( j = (c + (best_fit_location * ncores_nb));
j < (c + ((best_fit_location + 1) * ncores_nb));
j++ ) {
/*
* if no more cpus to select
* release remaining cores unless
* we are allocating whole sockets
*/
if (cpus == 0) {
if (alloc_sockets) {
bit_set(job_res->core_bitmap,j);
core_cnt++;
} else {
bit_clear(job_res->core_bitmap,j);
}
continue;
}
/*
* remove cores from socket count and
* cpus count using hyperthreading requirement
*/
if ( bit_test(job_res->core_bitmap, j) ) {
sockets_core_cnt[best_fit_location]--;
core_cnt++;
if (cpus < vpus)
cpus = 0;
else if ((ntasks_per_core == 1) &&
(cpus_per_task > vpus)) {
int used = MIN(tmp_cpt, vpus);
cpus -= used;
if (tmp_cpt <= used)
tmp_cpt = cpus_per_task;
else
tmp_cpt -= used;
} else {
cpus -= vpus;
}
} else if (alloc_sockets) {
/* If the core is not used, add it
* anyway if allocating whole sockets */
bit_set(job_res->core_bitmap, j);
core_cnt++;
}
}
/* loop again if more cpus required */
if ( cpus > 0 )
continue;
/* release remaining cores of the unused sockets */
for (s = 0; s < nsockets_nb; s++) {
if ( sockets_used[s] )
continue;
bit_nclear(job_res->core_bitmap,
c+(s*ncores_nb),
c+((s+1)*ncores_nb)-1);
}
}
xfree(socket_list);
if (cpus > 0) {
/* cpu count should NEVER be greater than the number
* of set bits in the core bitmap for a given node */
fatal("cons_res: cpus computation error");
}
/* adjust cpus count of the current node */
if ((alloc_cores || alloc_sockets) &&
(select_node_record[n].vpus >= 1)) {
job_res->cpus[i] = core_cnt *
select_node_record[n].vpus;
}
i++;
/* move c to the next node in core_bitmap */
c += num_bits;
}
xfree(boards_core_cnt);
xfree(sort_brds_core_cnt);
xfree(sockets_core_cnt);
xfree(sockets_used);
}
/* Sync up the core_bitmap with the CPU array using cyclic distribution
*
* The CPU array contains the distribution of CPUs, which can include
* virtual CPUs (hyperthreads)
*/
static void _cyclic_sync_core_bitmap(struct job_record *job_ptr,
const uint16_t cr_type)
{
uint32_t c, i, j, s, n, *sock_start, *sock_end, size, csize, core_cnt;
uint16_t cps = 0, cpus, vpus, sockets, sock_size;
job_resources_t *job_res = job_ptr->job_resrcs;
bitstr_t *core_map;
bool *sock_used, alloc_cores = false, alloc_sockets = false;
uint16_t ntasks_per_core = 0xffff;
if ((job_res == NULL) || (job_res->core_bitmap == NULL))
return;
if (cr_type & CR_CORE)
alloc_cores = true;
#ifdef ALLOCATE_FULL_SOCKET
if (cr_type & CR_SOCKET)
alloc_sockets = true;
#else
if (cr_type & CR_SOCKET)
alloc_cores = true;
#endif
core_map = job_res->core_bitmap;
if (job_ptr->details && job_ptr->details->mc_ptr) {
multi_core_data_t *mc_ptr = job_ptr->details->mc_ptr;
if (mc_ptr->ntasks_per_core) {
ntasks_per_core = mc_ptr->ntasks_per_core;
}
if ((mc_ptr->threads_per_core != (uint16_t) NO_VAL) &&
(mc_ptr->threads_per_core < ntasks_per_core)) {
ntasks_per_core = mc_ptr->threads_per_core;
}
}
sock_size = select_node_record[0].sockets;
sock_start = xmalloc(sock_size * sizeof(uint32_t));
sock_end = xmalloc(sock_size * sizeof(uint32_t));
sock_used = xmalloc(sock_size * sizeof(bool));
size = bit_size(job_res->node_bitmap);
csize = bit_size(core_map);
for (c = 0, i = 0, n = 0; n < size; n++) {
if (bit_test(job_res->node_bitmap, n) == 0)
continue;
sockets = select_node_record[n].sockets;
cps = select_node_record[n].cores;
vpus = MIN(select_node_record[n].vpus, ntasks_per_core);
if (select_debug_flags & DEBUG_FLAG_CPU_BIND) {
info("DEBUG: job %u node %s vpus %u cpus %u",
job_ptr->job_id,
select_node_record[n].node_ptr->name,
vpus, job_res->cpus[i]);
}
if ((c + (sockets * cps)) > csize)
fatal("cons_res: _cyclic_sync_core_bitmap index error");
if (sockets > sock_size) {
sock_size = sockets;
xrealloc(sock_start, sock_size * sizeof(uint32_t));
xrealloc(sock_end, sock_size * sizeof(uint32_t));
xrealloc(sock_used, sock_size * sizeof(bool));
}
for (s = 0; s < sockets; s++) {
sock_start[s] = c + (s * cps);
sock_end[s] = sock_start[s] + cps;
}
core_cnt = 0;
cpus = job_res->cpus[i];
while (cpus > 0) {
uint16_t prev_cpus = cpus;
for (s = 0; s < sockets && cpus > 0; s++) {
while (sock_start[s] < sock_end[s]) {
if (bit_test(core_map,sock_start[s])) {
sock_used[s] = true;
core_cnt++;
break;
} else
sock_start[s]++;
}
if (sock_start[s] == sock_end[s])
/* this socket is unusable */
continue;
if (cpus < vpus)
cpus = 0;
else
cpus -= vpus;
sock_start[s]++;
}
if (prev_cpus == cpus) {
/* we're stuck! */
fatal("cons_res: sync loop not progressing");
}
}
/* clear the rest of the cores in each socket
* FIXME: do we need min_core/min_socket checks here? */
for (s = 0; s < sockets; s++) {
if (sock_start[s] == sock_end[s])
continue;
if (!alloc_sockets || !sock_used[s]) {
bit_nclear(core_map, sock_start[s],
sock_end[s]-1);
}
if ((select_node_record[n].vpus > 1) &&
(alloc_sockets || alloc_cores) && sock_used[s]) {
for (j=sock_start[s]; j<sock_end[s]; j++) {
if (bit_test(core_map, j))
core_cnt++;
}
}
}
if ((alloc_cores || alloc_sockets) &&
(select_node_record[n].vpus > 1)) {
job_res->cpus[i] = core_cnt *
select_node_record[n].vpus;
}
i++;
/* advance 'c' to the beginning of the next node */
c += sockets * cps;
}
xfree(sock_start);
xfree(sock_end);
xfree(sock_used);
}
/* sync up core bitmap with new CPU count using a best-fit approach
* on the available sockets
*
* The CPU array contains the distribution of CPUs, which can include
* virtual CPUs (hyperthreads)
*/
static void _block_sync_core_bitmap(struct job_record *job_ptr,
const uint16_t cr_type)
{
uint32_t c, s, i, j, n, size, csize, core_cnt;
uint16_t cpus, num_bits, vpus = 1;
job_resources_t *job_res = job_ptr->job_resrcs;
bool alloc_cores = false, alloc_sockets = false;
uint16_t ntasks_per_core = 0xffff;
int* sockets_cpu_cnt;
bool* sockets_used;
uint16_t sockets_nb;
uint16_t ncores_nb;
uint16_t nsockets_nb;
uint16_t req_cpus,best_fit_cpus = 0;
uint32_t best_fit_location = 0;
bool sufficient,best_fit_sufficient;
if (!job_res)
return;
if (cr_type & CR_CORE)
alloc_cores = true;
#ifdef ALLOCATE_FULL_SOCKET
if (cr_type & CR_SOCKET)
alloc_sockets = true;
#else
if (cr_type & CR_SOCKET)
alloc_cores = true;
#endif
if (job_ptr->details && job_ptr->details->mc_ptr) {
multi_core_data_t *mc_ptr = job_ptr->details->mc_ptr;
if (mc_ptr->ntasks_per_core) {
ntasks_per_core = mc_ptr->ntasks_per_core;
}
if ((mc_ptr->threads_per_core != (uint16_t) NO_VAL) &&
(mc_ptr->threads_per_core < ntasks_per_core)) {
ntasks_per_core = mc_ptr->threads_per_core;
}
}
size = bit_size(job_res->node_bitmap);
csize = bit_size(job_res->core_bitmap);
sockets_nb = select_node_record[0].sockets;
sockets_cpu_cnt = xmalloc(sockets_nb * sizeof(int));
sockets_used = xmalloc(sockets_nb * sizeof(bool));
for (c = 0, i = 0, n = 0; n < size; n++) {
if (bit_test(job_res->node_bitmap, n) == 0)
continue;
core_cnt = 0;
ncores_nb = select_node_record[n].cores;
nsockets_nb = select_node_record[n].sockets;
num_bits = nsockets_nb * ncores_nb;
if ((c + num_bits) > csize)
fatal ("cons_res: _block_sync_core_bitmap index error");
cpus = job_res->cpus[i];
vpus = MIN(select_node_record[n].vpus, ntasks_per_core);
if ( nsockets_nb > sockets_nb) {
sockets_nb = nsockets_nb;
xrealloc(sockets_cpu_cnt, sockets_nb * sizeof(int));
xrealloc(sockets_used,sockets_nb * sizeof(bool));
}
/* count cores provided by each socket */
for (s = 0; s < nsockets_nb; s++) {
sockets_cpu_cnt[s]=0;
sockets_used[s]=false;
for ( j = c + (s * ncores_nb) ;
j < c + ((s+1) * ncores_nb) ;
j++ ) {
if ( bit_test(job_res->core_bitmap,j) )
sockets_cpu_cnt[s]++;
}
}
/* select cores in the sockets using a best-fit approach */
while( cpus > 0 ) {
best_fit_cpus = 0;
best_fit_sufficient = false;
/* compute still required cores on the node */
req_cpus = cpus / vpus;
if ( cpus % vpus )
req_cpus++;
/* search for the best socket, */
/* starting from the last one to let more room */
/* in the first one for system usage */
for ( s = nsockets_nb - 1 ; (int) s >= (int) 0 ; s-- ) {
sufficient = sockets_cpu_cnt[s] >= req_cpus ;
if ( (best_fit_cpus == 0) ||
(sufficient && !best_fit_sufficient ) ||
(sufficient && (sockets_cpu_cnt[s] <
best_fit_cpus)) ||
(!sufficient && (sockets_cpu_cnt[s] >
best_fit_cpus)) ) {
best_fit_cpus = sockets_cpu_cnt[s];
best_fit_location = s;
best_fit_sufficient = sufficient;
}
}
/* check that we have found a usable socket */
if ( best_fit_cpus == 0 )
break;
debug3("dist_task: best_fit : using node[%u]:"
"socket[%u] : %u cores availablex",
n, best_fit_location,
sockets_cpu_cnt[best_fit_location]);
/* select socket cores from last to first */
/* socket[0]:Core[0] would be the last one */
sockets_used[best_fit_location] = true;
for ( j = c + ((best_fit_location+1) * ncores_nb)
- 1 ;
(int) j >= (int) (c + (best_fit_location *
ncores_nb)) ;
j-- ) {
/*
* if no more cpus to select
* release remaining cores unless
* we are allocating whole sockets
*/
if ( cpus == 0 && alloc_sockets ) {
if ( bit_test(job_res->core_bitmap,j) )
core_cnt++;
continue;
}
else if ( cpus == 0 ) {
bit_clear(job_res->core_bitmap,j);
continue;
}
/*
* remove cores from socket count and
* cpus count using hyperthreading requirement
*/
if ( bit_test(job_res->core_bitmap,j) ) {
sockets_cpu_cnt[best_fit_location]--;
core_cnt++;
if (cpus < vpus)
cpus = 0;
else
cpus -= vpus;
}
}
/* loop again if more cpus required */
if ( cpus > 0 )
continue;
/* release remaining cores of the unused sockets */
for (s = 0; s < nsockets_nb; s++) {
if ( sockets_used[s] )
continue;
bit_nclear(job_res->core_bitmap,
c+(s*ncores_nb),
c+((s+1)*ncores_nb)-1);
}
}
if (cpus > 0) {
/* cpu count should NEVER be greater than the number
* of set bits in the core bitmap for a given node */
fatal("cons_res: cpus computation error");
}
/* adjust cpus count of the current node */
if ((alloc_cores || alloc_sockets) &&
(select_node_record[n].vpus > 1)) {
job_res->cpus[i] = core_cnt *
select_node_record[n].vpus;
}
i++;
/* move c to the next node in core_bitmap */
c += num_bits;
}
xfree(sockets_cpu_cnt);
xfree(sockets_used);
}
int
main(int argc, char *argv[])
{
note("Testing static decl");
{
bitstr_t bit_decl(bs, 65);
/*bitstr_t *bsp = bs;*/
bit_set(bs,9);
bit_set(bs,14);
TEST(bit_test(bs,9), "bit 9 set");
TEST(!bit_test(bs,12), "bit 12 not set");
TEST(bit_test(bs,14), "bit 14 set" );
/*bit_free(bsp);*/ /* triggers TEST in bit_free - OK */
}
note("Testing basic vixie functions");
{
bitstr_t *bs = bit_alloc(16), *bs2;
/*bit_set(bs, 42);*/ /* triggers TEST in bit_set - OK */
bit_set(bs,9);
bit_set(bs,14);
TEST(bit_test(bs,9), "bit 9 set");
TEST(!bit_test(bs,12), "bit 12 not set" );
TEST(bit_test(bs,14), "bit 14 set");
bs2 = bit_copy(bs);
bit_fill_gaps(bs2);
TEST(bit_ffs(bs2) == 9, "first bit set = 9 ");
TEST(bit_fls(bs2) == 14, "last bit set = 14");
TEST(bit_set_count(bs2) == 6, "bitstring");
TEST(bit_test(bs2,12), "bitstring");
TEST(bit_super_set(bs,bs2) == 1, "bitstring");
TEST(bit_super_set(bs2,bs) == 0, "bitstring");
bit_clear(bs,14);
TEST(!bit_test(bs,14), "bitstring");
bit_nclear(bs,9,14);
TEST(!bit_test(bs,9), "bitstring");
TEST(!bit_test(bs,12), "bitstring");
TEST(!bit_test(bs,14), "bitstring");
bit_nset(bs,9,14);
TEST(bit_test(bs,9), "bitstring");
TEST(bit_test(bs,12), "bitstring");
TEST(bit_test(bs,14), "bitstring");
TEST(bit_ffs(bs) == 9, "ffs");
TEST(bit_ffc(bs) == 0, "ffc");
bit_nset(bs,0,8);
TEST(bit_ffc(bs) == 15, "ffc");
bit_free(bs);
/*bit_set(bs,9); */ /* triggers TEST in bit_set - OK */
}
note("Testing and/or/not");
{
bitstr_t *bs1 = bit_alloc(128);
bitstr_t *bs2 = bit_alloc(128);
bit_set(bs1, 100);
bit_set(bs1, 104);
bit_set(bs2, 100);
bit_and(bs1, bs2);
TEST(bit_test(bs1, 100), "and");
TEST(!bit_test(bs1, 104), "and");
bit_set(bs2, 110);
bit_set(bs2, 111);
bit_set(bs2, 112);
bit_or(bs1, bs2);
TEST(bit_test(bs1, 100), "or");
TEST(bit_test(bs1, 110), "or");
TEST(bit_test(bs1, 111), "or");
TEST(bit_test(bs1, 112), "or");
bit_not(bs1);
TEST(!bit_test(bs1, 100), "not");
TEST(bit_test(bs1, 12), "not");
bit_free(bs1);
bit_free(bs2);
}
note("testing bit selection");
{
bitstr_t *bs1 = bit_alloc(128), *bs2;
bit_set(bs1, 21);
bit_set(bs1, 100);
bit_fill_gaps(bs1);
bs2 = bit_pick_cnt(bs1,20);
if (bs2) {
TEST(bit_set_count(bs2) == 20, "pick");
TEST(bit_ffs(bs2) == 21, "pick");
TEST(bit_fls(bs2) == 40, "pick");
bit_free(bs2);
}
else
TEST(0, "alloc fail");
bit_free(bs1);
}
note("Testing realloc");
{
bitstr_t *bs = bit_alloc(1);
TEST(bit_ffs(bs) == -1, "bitstring");
bit_set(bs,0);
/*bit_set(bs, 1000);*/ /* triggers TEST in bit_set - OK */
bs = bit_realloc(bs,1048576);
bit_set(bs,1000);
bit_set(bs,1048575);
TEST(bit_test(bs, 0), "bitstring");
TEST(bit_test(bs, 1000), "bitstring");
TEST(bit_test(bs, 1048575), "bitstring");
TEST(bit_set_count(bs) == 3, "bitstring");
bit_clear(bs,0);
bit_clear(bs,1000);
TEST(bit_set_count(bs) == 1, "bitstring");
TEST(bit_ffs(bs) == 1048575, "bitstring");
bit_free(bs);
}
note("Testing bit_fmt");
{
char tmpstr[1024];
bitstr_t *bs = bit_alloc(1024);
TEST(!strcmp(bit_fmt(tmpstr,sizeof(tmpstr),bs), ""), "bitstring");
bit_set(bs,42);
TEST(!strcmp(bit_fmt(tmpstr,sizeof(tmpstr),bs), "42"), "bitstring");
bit_set(bs,102);
TEST(!strcmp(bit_fmt(tmpstr,sizeof(tmpstr),bs), "42,102"), "bitstring");
bit_nset(bs,9,14);
TEST(!strcmp(bit_fmt(tmpstr,sizeof(tmpstr), bs),
"9-14,42,102"), "bitstring");
}
note("Testing bit_nffc/bit_nffs");
{
bitstr_t *bs = bit_alloc(1024);
bit_set(bs, 2);
bit_set(bs, 6);
bit_set(bs, 7);
bit_nset(bs,12,1018);
TEST(bit_nffc(bs, 2) == 0, "bitstring");
TEST(bit_nffc(bs, 3) == 3, "bitstring");
TEST(bit_nffc(bs, 4) == 8, "bitstring");
TEST(bit_nffc(bs, 5) == 1019, "bitstring");
TEST(bit_nffc(bs, 6) == -1, "bitstring");
TEST(bit_nffs(bs, 1) == 2, "bitstring");
TEST(bit_nffs(bs, 2) == 6, "bitstring");
TEST(bit_nffs(bs, 100) == 12, "bitstring");
TEST(bit_nffs(bs, 1023) == -1, "bitstring");
bit_free(bs);
}
note("Testing bit_unfmt");
{
bitstr_t *bs = bit_alloc(1024);
bitstr_t *bs2 = bit_alloc(1024);
char tmpstr[4096];
bit_set(bs,1);
bit_set(bs,3);
bit_set(bs,30);
bit_nset(bs,42,64);
bit_nset(bs,97,1000);
bit_fmt(tmpstr, sizeof(tmpstr), bs);
TEST(bit_unfmt(bs2, tmpstr) != -1, "bitstring");
TEST(bit_equal(bs, bs2), "bitstring");
}
totals();
return failed;
}
/*
* _can_job_run_on_node - Given the job requirements, determine which
* resources from the given node (if any) can be
* allocated to this job. Returns the number of
* cpus that can be used by this node and a bitmap
* of available resources for allocation.
* NOTE: This process does NOT support overcommitting resources
*
* IN job_ptr - pointer to job requirements
* IN/OUT core_map - core_bitmap of available cores
* IN n - index of node to be evaluated
* IN cr_type - Consumable Resource setting
* IN test_only - ignore allocated memory check
*
* NOTE: The returned cpu_count may be less than the number of set bits in
* core_map for the given node. The cr_dist functions will determine
* which bits to deselect from the core_map to match the cpu_count.
*/
uint16_t _can_job_run_on_node(struct job_record *job_ptr, bitstr_t *core_map,
const uint32_t node_i,
struct node_use_record *node_usage,
uint16_t cr_type,
bool test_only)
{
uint16_t cpus;
uint32_t avail_mem, req_mem, gres_cpus, gres_cores, cpus_per_core;
int core_start_bit, core_end_bit;
struct node_record *node_ptr = node_record_table_ptr + node_i;
List gres_list;
if (!test_only && IS_NODE_COMPLETING(node_ptr)) {
/* Do not allocate more jobs to nodes with completing jobs */
cpus = 0;
return cpus;
}
cpus = _allocate_cores(job_ptr, core_map, node_i);
core_start_bit = cr_get_coremap_offset(node_i);
core_end_bit = cr_get_coremap_offset(node_i + 1) - 1;
node_ptr = select_node_record[node_i].node_ptr;
cpus_per_core = select_node_record[node_i].cpus /
(core_end_bit - core_start_bit + 1);
if (node_usage[node_i].gres_list)
gres_list = node_usage[node_i].gres_list;
else
gres_list = node_ptr->gres_list;
gres_plugin_job_core_filter(job_ptr->gres_list, gres_list, test_only,
core_map, core_start_bit, core_end_bit,
node_ptr->name);
if ((cr_type & CR_MEMORY) && cpus) {
req_mem = job_ptr->details->pn_min_memory & ~MEM_PER_CPU;
avail_mem = select_node_record[node_i].real_memory;
if (!test_only)
avail_mem -= node_usage[node_i].alloc_memory;
if (req_mem > avail_mem)
cpus = 0;
}
gres_cores = gres_plugin_job_test(job_ptr->gres_list,
gres_list, test_only,
core_map, core_start_bit,
core_end_bit, job_ptr->job_id,
node_ptr->name);
gres_cpus = gres_cores;
if (gres_cpus != NO_VAL)
gres_cpus *= cpus_per_core;
if ((gres_cpus < job_ptr->details->ntasks_per_node) ||
((job_ptr->details->cpus_per_task > 1) &&
(gres_cpus < job_ptr->details->cpus_per_task)))
gres_cpus = 0;
if (gres_cpus < cpus)
cpus = gres_cpus;
if (cpus == 0)
bit_nclear(core_map, core_start_bit, core_end_bit);
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: _can_job_run_on_node: %u cpus on %s(%d), "
"mem %u/%u",
cpus, select_node_record[node_i].node_ptr->name,
node_usage[node_i].node_state,
node_usage[node_i].alloc_memory,
select_node_record[node_i].real_memory);
}
return cpus;
}
static int _breakup_blocks(List block_list, List new_blocks,
select_ba_request_t *request, List my_block_list,
bool only_free, bool only_small)
{
int rc = SLURM_ERROR;
bg_record_t *bg_record = NULL;
ListIterator itr = NULL, bit_itr = NULL;
int total_cnode_cnt=0;
char start_char[SYSTEM_DIMENSIONS+1];
bitstr_t *ionodes = bit_alloc(bg_conf->ionodes_per_mp);
int cnodes = request->procs / bg_conf->cpu_ratio;
int curr_mp_bit = -1;
int dim;
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK)
info("cpu_count=%d cnodes=%d o_free=%d o_small=%d",
request->procs, cnodes, only_free, only_small);
switch(cnodes) {
case 16:
/* a 16 can go anywhere */
break;
case 32:
bit_itr = list_iterator_create(bg_lists->valid_small32);
break;
case 64:
bit_itr = list_iterator_create(bg_lists->valid_small64);
break;
case 128:
bit_itr = list_iterator_create(bg_lists->valid_small128);
break;
case 256:
bit_itr = list_iterator_create(bg_lists->valid_small256);
break;
default:
error("We shouldn't be here with this size %d", cnodes);
goto finished;
break;
}
/* First try with free blocks a midplane or less. Then try with the
* smallest blocks.
*/
itr = list_iterator_create(block_list);
while ((bg_record = list_next(itr))) {
if (bg_record->free_cnt) {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK)
info("%s being freed by other job(s), skipping",
bg_record->bg_block_id);
continue;
}
/* never look at a block if a job is running */
if (bg_record->job_running != NO_JOB_RUNNING)
continue;
/* on the third time through look for just a block
* that isn't used */
/* check for free blocks on the first and second time */
if (only_free && (bg_record->state != BG_BLOCK_FREE))
continue;
/* check small blocks first */
if (only_small
&& (bg_record->cnode_cnt > bg_conf->mp_cnode_cnt))
continue;
if (request->avail_mp_bitmap &&
!bit_super_set(bg_record->bitmap,
request->avail_mp_bitmap)) {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK)
info("bg block %s has nodes not usable "
"by this job",
bg_record->bg_block_id);
continue;
}
if (bg_record->cnode_cnt == cnodes) {
ba_mp_t *ba_mp = NULL;
if (bg_record->ba_mp_list)
ba_mp = list_peek(bg_record->ba_mp_list);
if (!ba_mp) {
for (dim=0; dim<SYSTEM_DIMENSIONS; dim++)
start_char[dim] = alpha_num[
bg_record->start[dim]];
start_char[dim] = '\0';
request->save_name = xstrdup(start_char);
} else
request->save_name = xstrdup(ba_mp->coord_str);
rc = SLURM_SUCCESS;
goto finished;
}
/* lets see if we can combine some small ones */
if (bg_record->cnode_cnt < cnodes) {
char bitstring[BITSIZE];
bitstr_t *bitstr = NULL;
int num_over = 0;
int num_cnodes = bg_record->cnode_cnt;
int rec_mp_bit = bit_ffs(bg_record->bitmap);
if (curr_mp_bit != rec_mp_bit) {
/* Got a different node than
* previously, since the list should
* be in order of nodes for small blocks
* just clear here since the last node
* doesn't have any more. */
curr_mp_bit = rec_mp_bit;
bit_nclear(ionodes, 0,
(bg_conf->ionodes_per_mp-1));
total_cnode_cnt = 0;
}
/* On really busy systems we can get
overlapping blocks here. If that is the
case only add that which doesn't overlap.
*/
if ((num_over = bit_overlap(
ionodes, bg_record->ionode_bitmap))) {
/* Since the smallest block size is
the number of cnodes in an io node,
just multiply the num_over by that to
get the number of cnodes to remove.
*/
if ((num_cnodes -=
num_over * bg_conf->smallest_block) <= 0)
continue;
}
bit_or(ionodes, bg_record->ionode_bitmap);
/* check and see if the bits set are a valid
combo */
if (bit_itr) {
while ((bitstr = list_next(bit_itr))) {
if (bit_super_set(ionodes, bitstr))
break;
}
list_iterator_reset(bit_itr);
}
if (!bitstr) {
bit_nclear(ionodes, 0,
(bg_conf->ionodes_per_mp-1));
bit_or(ionodes, bg_record->ionode_bitmap);
total_cnode_cnt = num_cnodes =
bg_record->cnode_cnt;
} else
total_cnode_cnt += num_cnodes;
bit_fmt(bitstring, BITSIZE, ionodes);
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK)
info("combine adding %s %s %d got %d set "
"ionodes %s total is %s",
bg_record->bg_block_id, bg_record->mp_str,
num_cnodes, total_cnode_cnt,
bg_record->ionode_str, bitstring);
if (total_cnode_cnt == cnodes) {
ba_mp_t *ba_mp = NULL;
if (bg_record->ba_mp_list)
ba_mp = list_peek(
bg_record->ba_mp_list);
if (!ba_mp) {
for (dim=0; dim<SYSTEM_DIMENSIONS;
dim++)
start_char[dim] = alpha_num[
bg_record->start[dim]];
start_char[dim] = '\0';
request->save_name =
xstrdup(start_char);
} else
request->save_name =
xstrdup(ba_mp->coord_str);
if (!my_block_list) {
rc = SLURM_SUCCESS;
goto finished;
}
bg_record = create_small_record(bg_record,
ionodes,
cnodes);
list_append(new_blocks, bg_record);
rc = SLURM_SUCCESS;
goto finished;
}
continue;
}
/* we found a block that is bigger than requested */
break;
}
if (bg_record) {
ba_mp_t *ba_mp = NULL;
if (bg_record->ba_mp_list)
ba_mp = list_peek(bg_record->ba_mp_list);
if (!ba_mp) {
for (dim=0; dim<SYSTEM_DIMENSIONS; dim++)
start_char[dim] = alpha_num[
bg_record->start[dim]];
start_char[dim] = '\0';
request->save_name = xstrdup(start_char);
} else
request->save_name = xstrdup(ba_mp->coord_str);
/* It appears we don't need this original record
* anymore, just work off the copy if indeed it is a copy. */
/* bg_record_t *found_record = NULL; */
/* if (bg_record->original) { */
/* if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK) */
/* info("1 This was a copy %s", */
/* bg_record->bg_block_id); */
/* found_record = bg_record->original; */
/* } else { */
/* if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK) */
/* info("looking for original"); */
/* found_record = find_org_in_bg_list( */
/* bg_lists->main, bg_record); */
/* } */
if ((bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK)
&& bg_record->original
&& (bg_record->original->magic != BLOCK_MAGIC)) {
info("This record %s has bad magic, it must be "
"getting freed. No worries it will all be "
"figured out later.",
bg_record->bg_block_id);
}
/* if (!found_record || found_record->magic != BLOCK_MAGIC) { */
/* error("this record wasn't found in the list!"); */
/* rc = SLURM_ERROR; */
/* goto finished; */
/* } */
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK) {
char tmp_char[256];
format_node_name(bg_record, tmp_char,
sizeof(tmp_char));
info("going to split %s, %s",
bg_record->bg_block_id,
tmp_char);
}
if (!my_block_list) {
rc = SLURM_SUCCESS;
goto finished;
}
_split_block(block_list, new_blocks, bg_record, cnodes);
rc = SLURM_SUCCESS;
goto finished;
}
finished:
if (bit_itr)
list_iterator_destroy(bit_itr);
FREE_NULL_BITMAP(ionodes);
if (itr)
list_iterator_destroy(itr);
return rc;
}
extern List as_mysql_modify_qos(mysql_conn_t *mysql_conn, uint32_t uid,
slurmdb_qos_cond_t *qos_cond,
slurmdb_qos_rec_t *qos)
{
ListIterator itr = NULL;
List ret_list = NULL;
int rc = SLURM_SUCCESS;
char *object = NULL;
char *vals = NULL, *extra = NULL, *query = NULL, *name_char = NULL;
time_t now = time(NULL);
char *user_name = NULL;
int set = 0, i;
MYSQL_RES *result = NULL;
MYSQL_ROW row;
char *tmp_char1=NULL, *tmp_char2=NULL;
bitstr_t *preempt_bitstr = NULL;
char *added_preempt = NULL;
uint32_t qos_cnt;
assoc_mgr_lock_t locks = { NO_LOCK, NO_LOCK, READ_LOCK, NO_LOCK,
NO_LOCK, NO_LOCK, NO_LOCK };
if (!qos_cond || !qos) {
error("we need something to change");
return NULL;
}
if (check_connection(mysql_conn) != SLURM_SUCCESS)
return NULL;
if (!is_user_min_admin_level(mysql_conn, uid,
SLURMDB_ADMIN_SUPER_USER)) {
errno = ESLURM_ACCESS_DENIED;
return NULL;
}
xstrcat(extra, "where deleted=0");
if (qos_cond->description_list
&& list_count(qos_cond->description_list)) {
set = 0;
xstrcat(extra, " && (");
itr = list_iterator_create(qos_cond->description_list);
while ((object = list_next(itr))) {
if (set)
xstrcat(extra, " || ");
xstrfmtcat(extra, "description='%s'", object);
set = 1;
}
list_iterator_destroy(itr);
xstrcat(extra, ")");
}
if (qos_cond->id_list
&& list_count(qos_cond->id_list)) {
set = 0;
xstrcat(extra, " && (");
itr = list_iterator_create(qos_cond->id_list);
while ((object = list_next(itr))) {
if (set)
xstrcat(extra, " || ");
xstrfmtcat(extra, "id='%s'", object);
set = 1;
}
list_iterator_destroy(itr);
xstrcat(extra, ")");
}
if (qos_cond->name_list
&& list_count(qos_cond->name_list)) {
set = 0;
xstrcat(extra, " && (");
itr = list_iterator_create(qos_cond->name_list);
while ((object = list_next(itr))) {
if (set)
xstrcat(extra, " || ");
xstrfmtcat(extra, "name='%s'", object);
set = 1;
}
list_iterator_destroy(itr);
xstrcat(extra, ")");
}
_setup_qos_limits(qos, &tmp_char1, &tmp_char2,
&vals, &added_preempt, 0);
assoc_mgr_lock(&locks);
qos_cnt = g_qos_count;
assoc_mgr_unlock(&locks);
if (added_preempt) {
preempt_bitstr = bit_alloc(qos_cnt);
bit_unfmt(preempt_bitstr, added_preempt+1);
xfree(added_preempt);
}
xfree(tmp_char1);
xfree(tmp_char2);
if (!extra || !vals) {
errno = SLURM_NO_CHANGE_IN_DATA;
FREE_NULL_BITMAP(preempt_bitstr);
error("Nothing to change");
return NULL;
}
object = xstrdup(mqos_req_inx[0]);
for (i = 1; i < MQOS_COUNT; i++)
xstrfmtcat(object, ", %s", mqos_req_inx[i]);
query = xstrdup_printf("select %s from %s %s;",
object, qos_table, extra);
xfree(extra);
xfree(object);
if (!(result = mysql_db_query_ret(mysql_conn, query, 0))) {
xfree(query);
FREE_NULL_BITMAP(preempt_bitstr);
return NULL;
}
rc = 0;
ret_list = list_create(slurm_destroy_char);
while ((row = mysql_fetch_row(result))) {
slurmdb_qos_rec_t *qos_rec = NULL;
uint32_t id = slurm_atoul(row[MQOS_ID]);
if (preempt_bitstr) {
if (_preemption_loop(mysql_conn, id, preempt_bitstr))
break;
}
object = xstrdup(row[MQOS_NAME]);
list_append(ret_list, object);
if (!rc) {
xstrfmtcat(name_char, "(name='%s'", object);
rc = 1;
} else {
xstrfmtcat(name_char, " || name='%s'", object);
}
qos_rec = xmalloc(sizeof(slurmdb_qos_rec_t));
qos_rec->name = xstrdup(object);
qos_rec->id = id;
qos_rec->flags = qos->flags;
qos_rec->grace_time = qos->grace_time;
mod_tres_str(&qos_rec->grp_tres,
qos->grp_tres, row[MQOS_GT],
NULL, "grp_tres", &vals, qos_rec->id, 0);
mod_tres_str(&qos_rec->grp_tres_mins,
qos->grp_tres_mins, row[MQOS_GTM],
NULL, "grp_tres_mins", &vals, qos_rec->id, 0);
mod_tres_str(&qos_rec->grp_tres_run_mins,
qos->grp_tres_run_mins, row[MQOS_GTRM],
NULL, "grp_tres_run_mins", &vals,
qos_rec->id, 0);
qos_rec->grp_jobs = qos->grp_jobs;
qos_rec->grp_submit_jobs = qos->grp_submit_jobs;
qos_rec->grp_wall = qos->grp_wall;
mod_tres_str(&qos_rec->max_tres_pa,
qos->max_tres_pa, row[MQOS_MTPA],
NULL, "max_tres_pa", &vals, qos_rec->id, 0);
mod_tres_str(&qos_rec->max_tres_pj,
qos->max_tres_pj, row[MQOS_MTPJ],
NULL, "max_tres_pj", &vals, qos_rec->id, 0);
mod_tres_str(&qos_rec->max_tres_pn,
qos->max_tres_pn, row[MQOS_MTPN],
NULL, "max_tres_pn", &vals, qos_rec->id, 0);
mod_tres_str(&qos_rec->max_tres_pu,
qos->max_tres_pu, row[MQOS_MTPU],
NULL, "max_tres_pu", &vals, qos_rec->id, 0);
mod_tres_str(&qos_rec->max_tres_mins_pj,
qos->max_tres_mins_pj, row[MQOS_MTMPJ],
NULL, "max_tres_mins_pj", &vals, qos_rec->id, 0);
mod_tres_str(&qos_rec->max_tres_run_mins_pa,
qos->max_tres_run_mins_pa, row[MQOS_MTRM],
NULL, "max_tres_run_mins_pa", &vals,
qos_rec->id, 0);
mod_tres_str(&qos_rec->max_tres_run_mins_pu,
qos->max_tres_run_mins_pu, row[MQOS_MTRM],
NULL, "max_tres_run_mins_pu", &vals,
qos_rec->id, 0);
qos_rec->max_jobs_pa = qos->max_jobs_pa;
qos_rec->max_jobs_pu = qos->max_jobs_pu;
qos_rec->max_submit_jobs_pa = qos->max_submit_jobs_pa;
qos_rec->max_submit_jobs_pu = qos->max_submit_jobs_pu;
qos_rec->max_wall_pj = qos->max_wall_pj;
mod_tres_str(&qos_rec->min_tres_pj,
qos->min_tres_pj, row[MQOS_MITPJ],
NULL, "min_tres_pj", &vals, qos_rec->id, 0);
qos_rec->preempt_mode = qos->preempt_mode;
qos_rec->priority = qos->priority;
if (qos->preempt_list) {
ListIterator new_preempt_itr =
list_iterator_create(qos->preempt_list);
char *new_preempt = NULL;
bool cleared = 0;
qos_rec->preempt_bitstr = bit_alloc(qos_cnt);
if (row[MQOS_PREEMPT] && row[MQOS_PREEMPT][0])
bit_unfmt(qos_rec->preempt_bitstr,
row[MQOS_PREEMPT]+1);
while ((new_preempt = list_next(new_preempt_itr))) {
if (new_preempt[0] == '-') {
bit_clear(qos_rec->preempt_bitstr,
atol(new_preempt+1));
} else if (new_preempt[0] == '+') {
bit_set(qos_rec->preempt_bitstr,
atol(new_preempt+1));
} else {
if (!cleared) {
cleared = 1;
bit_nclear(
qos_rec->preempt_bitstr,
0,
qos_cnt-1);
}
bit_set(qos_rec->preempt_bitstr,
atol(new_preempt));
}
}
list_iterator_destroy(new_preempt_itr);
}
qos_rec->usage_factor = qos->usage_factor;
qos_rec->usage_thres = qos->usage_thres;
if (addto_update_list(mysql_conn->update_list,
SLURMDB_MODIFY_QOS, qos_rec)
!= SLURM_SUCCESS)
slurmdb_destroy_qos_rec(qos_rec);
}
mysql_free_result(result);
FREE_NULL_BITMAP(preempt_bitstr);
if (row) {
xfree(vals);
xfree(name_char);
xfree(query);
FREE_NULL_LIST(ret_list);
ret_list = NULL;
errno = ESLURM_QOS_PREEMPTION_LOOP;
return ret_list;
}
if (!list_count(ret_list)) {
errno = SLURM_NO_CHANGE_IN_DATA;
if (debug_flags & DEBUG_FLAG_DB_QOS)
DB_DEBUG(mysql_conn->conn,
"didn't effect anything\n%s", query);
xfree(vals);
xfree(query);
return ret_list;
}
xfree(query);
xstrcat(name_char, ")");
user_name = uid_to_string((uid_t) uid);
rc = modify_common(mysql_conn, DBD_MODIFY_QOS, now,
user_name, qos_table, name_char, vals, NULL);
xfree(user_name);
xfree(name_char);
xfree(vals);
if (rc == SLURM_ERROR) {
error("Couldn't modify qos");
FREE_NULL_LIST(ret_list);
ret_list = NULL;
}
return ret_list;
}
/* Execute control sequence. */
int
input_csi_dispatch(struct input_ctx *ictx)
{
struct screen_write_ctx *sctx = &ictx->ctx;
struct window_pane *wp = ictx->wp;
struct screen *s = sctx->s;
struct input_table_entry *entry;
int n, m;
if (ictx->flags & INPUT_DISCARD)
return (0);
if (input_split(ictx) != 0)
return (0);
log_debug("%s: '%c' \"%s\" \"%s\"",
__func__, ictx->ch, ictx->interm_buf, ictx->param_buf);
entry = bsearch(ictx, input_csi_table, nitems(input_csi_table),
sizeof input_csi_table[0], input_table_compare);
if (entry == NULL) {
log_debug("%s: unknown '%c'", __func__, ictx->ch);
return (0);
}
switch (entry->type) {
case INPUT_CSI_CBT:
/* Find the previous tab point, n times. */
n = input_get(ictx, 0, 1, 1);
while (s->cx > 0 && n-- > 0) {
do
s->cx--;
while (s->cx > 0 && !bit_test(s->tabs, s->cx));
}
break;
case INPUT_CSI_CUB:
screen_write_cursorleft(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_CUD:
screen_write_cursordown(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_CUF:
screen_write_cursorright(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_CUP:
n = input_get(ictx, 0, 1, 1);
m = input_get(ictx, 1, 1, 1);
screen_write_cursormove(sctx, m - 1, n - 1);
break;
case INPUT_CSI_CUU:
screen_write_cursorup(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_CNL:
screen_write_carriagereturn(sctx);
screen_write_cursordown(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_CPL:
screen_write_carriagereturn(sctx);
screen_write_cursorup(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_DA:
switch (input_get(ictx, 0, 0, 0)) {
case 0:
input_reply(ictx, "\033[?1;2c");
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_DA_TWO:
switch (input_get(ictx, 0, 0, 0)) {
case 0:
input_reply(ictx, "\033[>0;95;0c");
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_ECH:
screen_write_clearcharacter(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_DCH:
screen_write_deletecharacter(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_DECSTBM:
n = input_get(ictx, 0, 1, 1);
m = input_get(ictx, 1, 1, screen_size_y(s));
screen_write_scrollregion(sctx, n - 1, m - 1);
break;
case INPUT_CSI_DL:
screen_write_deleteline(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_DSR:
switch (input_get(ictx, 0, 0, 0)) {
case 5:
input_reply(ictx, "\033[0n");
break;
case 6:
input_reply(ictx, "\033[%u;%uR", s->cy + 1, s->cx + 1);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_ED:
switch (input_get(ictx, 0, 0, 0)) {
case 0:
screen_write_clearendofscreen(sctx);
break;
case 1:
screen_write_clearstartofscreen(sctx);
break;
case 2:
screen_write_clearscreen(sctx);
break;
case 3:
switch (input_get(ictx, 1, 0, 0)) {
case 0:
/*
* Linux console extension to clear history
* (for example before locking the screen).
*/
screen_write_clearhistory(sctx);
break;
}
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_EL:
switch (input_get(ictx, 0, 0, 0)) {
case 0:
screen_write_clearendofline(sctx);
break;
case 1:
screen_write_clearstartofline(sctx);
break;
case 2:
screen_write_clearline(sctx);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_HPA:
n = input_get(ictx, 0, 1, 1);
screen_write_cursormove(sctx, n - 1, s->cy);
break;
case INPUT_CSI_ICH:
screen_write_insertcharacter(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_IL:
screen_write_insertline(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_RCP:
memcpy(&ictx->cell, &ictx->old_cell, sizeof ictx->cell);
screen_write_cursormove(sctx, ictx->old_cx, ictx->old_cy);
break;
case INPUT_CSI_RM:
switch (input_get(ictx, 0, 0, -1)) {
case 4: /* IRM */
screen_write_mode_clear(&ictx->ctx, MODE_INSERT);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_RM_PRIVATE:
switch (input_get(ictx, 0, 0, -1)) {
case 1: /* GATM */
screen_write_mode_clear(&ictx->ctx, MODE_KCURSOR);
break;
case 3: /* DECCOLM */
screen_write_cursormove(&ictx->ctx, 0, 0);
screen_write_clearscreen(&ictx->ctx);
break;
case 7: /* DECAWM */
screen_write_mode_clear(&ictx->ctx, MODE_WRAP);
break;
case 25: /* TCEM */
screen_write_mode_clear(&ictx->ctx, MODE_CURSOR);
break;
case 1000:
case 1001:
case 1002:
case 1003:
screen_write_mode_clear(&ictx->ctx, ALL_MOUSE_MODES);
break;
case 1004:
wp->focus_notify &= ~PANE_FOCUS_NOTIFY;
break;
case 1005:
screen_write_mode_clear(&ictx->ctx, MODE_MOUSE_UTF8);
break;
case 1006:
screen_write_mode_clear(&ictx->ctx, MODE_MOUSE_SGR);
break;
case 47:
case 1047:
window_pane_alternate_off(wp, &ictx->cell, 0);
break;
case 1049:
window_pane_alternate_off(wp, &ictx->cell, 1);
break;
case 2004:
screen_write_mode_clear(&ictx->ctx, MODE_BRACKETPASTE);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_SCP:
memcpy(&ictx->old_cell, &ictx->cell, sizeof ictx->old_cell);
ictx->old_cx = s->cx;
ictx->old_cy = s->cy;
break;
case INPUT_CSI_SGR:
input_csi_dispatch_sgr(ictx);
break;
case INPUT_CSI_SM:
switch (input_get(ictx, 0, 0, -1)) {
case 4: /* IRM */
screen_write_mode_set(&ictx->ctx, MODE_INSERT);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_SM_PRIVATE:
switch (input_get(ictx, 0, 0, -1)) {
case 1: /* GATM */
screen_write_mode_set(&ictx->ctx, MODE_KCURSOR);
break;
case 3: /* DECCOLM */
screen_write_cursormove(&ictx->ctx, 0, 0);
screen_write_clearscreen(&ictx->ctx);
break;
case 7: /* DECAWM */
screen_write_mode_set(&ictx->ctx, MODE_WRAP);
break;
case 25: /* TCEM */
screen_write_mode_set(&ictx->ctx, MODE_CURSOR);
break;
case 1000:
screen_write_mode_clear(&ictx->ctx, ALL_MOUSE_MODES);
screen_write_mode_set(&ictx->ctx, MODE_MOUSE_STANDARD);
break;
case 1002:
screen_write_mode_clear(&ictx->ctx, ALL_MOUSE_MODES);
screen_write_mode_set(&ictx->ctx, MODE_MOUSE_BUTTON);
break;
case 1003:
screen_write_mode_clear(&ictx->ctx, ALL_MOUSE_MODES);
screen_write_mode_set(&ictx->ctx, MODE_MOUSE_ANY);
break;
case 1004:
wp->focus_notify |= PANE_FOCUS_NOTIFY;
break;
case 1005:
screen_write_mode_set(&ictx->ctx, MODE_MOUSE_UTF8);
break;
case 1006:
screen_write_mode_set(&ictx->ctx, MODE_MOUSE_SGR);
break;
case 47:
case 1047:
window_pane_alternate_on(wp, &ictx->cell, 0);
break;
case 1049:
window_pane_alternate_on(wp, &ictx->cell, 1);
break;
case 2004:
screen_write_mode_set(&ictx->ctx, MODE_BRACKETPASTE);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_TBC:
switch (input_get(ictx, 0, 0, 0)) {
case 0:
if (s->cx < screen_size_x(s))
bit_clear(s->tabs, s->cx);
break;
case 3:
bit_nclear(s->tabs, 0, screen_size_x(s) - 1);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_VPA:
n = input_get(ictx, 0, 1, 1);
screen_write_cursormove(sctx, s->cx, n - 1);
break;
case INPUT_CSI_DECSCUSR:
n = input_get(ictx, 0, 0, 0);
screen_set_cursor_style(s, n);
break;
}
return (0);
}
int
taskset_init( taskset_t *ts, int thread_count, size_t arg_size )
{
int init_step;
init_step = 0;
memset( ts, 0, sizeof( *ts ) );
ts->thread_count = thread_count;
ts->arg_size = arg_size;
ts->bitmap_elements = ( ts->thread_count + 31 ) / 32;
ts->task_threads = malloc( sizeof( hb_thread_t* ) * ts->thread_count );
if( ts->task_threads == NULL )
goto fail;
init_step++;
if( arg_size != 0 )
{
ts->task_threads_args = malloc( arg_size * ts->thread_count );
if( ts->task_threads == NULL )
goto fail;
}
init_step++;
ts->task_begin_bitmap = malloc( sizeof( uint32_t ) * ts->bitmap_elements );
if( ts->task_begin_bitmap == NULL )
goto fail;
init_step++;
ts->task_complete_bitmap = malloc( sizeof( uint32_t ) * ts->bitmap_elements );
if( ts->task_complete_bitmap == NULL )
goto fail;
init_step++;
ts->task_stop_bitmap = malloc( sizeof( uint32_t ) * ts->bitmap_elements );
if( ts->task_stop_bitmap == NULL )
goto fail;
init_step++;
ts->task_cond_lock = hb_lock_init();
if( ts->task_cond_lock == NULL)
goto fail;
init_step++;
ts->task_begin = hb_cond_init();
if( ts->task_begin == NULL)
goto fail;
init_step++;
ts->task_complete = hb_cond_init();
if( ts->task_complete == NULL)
goto fail;
init_step++;
/*
* Initialize all arg data to 0.
*/
memset(ts->task_threads_args, 0, ts->arg_size * ts->thread_count );
/*
* Inialize bitmaps to all bits set. This means that any unused bits
* in the bitmap are already in the "condition satisfied" state allowing
* us to test the bitmap 32bits at a time without having to mask off
* the end.
*/
memset(ts->task_begin_bitmap, 0xFF, sizeof( uint32_t ) * ts->bitmap_elements );
memset(ts->task_complete_bitmap, 0xFF, sizeof( uint32_t ) * ts->bitmap_elements );
memset(ts->task_stop_bitmap, 0, sizeof( uint32_t ) * ts->bitmap_elements );
/*
* Important to start off with the threads locked waiting
* on input, no work completed, and not asked to stop.
*/
bit_nclear( ts->task_begin_bitmap, 0, ts->thread_count - 1 );
bit_nclear( ts->task_complete_bitmap, 0, ts->thread_count - 1 );
bit_nclear( ts->task_stop_bitmap, 0, ts->thread_count - 1 );
return (1);
fail:
switch (init_step)
{
default:
hb_cond_close( &ts->task_complete );
/* FALL THROUGH */
case 7:
hb_cond_close( &ts->task_begin );
/* FALL THROUGH */
case 6:
hb_lock_close( &ts->task_cond_lock );
/* FALL THROUGH */
case 5:
free( ts->task_stop_bitmap );
/* FALL THROUGH */
case 4:
free( ts->task_complete_bitmap );
/* FALL THROUGH */
case 3:
free( ts->task_begin_bitmap );
/* FALL THROUGH */
case 2:
if( ts->task_threads_args == NULL )
free( ts->task_threads_args );
/* FALL THROUGH */
case 1:
free( ts->task_threads );
/* FALL THROUGH */
case 0:
break;
}
return (0);
}
/* Add the given job to the "active" structures of
* the given partition and increment the run count */
static void _add_job_to_active(struct job_record *job_ptr,
struct gs_part *p_ptr)
{
job_resources_t *job_res = job_ptr->job_resrcs;
uint16_t job_gr_type;
/* add job to active_resmap */
job_gr_type = _get_part_gr_type(job_ptr->part_ptr);
if ((job_gr_type == GS_CPU2) || (job_gr_type == GS_CORE) ||
(job_gr_type == GS_SOCKET)) {
if (p_ptr->jobs_active == 0 && p_ptr->active_resmap) {
uint32_t size = bit_size(p_ptr->active_resmap);
bit_nclear(p_ptr->active_resmap, 0, size-1);
}
add_job_to_cores(job_res, &(p_ptr->active_resmap),
gs_bits_per_node);
if (job_gr_type == GS_SOCKET)
_fill_sockets(job_res->node_bitmap, p_ptr);
} else { /* GS_NODE or GS_CPU */
if (!p_ptr->active_resmap) {
if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG) {
info("gang: _add_job_to_active: job %u first",
job_ptr->job_id);
}
p_ptr->active_resmap = bit_copy(job_res->node_bitmap);
} else if (p_ptr->jobs_active == 0) {
if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG) {
info("gang: _add_job_to_active: job %u copied",
job_ptr->job_id);
}
bit_copybits(p_ptr->active_resmap,
job_res->node_bitmap);
} else {
if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG) {
info("gang: _add_job_to_active: adding job %u",
job_ptr->job_id);
}
bit_or(p_ptr->active_resmap, job_res->node_bitmap);
}
}
/* add job to the active_cpus array */
if (job_gr_type == GS_CPU) {
uint32_t i, a, sz = bit_size(p_ptr->active_resmap);
if (!p_ptr->active_cpus) {
/* create active_cpus array */
p_ptr->active_cpus = xmalloc(sz * sizeof(uint16_t));
}
if (p_ptr->jobs_active == 0) {
/* overwrite the existing values in active_cpus */
for (a = 0, i = 0; i < sz; i++) {
if (bit_test(job_res->node_bitmap, i)) {
p_ptr->active_cpus[i] =
job_res->cpus[a++];
} else {
p_ptr->active_cpus[i] = 0;
}
}
} else {
/* add job to existing jobs in the active cpus */
for (a = 0, i = 0; i < sz; i++) {
if (bit_test(job_res->node_bitmap, i)) {
uint16_t limit = _get_phys_bit_cnt(i);
p_ptr->active_cpus[i] +=
job_res->cpus[a++];
/* when adding shadows, the resources
* may get overcommitted */
if (p_ptr->active_cpus[i] > limit)
p_ptr->active_cpus[i] = limit;
}
}
}
}
p_ptr->jobs_active += 1;
}
/*
* When we are "the server", this starts SET/ACK mode
* When we are "the client", this starts REQ/REPLY mode
*/
static int
cfg_initiator_send_ATTR(struct message *msg)
{
struct sa *isakmp_sa = msg->isakmp_sa;
struct ipsec_exch *ie = msg->exchange->data;
u_int8_t *hashp = 0, *attrp, *attr;
size_t attrlen, off;
char *id_string, *cfg_mode, *field;
struct sockaddr *sa;
#define CFG_ATTR_BIT_MAX ISAKMP_CFG_ATTR_FUTURE_MIN /* XXX */
bitstr_t bit_decl(attrbits, CFG_ATTR_BIT_MAX);
u_int16_t bit, length;
u_int32_t life;
if (msg->exchange->phase == 2) {
hashp = cfg_add_hash(msg);
if (!hashp)
return -1;
}
/* We initiated this exchange, check isakmp_sa for other side. */
if (isakmp_sa->initiator)
id_string = ipsec_id_string(isakmp_sa->id_r,
isakmp_sa->id_r_len);
else
id_string = ipsec_id_string(isakmp_sa->id_i,
isakmp_sa->id_i_len);
if (!id_string) {
log_print("cfg_initiator_send_ATTR: cannot parse ID");
goto fail;
}
/* Check for attribute list to send to the other side */
attrlen = 0;
bit_nclear(attrbits, 0, CFG_ATTR_BIT_MAX - 1);
cfg_mode = conf_get_str(id_string, "Mode");
if (!cfg_mode || strcmp(cfg_mode, "SET") == 0) {
/* SET/ACK mode */
ie->cfg_type = ISAKMP_CFG_SET;
LOG_DBG((LOG_NEGOTIATION, 10,
"cfg_initiator_send_ATTR: SET/ACK mode"));
#define ATTRFIND(STR,ATTR4,LEN4,ATTR6,LEN6) do \
{ \
if ((sa = conf_get_address (id_string, STR)) != NULL) \
switch (sa->sa_family) { \
case AF_INET: \
bit_set (attrbits, ATTR4); \
attrlen += ISAKMP_ATTR_SZ + LEN4; \
break; \
case AF_INET6: \
bit_set (attrbits, ATTR6); \
attrlen += ISAKMP_ATTR_SZ + LEN6; \
break; \
default: \
break; \
} \
free (sa); \
} while (0)
/*
* XXX We don't simultaneously support IPv4 and IPv6
* addresses.
*/
ATTRFIND("Address", ISAKMP_CFG_ATTR_INTERNAL_IP4_ADDRESS, 4,
ISAKMP_CFG_ATTR_INTERNAL_IP6_ADDRESS, 16);
ATTRFIND("Netmask", ISAKMP_CFG_ATTR_INTERNAL_IP4_NETMASK, 4,
ISAKMP_CFG_ATTR_INTERNAL_IP6_NETMASK, 16);
ATTRFIND("Nameserver", ISAKMP_CFG_ATTR_INTERNAL_IP4_DNS, 4,
ISAKMP_CFG_ATTR_INTERNAL_IP6_DNS, 16);
ATTRFIND("WINS-server", ISAKMP_CFG_ATTR_INTERNAL_IP4_NBNS, 4,
ISAKMP_CFG_ATTR_INTERNAL_IP6_NBNS, 16);
ATTRFIND("DHCP-server", ISAKMP_CFG_ATTR_INTERNAL_IP4_DHCP, 4,
ISAKMP_CFG_ATTR_INTERNAL_IP6_DHCP, 16);
#ifdef notyet
ATTRFIND("Network", ISAKMP_CFG_ATTR_INTERNAL_IP4_SUBNET, 8,
ISAKMP_CFG_ATTR_INTERNAL_IP6_SUBNET, 17);
#endif
#undef ATTRFIND
if (conf_get_str(id_string, "Lifetime")) {
bit_set(attrbits,
ISAKMP_CFG_ATTR_INTERNAL_ADDRESS_EXPIRY);
attrlen += ISAKMP_ATTR_SZ + 4;
}
} else {
struct conf_list *alist;
struct conf_list_node *anode;
ie->cfg_type = ISAKMP_CFG_REQUEST;
LOG_DBG((LOG_NEGOTIATION, 10,
"cfg_initiator_send_ATTR: REQ/REPLY mode"));
alist = conf_get_list(id_string, "Attributes");
if (alist) {
for (anode = TAILQ_FIRST(&alist->fields); anode;
anode = TAILQ_NEXT(anode, link)) {
if (strcasecmp(anode->field, "Address") == 0) {
bit_set(attrbits, ISAKMP_CFG_ATTR_INTERNAL_IP4_ADDRESS);
bit_set(attrbits, ISAKMP_CFG_ATTR_INTERNAL_IP6_ADDRESS);
attrlen += ISAKMP_ATTR_SZ * 2;
} else if (strcasecmp(anode->field, "Netmask")
== 0) {
bit_set(attrbits, ISAKMP_CFG_ATTR_INTERNAL_IP4_NETMASK);
bit_set(attrbits, ISAKMP_CFG_ATTR_INTERNAL_IP6_NETMASK);
attrlen += ISAKMP_ATTR_SZ * 2;
} else if (strcasecmp(anode->field,
"Nameserver") == 0) {
bit_set(attrbits, ISAKMP_CFG_ATTR_INTERNAL_IP4_DNS);
bit_set(attrbits, ISAKMP_CFG_ATTR_INTERNAL_IP6_DNS);
attrlen += ISAKMP_ATTR_SZ * 2;
} else if (strcasecmp(anode->field,
"WINS-server") == 0) {
bit_set(attrbits, ISAKMP_CFG_ATTR_INTERNAL_IP4_NBNS);
bit_set(attrbits, ISAKMP_CFG_ATTR_INTERNAL_IP6_NBNS);
attrlen += ISAKMP_ATTR_SZ * 2;
} else if (strcasecmp(anode->field,
"DHCP-server") == 0) {
bit_set(attrbits, ISAKMP_CFG_ATTR_INTERNAL_IP4_DHCP);
bit_set(attrbits, ISAKMP_CFG_ATTR_INTERNAL_IP6_DHCP);
attrlen += ISAKMP_ATTR_SZ * 2;
} else if (strcasecmp(anode->field,
"Lifetime") == 0) {
bit_set(attrbits, ISAKMP_CFG_ATTR_INTERNAL_ADDRESS_EXPIRY);
attrlen += ISAKMP_ATTR_SZ;
} else {
log_print("cfg_initiator_send_ATTR: "
"unknown attribute %.20s in "
"section [%s]", anode->field,
id_string);
}
}
conf_free_list(alist);
}
}
if (attrlen == 0) {
/* No data found. */
log_print("cfg_initiator_send_ATTR: no IKECFG attributes "
"found for [%s]", id_string);
/*
* We can continue, but this indicates a configuration error
* that the user probably will want to correct.
*/
free(id_string);
return 0;
}
attrlen += ISAKMP_ATTRIBUTE_SZ;
attrp = calloc(1, attrlen);
if (!attrp) {
log_error("cfg_initiator_send_ATTR: calloc (1, %lu) failed",
(unsigned long)attrlen);
goto fail;
}
if (message_add_payload(msg, ISAKMP_PAYLOAD_ATTRIBUTE, attrp, attrlen,
1)) {
free(attrp);
goto fail;
}
SET_ISAKMP_ATTRIBUTE_TYPE(attrp, ie->cfg_type);
getrandom((u_int8_t *) & ie->cfg_id, sizeof ie->cfg_id);
SET_ISAKMP_ATTRIBUTE_ID(attrp, ie->cfg_id);
off = ISAKMP_ATTRIBUTE_SZ;
/*
* Use the bitstring built previously to collect the right
* parameters for attrp.
*/
for (bit = 0; bit < CFG_ATTR_BIT_MAX; bit++)
if (bit_test(attrbits, bit)) {
attr = attrp + off;
SET_ISAKMP_ATTR_TYPE(attr, bit);
if (ie->cfg_type == ISAKMP_CFG_REQUEST) {
off += ISAKMP_ATTR_SZ;
continue;
}
/* All the other are similar, this is the odd one. */
if (bit == ISAKMP_CFG_ATTR_INTERNAL_ADDRESS_EXPIRY) {
life = conf_get_num(id_string, "Lifetime",
1200);
SET_ISAKMP_ATTR_LENGTH_VALUE(attr, 4);
encode_32(attr + ISAKMP_ATTR_VALUE_OFF, life);
off += ISAKMP_ATTR_SZ + 4;
continue;
}
switch (bit) {
case ISAKMP_CFG_ATTR_INTERNAL_IP4_ADDRESS:
case ISAKMP_CFG_ATTR_INTERNAL_IP4_NETMASK:
case ISAKMP_CFG_ATTR_INTERNAL_IP4_DNS:
case ISAKMP_CFG_ATTR_INTERNAL_IP4_DHCP:
case ISAKMP_CFG_ATTR_INTERNAL_IP4_NBNS:
length = 4;
break;
case ISAKMP_CFG_ATTR_INTERNAL_IP6_ADDRESS:
case ISAKMP_CFG_ATTR_INTERNAL_IP6_NETMASK:
case ISAKMP_CFG_ATTR_INTERNAL_IP6_DNS:
case ISAKMP_CFG_ATTR_INTERNAL_IP6_DHCP:
case ISAKMP_CFG_ATTR_INTERNAL_IP6_NBNS:
length = 16;
break;
default:
length = 0; /* Silence gcc. */
}
switch (bit) {
case ISAKMP_CFG_ATTR_INTERNAL_IP4_ADDRESS:
case ISAKMP_CFG_ATTR_INTERNAL_IP6_ADDRESS:
field = "Address";
break;
case ISAKMP_CFG_ATTR_INTERNAL_IP4_NETMASK:
case ISAKMP_CFG_ATTR_INTERNAL_IP6_NETMASK:
field = "Netmask";
break;
case ISAKMP_CFG_ATTR_INTERNAL_IP4_DNS:
case ISAKMP_CFG_ATTR_INTERNAL_IP6_DNS:
field = "Nameserver";
break;
case ISAKMP_CFG_ATTR_INTERNAL_IP4_DHCP:
case ISAKMP_CFG_ATTR_INTERNAL_IP6_DHCP:
field = "DHCP-server";
break;
case ISAKMP_CFG_ATTR_INTERNAL_IP4_NBNS:
case ISAKMP_CFG_ATTR_INTERNAL_IP6_NBNS:
field = "WINS-server";
break;
default:
field = 0; /* Silence gcc. */
}
sa = conf_get_address(id_string, field);
SET_ISAKMP_ATTR_LENGTH_VALUE(attr, length);
memcpy(attr + ISAKMP_ATTR_VALUE_OFF,
sockaddr_addrdata(sa), length);
free(sa);
off += ISAKMP_ATTR_SZ + length;
}
if (msg->exchange->phase == 2)
if (cfg_finalize_hash(msg, hashp, attrp, attrlen))
goto fail;
return 0;
fail:
free(id_string);
return -1;
}
/* sync up core bitmap with new CPU count using a best-fit approach
* on the available resources on each node
*
* "Best-fit" means:
* 1st priority: Use smallest number of boards with sufficient
* available CPUs
* 2nd priority: Use smallest number of sockets with sufficient
* available CPUs
* 3rd priority: Use board combination with the smallest number
* of available CPUs
* 4th priority: Use higher-numbered boards/sockets/cores first
*
* The CPU array contains the distribution of CPUs, which can include
* virtual CPUs (hyperthreads)
*/
static void _block_sync_core_bitmap(struct job_record *job_ptr,
const uint16_t cr_type)
{
uint32_t c, s, i, j, n, b, z, size, csize, core_cnt;
uint16_t cpus, num_bits, vpus = 1;
job_resources_t *job_res = job_ptr->job_resrcs;
bool alloc_cores = false, alloc_sockets = false;
uint16_t ntasks_per_core = 0xffff;
int count, cpu_min, b_min, elig, s_min, comb_idx, sock_idx;
int elig_idx, comb_brd_idx, sock_list_idx, comb_min, board_num;
int* boards_cpu_cnt;
int* sort_brds_cpu_cnt;
int* board_combs;
int* socket_list;
int* elig_brd_combs;
int* elig_cpu_cnt;
bool* sockets_used;
uint16_t boards_nb;
uint16_t nboards_nb;
uint16_t sockets_nb;
uint16_t ncores_nb;
uint16_t nsockets_nb;
uint16_t sock_per_brd;
uint16_t sock_per_comb;
uint16_t req_cpus,best_fit_cpus = 0;
uint32_t best_fit_location = 0;
uint64_t ncomb_brd;
bool sufficient, best_fit_sufficient;
if (!job_res)
return;
if (cr_type & CR_CORE)
alloc_cores = true;
if (slurmctld_conf.select_type_param & CR_ALLOCATE_FULL_SOCKET) {
if (cr_type & CR_SOCKET)
alloc_sockets = true;
} else {
if (cr_type & CR_SOCKET)
alloc_cores = true;
}
if (job_ptr->details && job_ptr->details->mc_ptr) {
multi_core_data_t *mc_ptr = job_ptr->details->mc_ptr;
if (mc_ptr->ntasks_per_core) {
ntasks_per_core = mc_ptr->ntasks_per_core;
}
if ((mc_ptr->threads_per_core != (uint16_t) NO_VAL) &&
(mc_ptr->threads_per_core < ntasks_per_core)) {
ntasks_per_core = mc_ptr->threads_per_core;
}
}
size = bit_size(job_res->node_bitmap);
csize = bit_size(job_res->core_bitmap);
sockets_nb = select_node_record[0].sockets;
sockets_cpu_cnt = xmalloc(sockets_nb * sizeof(int));
sockets_used = xmalloc(sockets_nb * sizeof(bool));
boards_nb = select_node_record[0].boards;
boards_cpu_cnt = xmalloc(boards_nb * sizeof(int));
sort_brds_cpu_cnt = xmalloc(boards_nb * sizeof(int));
for (c = 0, i = 0, n = 0; n < size; n++) {
if (bit_test(job_res->node_bitmap, n) == 0)
continue;
core_cnt = 0;
ncores_nb = select_node_record[n].cores;
nsockets_nb = select_node_record[n].sockets;
nboards_nb = select_node_record[n].boards;
num_bits = nsockets_nb * ncores_nb;
if ((c + num_bits) > csize)
fatal("cons_res: _block_sync_core_bitmap index error");
cpus = job_res->cpus[i];
vpus = MIN(select_node_record[n].vpus, ntasks_per_core);
/* compute still required cores on the node */
req_cpus = cpus / vpus;
if ( cpus % vpus )
req_cpus++;
if (nboards_nb > MAX_BOARDS) {
debug3("cons_res: node[%u]: exceeds max boards; "
"doing best-fit across sockets only", n);
nboards_nb = 1;
}
if ( nsockets_nb > sockets_nb) {
sockets_nb = nsockets_nb;
xrealloc(sockets_cpu_cnt, sockets_nb * sizeof(int));
xrealloc(sockets_used,sockets_nb * sizeof(bool));
}
if ( nboards_nb > boards_nb) {
boards_nb = nboards_nb;
xrealloc(boards_cpu_cnt, boards_nb * sizeof(int));
xrealloc(sort_brds_cpu_cnt, boards_nb * sizeof(int));
}
/* Count available cores on each socket and board */
sock_per_brd = nsockets_nb / nboards_nb;
for (b = 0; b < nboards_nb; b++) {
boards_cpu_cnt[b] = 0;
sort_brds_cpu_cnt[b] = 0;
}
for (s = 0; s < nsockets_nb; s++) {
sockets_cpu_cnt[s]=0;
sockets_used[s]=false;
b = s/sock_per_brd;
for ( j = c + (s * ncores_nb) ;
j < c + ((s+1) * ncores_nb) ;
j++ ) {
if ( bit_test(job_res->core_bitmap,j) ) {
sockets_cpu_cnt[s]++;
boards_cpu_cnt[b]++;
sort_brds_cpu_cnt[b]++;
}
}
}
/* Sort boards in descending order of available core count */
qsort(sort_brds_cpu_cnt, nboards_nb, sizeof (int),
_cmp_int_descend);
/* Determine minimum number of boards required for the
* allocation (b_min) */
count = 0;
for (b = 0; b < nboards_nb; b++) {
count+=sort_brds_cpu_cnt[b];
if (count >= req_cpus)
break;
}
b_min = b+1;
sock_per_comb = b_min * sock_per_brd;
/* Allocate space for list of board combinations */
ncomb_brd = comb_counts[nboards_nb-1][b_min-1];
board_combs = xmalloc(ncomb_brd * b_min * sizeof(int));
/* Generate all combinations of b_min boards on the node */
_gen_combs(board_combs, nboards_nb, b_min);
/* Determine which combinations have enough available cores
* for the allocation (eligible board combinations)
*/
elig_brd_combs = xmalloc(ncomb_brd * sizeof(int));
elig_cpu_cnt = xmalloc(ncomb_brd * sizeof(int));
elig = 0;
for (comb_idx = 0; comb_idx < ncomb_brd; comb_idx++) {
count = 0;
for (comb_brd_idx = 0; comb_brd_idx < b_min;
comb_brd_idx++) {
board_num = board_combs[(comb_idx * b_min)
+ comb_brd_idx];
count += boards_cpu_cnt[board_num];
}
if (count >= req_cpus) {
elig_brd_combs[elig] = comb_idx;
elig_cpu_cnt[elig] = count;
elig++;
}
}
/* Allocate space for list of sockets for each eligible board
* combination */
socket_list = xmalloc(elig * sock_per_comb * sizeof(int));
/* Generate sorted list of sockets for each eligible board
* combination, and find combination with minimum number
* of sockets and minimum number of cpus required for the
* allocation
*/
s_min = sock_per_comb;
comb_min = 0;
cpu_min = sock_per_comb * ncores_nb;
for (elig_idx = 0; elig_idx < elig; elig_idx++) {
comb_idx = elig_brd_combs[elig_idx];
for (comb_brd_idx = 0; comb_brd_idx < b_min;
comb_brd_idx++) {
board_num = board_combs[(comb_idx * b_min)
+ comb_brd_idx];
sock_list_idx = (elig_idx * sock_per_comb) +
(comb_brd_idx * sock_per_brd);
for (sock_idx = 0; sock_idx < sock_per_brd;
sock_idx++) {
socket_list[sock_list_idx + sock_idx]
= (board_num * sock_per_brd)
+ sock_idx;
}
}
/* Sort this socket list in descending order of
* available core count */
qsort(&socket_list[elig_idx*sock_per_comb],
sock_per_comb, sizeof (int), _cmp_sock);
/* Determine minimum number of sockets required for
* the allocation from this socket list */
count = 0;
for (b = 0; b < sock_per_comb; b++) {
sock_idx =
socket_list[(int)((elig_idx*sock_per_comb)+b)];
count+=sockets_cpu_cnt[sock_idx];
if (count >= req_cpus)
break;
}
b++;
/* Use board combination with minimum number
* of required sockets and minimum number of CPUs
*/
if ((b < s_min) ||
(b == s_min && elig_cpu_cnt[elig_idx]
<= cpu_min)) {
s_min = b;
comb_min = elig_idx;
cpu_min = elig_cpu_cnt[elig_idx];
}
}
debug3("cons_res: best_fit: node[%u]: required cpus: %u, "
"min req boards: %u,", n, cpus, b_min);
debug3("cons_res: best_fit: node[%u]: min req sockets: %u, "
"min avail cores: %u", n, s_min, cpu_min);
/* Re-sort socket list for best-fit board combination in
* ascending order of socket number */
qsort(&socket_list[comb_min * sock_per_comb], sock_per_comb,
sizeof (int), _cmp_int_ascend);
xfree(board_combs);
xfree(elig_brd_combs);
xfree(elig_cpu_cnt);
/* select cores from the sockets of the best-fit board
* combination using a best-fit approach */
while( cpus > 0 ) {
best_fit_cpus = 0;
best_fit_sufficient = false;
/* search for the best socket, */
/* starting from the last one to let more room */
/* in the first one for system usage */
for ( z = sock_per_comb-1; (int) z >= (int) 0; z-- ) {
s = socket_list[(comb_min*sock_per_comb)+z];
sufficient = sockets_cpu_cnt[s] >= req_cpus ;
if ( (best_fit_cpus == 0) ||
(sufficient && !best_fit_sufficient ) ||
(sufficient && (sockets_cpu_cnt[s] <
best_fit_cpus)) ||
(!sufficient && (sockets_cpu_cnt[s] >
best_fit_cpus)) ) {
best_fit_cpus = sockets_cpu_cnt[s];
best_fit_location = s;
best_fit_sufficient = sufficient;
}
}
/* check that we have found a usable socket */
if ( best_fit_cpus == 0 )
break;
j = best_fit_location;
if (sock_per_brd)
j /= sock_per_brd;
debug3("cons_res: best_fit: using node[%u]: "
"board[%u]: socket[%u]: %u cores available",
n, j, best_fit_location,
sockets_cpu_cnt[best_fit_location]);
/* select socket cores from last to first */
/* socket[0]:Core[0] would be the last one */
sockets_used[best_fit_location] = true;
for ( j = c + ((best_fit_location+1) * ncores_nb)
- 1 ;
(int) j >= (int) (c + (best_fit_location *
ncores_nb)) ;
j-- ) {
/*
* if no more cpus to select
* release remaining cores unless
* we are allocating whole sockets
*/
if (cpus == 0) {
if (alloc_sockets) {
bit_set(job_res->core_bitmap,j);
core_cnt++;
} else {
bit_clear(job_res->core_bitmap,j);
}
continue;
}
/*
* remove cores from socket count and
* cpus count using hyperthreading requirement
*/
if ( bit_test(job_res->core_bitmap,j) ) {
sockets_cpu_cnt[best_fit_location]--;
core_cnt++;
if (cpus < vpus)
cpus = 0;
else
cpus -= vpus;
} else if (alloc_sockets) {
/* If the core is not used, add it
* anyway if allocating whole sockets */
bit_set(job_res->core_bitmap, j);
core_cnt++;
}
}
/* loop again if more cpus required */
if ( cpus > 0 )
continue;
/* release remaining cores of the unused sockets */
for (s = 0; s < nsockets_nb; s++) {
if ( sockets_used[s] )
continue;
bit_nclear(job_res->core_bitmap,
c+(s*ncores_nb),
c+((s+1)*ncores_nb)-1);
}
}
xfree(socket_list);
if (cpus > 0) {
/* cpu count should NEVER be greater than the number
* of set bits in the core bitmap for a given node */
fatal("cons_res: cpus computation error");
}
/* adjust cpus count of the current node */
if ((alloc_cores || alloc_sockets) &&
(select_node_record[n].vpus >= 1)) {
job_res->cpus[i] = core_cnt *
select_node_record[n].vpus;
}
i++;
/* move c to the next node in core_bitmap */
c += num_bits;
}
xfree(boards_cpu_cnt);
xfree(sort_brds_cpu_cnt);
xfree(sockets_cpu_cnt);
xfree(sockets_used);
}
/* Execute control sequence. */
int
input_csi_dispatch(struct input_ctx *ictx)
{
struct screen_write_ctx *sctx = &ictx->ctx;
struct screen *s = sctx->s;
struct input_table_entry *entry;
int n, m;
if (ictx->flags & INPUT_DISCARD)
return (0);
if (input_split(ictx) != 0)
return (0);
log_debug("%s: '%c' \"%s\" \"%s\"",
__func__, ictx->ch, ictx->interm_buf, ictx->param_buf);
entry = bsearch(ictx, input_csi_table, nitems(input_csi_table),
sizeof input_csi_table[0], input_table_compare);
if (entry == NULL) {
log_debug("%s: unknown '%c'", __func__, ictx->ch);
return (0);
}
switch (entry->type) {
case INPUT_CSI_CBT:
/* Find the previous tab point, n times. */
n = input_get(ictx, 0, 1, 1);
while (s->cx > 0 && n-- > 0) {
do
s->cx--;
while (s->cx > 0 && !bit_test(s->tabs, s->cx));
}
break;
case INPUT_CSI_CUB:
screen_write_cursorleft(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_CUD:
screen_write_cursordown(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_CUF:
screen_write_cursorright(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_CUP:
n = input_get(ictx, 0, 1, 1);
m = input_get(ictx, 1, 1, 1);
screen_write_cursormove(sctx, m - 1, n - 1);
break;
case INPUT_CSI_CUU:
screen_write_cursorup(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_CNL:
screen_write_carriagereturn(sctx);
screen_write_cursordown(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_CPL:
screen_write_carriagereturn(sctx);
screen_write_cursorup(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_DA:
switch (input_get(ictx, 0, 0, 0)) {
case 0:
input_reply(ictx, "\033[?1;2c");
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_DA_TWO:
switch (input_get(ictx, 0, 0, 0)) {
case 0:
input_reply(ictx, "\033[>0;95;0c");
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_ECH:
screen_write_clearcharacter(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_DCH:
screen_write_deletecharacter(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_DECSTBM:
n = input_get(ictx, 0, 1, 1);
m = input_get(ictx, 1, 1, screen_size_y(s));
screen_write_scrollregion(sctx, n - 1, m - 1);
break;
case INPUT_CSI_DL:
screen_write_deleteline(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_DSR:
switch (input_get(ictx, 0, 0, 0)) {
case 5:
input_reply(ictx, "\033[0n");
break;
case 6:
input_reply(ictx, "\033[%u;%uR", s->cy + 1, s->cx + 1);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_ED:
switch (input_get(ictx, 0, 0, 0)) {
case 0:
screen_write_clearendofscreen(sctx);
break;
case 1:
screen_write_clearstartofscreen(sctx);
break;
case 2:
screen_write_clearscreen(sctx);
break;
case 3:
switch (input_get(ictx, 1, 0, 0)) {
case 0:
/*
* Linux console extension to clear history
* (for example before locking the screen).
*/
screen_write_clearhistory(sctx);
break;
}
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_EL:
switch (input_get(ictx, 0, 0, 0)) {
case 0:
screen_write_clearendofline(sctx);
break;
case 1:
screen_write_clearstartofline(sctx);
break;
case 2:
screen_write_clearline(sctx);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_HPA:
n = input_get(ictx, 0, 1, 1);
screen_write_cursormove(sctx, n - 1, s->cy);
break;
case INPUT_CSI_ICH:
screen_write_insertcharacter(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_IL:
screen_write_insertline(sctx, input_get(ictx, 0, 1, 1));
break;
case INPUT_CSI_RCP:
memcpy(&ictx->cell, &ictx->old_cell, sizeof ictx->cell);
screen_write_cursormove(sctx, ictx->old_cx, ictx->old_cy);
break;
case INPUT_CSI_RM:
input_csi_dispatch_rm(ictx);
break;
case INPUT_CSI_RM_PRIVATE:
input_csi_dispatch_rm_private(ictx);
break;
case INPUT_CSI_SCP:
memcpy(&ictx->old_cell, &ictx->cell, sizeof ictx->old_cell);
ictx->old_cx = s->cx;
ictx->old_cy = s->cy;
break;
case INPUT_CSI_SGR:
input_csi_dispatch_sgr(ictx);
break;
case INPUT_CSI_SM:
input_csi_dispatch_sm(ictx);
break;
case INPUT_CSI_SM_PRIVATE:
input_csi_dispatch_sm_private(ictx);
break;
case INPUT_CSI_TBC:
switch (input_get(ictx, 0, 0, 0)) {
case 0:
if (s->cx < screen_size_x(s))
bit_clear(s->tabs, s->cx);
break;
case 3:
bit_nclear(s->tabs, 0, screen_size_x(s) - 1);
break;
default:
log_debug("%s: unknown '%c'", __func__, ictx->ch);
break;
}
break;
case INPUT_CSI_VPA:
n = input_get(ictx, 0, 1, 1);
screen_write_cursormove(sctx, s->cx, n - 1);
break;
case INPUT_CSI_DECSCUSR:
n = input_get(ictx, 0, 0, 0);
screen_set_cursor_style(s, n);
break;
}
return (0);
}
extern int bg_free_block(bg_record_t *bg_record, bool wait, bool locked)
{
int rc = SLURM_SUCCESS;
int count = 0;
if (!bg_record) {
error("bg_free_block: there was no bg_record");
return SLURM_ERROR;
}
if (!locked)
slurm_mutex_lock(&block_state_mutex);
while (count < MAX_FREE_RETRIES) {
/* block was removed */
if (bg_record->magic != BLOCK_MAGIC) {
error("block was removed while freeing it here");
xassert(0);
if (!locked)
slurm_mutex_unlock(&block_state_mutex);
return SLURM_SUCCESS;
}
/* Reset these here so we don't try to reboot it
when the state goes to free.
*/
bg_record->boot_state = 0;
bg_record->boot_count = 0;
/* Here we don't need to check if the block is still
* in exsistance since this function can't be called on
* the same block twice. It may
* had already been removed at this point also.
*/
#ifdef HAVE_BG_FILES
if (bg_record->state != BG_BLOCK_FREE
&& bg_record->state != BG_BLOCK_TERM) {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("bridge_destroy %s",
bg_record->bg_block_id);
rc = bridge_block_free(bg_record);
if (rc != SLURM_SUCCESS) {
if (rc == BG_ERROR_BLOCK_NOT_FOUND) {
debug("block %s is not found",
bg_record->bg_block_id);
bg_record->state = BG_BLOCK_FREE;
break;
} else if (rc == BG_ERROR_FREE) {
if (bg_conf->slurm_debug_flags
& DEBUG_FLAG_SELECT_TYPE)
info("bridge_block_free"
"(%s): %s State = %s",
bg_record->bg_block_id,
bg_err_str(rc),
bg_block_state_string(
bg_record->state));
} else if (rc == BG_ERROR_INVALID_STATE) {
#ifndef HAVE_BGL
/* If the state is error and
we get an incompatible
state back here, it means
we set it ourselves so
break out.
*/
if (bg_record->state
& BG_BLOCK_ERROR_FLAG)
break;
#endif
if (bg_conf->slurm_debug_flags
& DEBUG_FLAG_SELECT_TYPE)
info("bridge_block_free"
"(%s): %s State = %s",
bg_record->bg_block_id,
bg_err_str(rc),
bg_block_state_string(
bg_record->state));
#ifdef HAVE_BGQ
if (bg_record->state != BG_BLOCK_FREE
&& bg_record->state
!= BG_BLOCK_TERM)
bg_record->state = BG_BLOCK_TERM;
#endif
} else {
error("bridge_block_free"
"(%s): %s State = %s",
bg_record->bg_block_id,
bg_err_str(rc),
bg_block_state_string(
bg_record->state));
}
}
}
#else
/* Fake a free since we are n deallocating
state before this.
*/
if (bg_record->state & BG_BLOCK_ERROR_FLAG) {
/* This will set the state to ERROR(Free)
* just incase the state was ERROR(SOMETHING ELSE) */
bg_record->state = BG_BLOCK_ERROR_FLAG;
break;
} else if (!wait || (count >= 3))
bg_record->state = BG_BLOCK_FREE;
else if (bg_record->state != BG_BLOCK_FREE)
bg_record->state = BG_BLOCK_TERM;
#endif
if (!wait || (bg_record->state == BG_BLOCK_FREE)
#ifndef HAVE_BGL
|| (bg_record->state & BG_BLOCK_ERROR_FLAG)
#endif
) {
break;
}
/* If we were locked outside of this we need to unlock
to not cause deadlock on this mutex until we are
done.
*/
slurm_mutex_unlock(&block_state_mutex);
sleep(FREE_SLEEP_INTERVAL);
count++;
slurm_mutex_lock(&block_state_mutex);
}
rc = SLURM_SUCCESS;
if ((bg_record->state == BG_BLOCK_FREE)
|| (bg_record->state & BG_BLOCK_ERROR_FLAG)) {
if (bg_record->err_ratio
&& (bg_record->state == BG_BLOCK_FREE)) {
/* Sometime the realtime server can report
software error on cnodes even though the
block is free. If this is the case we need
to manually clear them.
*/
ba_mp_t *found_ba_mp;
ListIterator itr =
list_iterator_create(bg_record->ba_mp_list);
debug("Block %s is free, but has %u cnodes in error. "
"This can happen if a large block goes into "
"error and then is freed and the state of "
"the block changes before the "
"database informs all the cnodes are back to "
"normal. This is no big deal.",
bg_record->bg_block_id, bg_record->cnode_err_cnt);
while ((found_ba_mp = list_next(itr))) {
if (!found_ba_mp->used)
continue;
if (!found_ba_mp->cnode_err_bitmap)
found_ba_mp->cnode_err_bitmap =
bit_alloc(
bg_conf->mp_cnode_cnt);
bit_nclear(found_ba_mp->cnode_err_bitmap, 0,
bit_size(found_ba_mp->
cnode_err_bitmap)-1);
}
list_iterator_destroy(itr);
bg_record->cnode_err_cnt = 0;
bg_record->err_ratio = 0;
}
remove_from_bg_list(bg_lists->booted, bg_record);
} else if (count >= MAX_FREE_RETRIES) {
/* Something isn't right, go mark this one in an error
state. */
update_block_msg_t block_msg;
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("bg_free_block: block %s is not in state "
"free (%s), putting it in error state.",
bg_record->bg_block_id,
bg_block_state_string(bg_record->state));
slurm_init_update_block_msg(&block_msg);
block_msg.bg_block_id = bg_record->bg_block_id;
block_msg.state = BG_BLOCK_ERROR_FLAG;
block_msg.reason = "Block would not deallocate";
slurm_mutex_unlock(&block_state_mutex);
select_g_update_block(&block_msg);
slurm_mutex_lock(&block_state_mutex);
rc = SLURM_ERROR;
}
if (!locked)
slurm_mutex_unlock(&block_state_mutex);
return rc;
}
/* Given a string of comma separated values of the form:
* n
* n-m
* n-m/s
* build a representative bit string for the specified values.
*/
static int decode_elem(char *str, int nbits, const char *const abbrs[], bitstr_t *bits, int zerov) {
char *p;
int sval, eval, step;
int i;
int base;
if (str == NULL)
return 0;
if (zerov < 0)
base = 1;
else
base = 0;
bit_nclear(bits, base, nbits-1);
for (;str != NULL && *str != '\0'; str=p) {
/* See if we're comma separated and if so grab the first */
p = strchr(str, ',');
if (p != NULL)
*p++ = '\0';
step = 1;
if (*str == '*') { /* Special case: from start to end */
sval = base;
eval = nbits-1;
str++;
} else {
/* Find the initial value */
if (isdigit(*str)) sval = strtol(str, &str, 10);
else if (abbrs != NULL) sval = loc_string(&str, abbrs);
else return 0;
if (sval == zerov) sval = 0;
if (sval < base || sval >= nbits)
return 0;
eval = sval;
/* Check for a range */
if (*str == '-') {
str++;
if (isdigit(*str)) eval = strtol(str, &str, 10);
else if (abbrs != NULL) eval = loc_string(&str, abbrs);
else return 0;
if (eval == zerov) eval = 0;
if (eval < base || eval >= nbits)
return 0;
if (eval < sval) {
i = sval;
sval = eval;
eval = i;
}
}
}
/* Do we have an increment here? */
if (*str == '/') {
str++;
if (!isdigit(*str))
return 0;
step = atoi(str);
if (step <= 0 || step > (eval - sval))
return 0;
}
/* Set the corresponding bits */
for (i=sval; i<=eval; i += step)
bit_set(bits, i);
}
return 1;
}