int schd_bits2mask(char *string, Bitfield *mask) { int i, bit, len; Bitfield new_bit; BITFIELD_CLRALL(&new_bit); len = (int)strlen(string); /* Start with the low bit. */ bit = 0; for (i = 0; i < len; i++) { if (string[len - i - 1] == '1') BITFIELD_SETB(&new_bit, bit); else if (string[len - i - 1] != '0') return (-1); ++ bit; } /* Done. Copy the new nodemask and return success. */ BITFIELD_CPY(mask, &new_bit); return (0); }
/* * mask_mask is the mask of bits that are in this particular part of the * nodemask. ndmask is the actual nodemask in question. * * For instance, a mask_mask of: 00000000111111111111111100000000 * and a nodemask of : 00000000000000111111111100000000 * generate the output: ........------XXXXXXXXXX........ * * Note that the storage is static - copy the return before the next call! */ char * schd_format_nodemask(Bitfield *mask_mask, Bitfield *ndmask) { static char prt_bits[BITFIELD_SIZE + 2]; Bitfield bit; int idx; /* Find the most significant bit in the nodemask. */ BITFIELD_CLRALL(&bit); BITFIELD_SETB(&bit, (BITFIELD_SIZE - 1)); /* Walk the bits in the nodemask, from MSB to LSB. */ for (idx = 0; idx < BITFIELD_SIZE; idx ++) { if (BITFIELD_TSTM(&bit, mask_mask)) prt_bits[idx] = (BITFIELD_TSTM(ndmask, &bit)) ? 'X' : '-'; else prt_bits[idx] = '.'; BITFIELD_SHIFTR(&bit); } /* Null terminate the resulting string. */ prt_bits[idx] = '\0'; return (prt_bits); }
/** * @brief * availnodes() - Fill the supplied bitfield with a set bit for each node on which sufficient * resources (c.f. mapnodes.h) are configured. This is the set of nodes that * are physically available to be allocated -- policy, reserved nodes, etc, * may reduce the total number of nodes that are usable by jobs. * * NOTE: Must be called after mapnodes(). * * @param[out] maskp - pointer to Bitfield structure * * @return int * @retval 0 success * @retval !0 failure * */ int availnodes(Bitfield *maskp) { int node, i, count; Bitfield avail; /* This function needs a valid node map in order to find available nodes. */ if (nodemap == NULL) return 1; /* * If minimum resource values are not set, default them to reasonable * defaults. */ if (minnodemem < 0) minnodemem = minmem; if (minnodecpus < 0) minnodecpus = mincpus; if (memreserved < 0) memreserved = 0; (void)sprintf(log_buffer, "Minimum node resources: %d cpus, %d MB, %d MB rsvd", minnodecpus, minnodemem, memreserved); log_err(-1, __func__, log_buffer); BITFIELD_CLRALL(&avail); /* * Walk through the node map, checking for sufficient resources on each * node, and setting the appropriate bit on the mask for that node if it * is sufficiently endowed. See mapnodes.h for definition of "sufficient". */ for (node = 0; node <= maxnodeid; node++) { /* Enough CPUs? If not, skip it. */ for (count = i = 0; i < MAX_CPUS_PER_NODE; i++) if (nodemap[node].cpu[i] >= 0) count ++; /* Enough memory and cpus? If not, skip it. */ if (nodemap[node].memory < minnodemem || count < minnodecpus) { (void)sprintf(log_buffer, "node %d has only %luMB and %d cpus - cannot use", node, nodemap[node].memory, count); log_err(-1, __func__, log_buffer); continue; } /* Node has sufficient resources. Count this node as available. */ BITFIELD_SETB(&avail, node); } /* Copy the available mask to the passed-in storage, and return success. */ BITFIELD_CPY(maskp, &avail); return 0; }
int schd_alloc_nodes(int nnodes, Queue *queue, Bitfield *job_mask) { Bitfield *queue_mask; Bitfield *q_avail_mask; Bitfield new_bit; int candidate; int n_found; n_found = 0; BITFIELD_CLRALL(&new_bit); queue_mask = &(queue->queuemask); q_avail_mask = &(queue->availmask); candidate = (BITFIELD_SIZE - 1); while ((n_found < nnodes) && (candidate >= 0)) { if (BITFIELD_TSTB(queue_mask, candidate)) { if (BITFIELD_TSTB(q_avail_mask, candidate)) { /* available */ BITFIELD_SETB(&new_bit, candidate); ++n_found; } } -- candidate; } /* * If enough nodes were allocated, copy the resultant new nodemask * into the supplied area. */ if (nnodes == n_found) { BITFIELD_CPY(job_mask, &new_bit); return nnodes; } /* Didn't get enough nodes. Return 0 as an error. */ return 0; }
static int find_nodemasks(Queue *queue, Resources *rsrcs) { Job *job; Bitfield jobs_using; BITFIELD_CLRALL(&jobs_using); /* * Compute the set of nodes that are both physically available and also * assigned to this queue. */ BITFIELD_CPY(&queue->availmask, &(queue->queuemask)); BITFIELD_ANDM(&queue->availmask, &(queue->rsrcs->availmask)); /* * Compute the set of nodes in use by jobs running on the queue (if * there are any) and remove those nodes from the available node mask. */ if (queue->running) { for (job = queue->jobs; job != NULL; job = job->next) { if (job->state == 'R') BITFIELD_SETM(&jobs_using, &(job->nodemask)); } } /* * Remove the used node bits from the queue's availmask, and add them to * the resources' nodes used bits. */ BITFIELD_CLRM(&queue->availmask, &jobs_using); BITFIELD_SETM(&rsrcs->nodes_used, &jobs_using); return (0); }
/* * schd_get_queue_limits - query queue information from the server. * * Returns 0 on success, -1 for "fatal errors", and 1 for a transient * error (i.e., the queue failed the sanity checks imposed by the * queue_sanity() function). */ int schd_get_queue_limits(Queue *queue) { char *id = "schd_get_queue_limits"; int moved = 0, istrue; Batch_Status *bs; AttrList *attr; static AttrList alist[] = { {&alist[1], ATTR_start, "", ""}, {&alist[2], ATTR_enable, "", ""}, {&alist[3], ATTR_count, "", ""}, {&alist[4], ATTR_maxuserrun, "", ""}, {&alist[5], ATTR_rescavail, "", ""}, {&alist[6], ATTR_rescassn, "", ""}, {&alist[7], ATTR_rescdflt, "", ""}, {&alist[8], ATTR_rescmax, "", ""}, {&alist[9], ATTR_rescmin, "", ""}, {&alist[10], ATTR_acluren, "", ""}, {&alist[11], ATTR_acluser, "", ""}, {NULL, ATTR_maxrun, "", ""} }; size_t mem_default = UNSPECIFIED; size_t mem_assn = UNSPECIFIED; size_t mem_max = UNSPECIFIED; size_t mem_min = UNSPECIFIED; int cpu_default = UNSPECIFIED; int cpu_assn = UNSPECIFIED; int cpu_max = UNSPECIFIED; int cpu_min = UNSPECIFIED; int nodes_from_cpu, nodes_from_mem; queue->running = UNSPECIFIED; queue->queued = UNSPECIFIED; queue->maxrun = UNSPECIFIED; queue->userrun = UNSPECIFIED; queue->nodes_max = UNSPECIFIED; queue->nodes_min = UNSPECIFIED; queue->nodes_default = UNSPECIFIED; queue->nodes_assn = UNSPECIFIED; queue->nodes_rsvd = UNSPECIFIED; queue->wallt_max = UNSPECIFIED; queue->wallt_min = UNSPECIFIED; queue->wallt_default = UNSPECIFIED; queue->flags = 0; #ifdef NODEMASK BITFIELD_CLRALL(&queue->queuemask); BITFIELD_CLRALL(&queue->availmask); #endif /* NODEMASK */ queue->rsrcs = NULL; if (queue->jobs) { DBPRT(("%s: found jobs on queue '%s'! Freeing them...\n", id, queue->qname)); schd_free_jobs(queue->jobs); } if (queue->useracl) { DBPRT(("%s: found user ACL list on queue '%s'! Freeing it...\n", id, queue->qname)); schd_free_useracl(queue->useracl); } queue->jobs = NULL; queue->useracl = NULL; /* Ask the server for information about the specified queue. */ if ((bs = pbs_statque(connector, queue->qname, alist, NULL)) == NULL) { sprintf(log_buffer, "pbs_statque failed, \"%s\" %d", queue->qname, pbs_errno); log_record(PBSEVENT_ERROR, PBS_EVENTCLASS_SERVER, id, log_buffer); DBPRT(("%s: %s\n", id, log_buffer)); return (-1); } /* Process the list of attributes returned by the server. */ for (attr = bs->attribs; attr != NULL; attr = attr->next) { /* Is queue started? */ if (!strcmp(attr->name, ATTR_start)) { if (schd_val2bool(attr->value, &istrue) == 0) { if (istrue) /* if true, queue is not stopped. */ queue->flags &= ~QFLAGS_STOPPED; else queue->flags |= QFLAGS_STOPPED; } else { DBPRT(("%s: couldn't parse attr %s value %s to boolean\n", id, attr->name, attr->value)); } continue; } /* Is queue enabled? */ if (!strcmp(attr->name, ATTR_enable)) { if (schd_val2bool(attr->value, &istrue) == 0) { if (istrue) /* if true, queue is not disabled. */ queue->flags &= ~QFLAGS_DISABLED; else queue->flags |= QFLAGS_DISABLED; } else { DBPRT(("%s: couldn't parse attr %s value %s to boolean\n", id, attr->name, attr->value)); } continue; } /* How many jobs are queued and running? */ if (!strcmp(attr->name, ATTR_count)) { queue->queued = schd_how_many(attr->value, SC_QUEUED); queue->running = schd_how_many(attr->value, SC_RUNNING); continue; } /* Queue-wide maximum number of jobs running. */ if (!strcmp(attr->name, ATTR_maxrun)) { queue->maxrun = atoi(attr->value); continue; } /* Per-user maximum number of jobs running. */ if (!strcmp(attr->name, ATTR_maxuserrun)) { queue->userrun = atoi(attr->value); continue; } /* Is there an enabled user access control list on this queue? */ if (!strcmp(attr->name, ATTR_acluren)) { if (schd_val2bool(attr->value, &istrue) == 0) { if (istrue) /* if true, queue has an ACL */ queue->flags |= QFLAGS_USER_ACL; else queue->flags &= ~QFLAGS_USER_ACL; } else { DBPRT(("%s: couldn't parse attr %s value %s to boolean\n", id, attr->name, attr->value)); } continue; } if (!strcmp(attr->name, ATTR_acluser)) { if (queue->useracl) { DBPRT(("queue %s acluser already set!\n", queue->qname)); schd_free_useracl(queue->useracl); } queue->useracl = schd_create_useracl(attr->value); continue; } /* Queue maximum resource usage. */ if (!strcmp(attr->name, ATTR_rescmax)) { if (!strcmp("mem", attr->resource)) { mem_max = schd_val2byte(attr->value); continue; } if (!strcmp("ncpus", attr->resource)) { cpu_max = atoi(attr->value); continue; } if (!strcmp("walltime", attr->resource)) { queue->wallt_max = schd_val2sec(attr->value); continue; } #ifdef NODEMASK if (!strcmp("nodemask", attr->resource)) { if (schd_str2mask(attr->value, &queue->queuemask)) { (void)sprintf(log_buffer, "couldn't convert nodemask %s", attr->value); log_record(PBSEVENT_SYSTEM, PBS_EVENTCLASS_SERVER, id, log_buffer); } else queue->flags |= QFLAGS_NODEMASK; /* Valid nodemask. */ } #endif /* NODEMASK */ continue; } /* Queue minimum resource usage. */ if (!strcmp(attr->name, ATTR_rescmin)) { if (!strcmp("mem", attr->resource)) { mem_min = schd_val2byte(attr->value); continue; } if (!strcmp("ncpus", attr->resource)) { cpu_min = atoi(attr->value); continue; } if (!strcmp("walltime", attr->resource)) { queue->wallt_min = schd_val2sec(attr->value); continue; } continue; } /* Queue assigned (in use) resource usage. */ if (!strcmp(attr->name, ATTR_rescassn)) { if (!strcmp("mem", attr->resource)) { mem_assn = schd_val2byte(attr->value); continue; } if (!strcmp("ncpus", attr->resource)) { cpu_assn = atoi(attr->value); } continue; } if (!strcmp(attr->name, ATTR_rescdflt)) { if (!strcmp("mem", attr->resource)) { mem_default = schd_val2byte(attr->value); continue; } if (!strcmp("ncpus", attr->resource)) { cpu_default = atoi(attr->value); continue; } if (!strcmp("walltime", attr->resource)) queue->wallt_default = schd_val2sec(attr->value); } /* Ignore anything else */ } pbs_statfree(bs); /* * Calculate values for queue node limits, given memory and cpu values. * Note any discrepancies. */ nodes_from_cpu = NODES_FROM_CPU(cpu_default); nodes_from_mem = NODES_FROM_MEM(mem_default); if (nodes_from_cpu != nodes_from_mem) { sprintf(log_buffer, "%s: Queue '%s' default cpu/mem (%d/%s) convert to %d != %d nodes", id, queue->qname, cpu_default, schd_byte2val(mem_default), nodes_from_cpu, nodes_from_mem); log_record(PBSEVENT_ERROR, PBS_EVENTCLASS_SERVER, id, log_buffer); DBPRT(("%s: %s\n", id, log_buffer)); } nodes_from_cpu = NODES_FROM_CPU(cpu_max); nodes_from_mem = NODES_FROM_MEM(mem_max); if (nodes_from_cpu != nodes_from_mem) { sprintf(log_buffer, "%s: Queue '%s' maximum cpu/mem (%d/%s) convert to %d != %d nodes", id, queue->qname, cpu_max, schd_byte2val(mem_max), nodes_from_cpu, nodes_from_mem); log_record(PBSEVENT_ERROR, PBS_EVENTCLASS_SERVER, id, log_buffer); DBPRT(("%s: %s\n", id, log_buffer)); } nodes_from_cpu = NODES_FROM_CPU(cpu_min); nodes_from_mem = NODES_FROM_MEM(mem_min); if (nodes_from_cpu != nodes_from_mem) { sprintf(log_buffer, "%s: Queue '%s' minimum cpu/mem (%d/%s) convert to %d != %d nodes", id, queue->qname, cpu_min, schd_byte2val(mem_min), nodes_from_cpu, nodes_from_mem); log_record(PBSEVENT_ERROR, PBS_EVENTCLASS_SERVER, id, log_buffer); DBPRT(("%s: %s\n", id, log_buffer)); } /* * Note: The assigned cpus and memory need not be exactly the same * node equivalency. */ if ((cpu_default != UNSPECIFIED) && (mem_default != UNSPECIFIED)) queue->nodes_default = NODES_REQD(cpu_default, mem_default); if ((cpu_max != UNSPECIFIED) && (mem_max != UNSPECIFIED)) queue->nodes_max = NODES_REQD(cpu_max, mem_max); if ((cpu_min != UNSPECIFIED) && (mem_min != UNSPECIFIED)) queue->nodes_min = NODES_REQD(cpu_min, mem_min); if ((cpu_assn != UNSPECIFIED) && (mem_assn != UNSPECIFIED)) queue->nodes_assn = NODES_REQD(cpu_assn, mem_assn); /* * Move any jobs on this queue from the global list onto the queue's * list. Keep track of when the longest-running job will end, and set * the 'empty_by' field to that value. Maintain the ordering as it was * in "schd_AllJobs". */ if (schd_AllJobs) moved = queue_claim_jobs(queue, &schd_AllJobs); if (moved < 0) { sprintf(log_buffer, "%s: WARNING! Queue '%s' failed to claim jobs.", id, queue->qname); log_record(PBSEVENT_ERROR, PBS_EVENTCLASS_SERVER, id, log_buffer); DBPRT(("%s: %s\n", id, log_buffer)); } if (queue->nodes_assn == UNSPECIFIED) queue->nodes_assn = 0; if (queue->running == UNSPECIFIED) queue->running = 0; /* * Find out if the queue is idle, and if it was not before, set the idle * time to now. If there are running jobs, the queue is not idle at the * start of this iteration - set idle_since to 0. */ if (queue->running) { queue->idle_since = 0; } else { if (queue->idle_since == 0) queue->idle_since = schd_TimeNow; } /* * Get the resources for this queue from the resource monitor (if * available). If the resmom is not accessible, disable the queue. * If the resources were received okay, compute the available node * masks from the resources and jobs. * Don't bother with resources for the special or submit queues. */ if ((strcmp(queue->qname, schd_SubmitQueue->queue->qname) != 0) || ((schd_SpecialQueue != NULL) && (!strcmp(queue->qname, schd_SpecialQueue->queue->qname)))) { queue->rsrcs = schd_get_resources(queue->exechost); if (queue->rsrcs != NULL) { /* Account for this queue's resources. */ queue->rsrcs->nodes_alloc += queue->nodes_assn; queue->rsrcs->njobs += queue->running; /* * If the HPM counters do not appear to be in use on this host, * check for jobs on the queue that are using hpm. If so, set * the 'HPM_IN_USE' flag on the resources. This will prevent the * HPM counters from being released to global mode at the end * of the scheduling run (c.f. cleanup.c). * The 'HPM_IN_USE' flag will also be asserted if a job is run * that uses the HPM counters. */ if (schd_MANAGE_HPM && !(queue->rsrcs->flags & RSRCS_FLAGS_HPM_IN_USE)) { if (schd_hpm_job_count(queue->jobs)) queue->rsrcs->flags |= RSRCS_FLAGS_HPM_IN_USE; } #ifdef NODEMASK /* And find the nodemasks for the queue and resources. */ find_nodemasks(queue, queue->rsrcs); #endif /* NODEMASK */ } else { (void)sprintf(log_buffer, "Can't get resources for %s@%s - marking unavailable.", queue->qname, queue->exechost); log_record(PBSEVENT_SYSTEM, PBS_EVENTCLASS_SERVER, id, log_buffer); DBPRT(("%s: %s\n", id, log_buffer)); queue->flags |= QFLAGS_DISABLED; } } #ifdef DEBUG schd_dump_queue(queue, QUEUE_DUMP_JOBS); #endif /* DEBUG */ /* * It would probably be better to wait for the world to stabilize * than to try to impose some artificial order upon it. Do not do * the sanity check if the queue is stopped. */ if ((queue->flags & QFLAGS_STOPPED) == 0) { if (!queue_sanity(queue)) { sprintf(log_buffer, "WARNING! Queue '%s' failed sanity checks.", queue->qname); log_record(PBSEVENT_ERROR, PBS_EVENTCLASS_SERVER, id, log_buffer); DBPRT(("%s: %s\n", id, log_buffer)); return (1); } } return (0); }
int schd_str2mask(char *maskstr, Bitfield *maskp) { char *id = "str2mask"; Bitfield nodemask; char hex; int ndbit = 0, len, dec, hxbit; unsigned long long compat; char *ptr; char buf[32]; if (maskstr == NULL) return (-1); /* Nodemask string must be in the format '0x...'. */ if ((maskstr[0] != '0') || (tolower(maskstr[1]) != 'x')) { /* XXX * For backwards compatability, if the string will convert * into an unsigned long long, then assume this is an old- * style decimal nodemask (on 8-128P machines, nodemask is * an unsigned long long, and was treated as such). * * Note that this assumes that sizeof(unsigned long long) * is the same as sizeof(unsigned long), which is only the * case with the 64-bit SGI compiler ABI. */ compat = strtoul(maskstr, &ptr, 10); if (*ptr != '\0') return (-1); /* * String converted to an unsigned long long. Print it as * a hex back into a buffer, and point at that buffer instead. * This will cause the newly created hex string to be used. */ (void)sprintf(buf, "0x%llx", compat); maskstr = buf; (void)sprintf(log_buffer, "converted old-style nodemask %lu to %s", compat, maskstr); log_record(PBSEVENT_SYSTEM, PBS_EVENTCLASS_SERVER, id, log_buffer); } /* Zero out the new nodemask struct, and fill it from the maskstr. */ BITFIELD_CLRALL(&nodemask); /* Walk backwards through the hex digits in the string 0x... */ for (len = (int)strlen(maskstr); len > 2 /* 0x... */; len --) { hex = maskstr[len - 1]; /* Make sure it's a hex digit. */ if (!isxdigit((int)hex)) return (-1); /* If this digit is a '0', it contributes nothing. */ if (hex != '0') { /* Convert the hex digit to a decimal int. */ dec = isdigit((int)hex) ? (hex - '0') : (tolower((int)hex) - 'a' + 10); for (hxbit = 0; hxbit < 4; hxbit ++) { if (dec & (1 << hxbit)) BITFIELD_SETB(&nodemask, ndbit); ndbit ++; } } else ndbit += 4; /* Make sure we haven't walked past the end of the nodemask. */ if (ndbit > (sizeof(Bitfield) * 8)) return (-1); } /* Copy the newly created nodemask into the handed-in one. */ BITFIELD_CPY(maskp, &nodemask); return (0); }
int schd_alloc_nodes(int request, Queue *queue, Bitfield *maskp) { char *id = "schd_alloc_nodes"; Bitfield avail; Bitfield mask; Bitfield contig; int remain; int qmsb; int qlsb; int i, n; int count; int found; /* Make certain the nodecount request can be fulfilled. */ if (request <= 0 || request > BITFIELD_NUM_ONES(&(queue->availmask))) return 0; /* * Make a copy of the queue's available bit mask to play with, and clear * the allocated nodes mask. */ BITFIELD_CPY(&avail, &(queue->availmask)); BITFIELD_CLRALL(&mask); /* How many have been found, and how many remain. */ found = 0; remain = request; while (remain > 0) { /* * Find first and last available bit positions in the * queue's available node mask. */ qmsb = BITFIELD_MS_ONE(&avail); qlsb = BITFIELD_LS_ONE(&avail); /* * Starting with the size of the remaining nodes needed to satisfy * this request, look for a set of 'n' contiguous bits in the * available node mask. If that is not found, try the next smallest * contiguous vector, etc. */ for (n = remain; n > 0; n--) { /* * Create a contiguous bitmask of 'n' bits, starting at the * position of the highest bit in the avail mask. */ BITFIELD_CLRALL(&contig); for (i = 0; i < n; i++) BITFIELD_SETB(&contig, qmsb - i); /* * Calculate how many times this contiguous bitmask needs to be * shifted to the right to cover every set of 'n' bits between * the qmsb and qlsb, inclusive. Count the initial configuration * as well (the trailing '+ 1'). */ count = (qmsb + 1 - qlsb) - n + 1; /* * Shift the contiguous mask right one bit at a time, checking * if all the bits in the mask are set in the available mask. */ for (i = 0; i < count; i++) { /* Are all bits in contig also set in the avail mask? */ if (BITFIELD_TSTALLM(&avail, &contig)) { break; } BITFIELD_SHIFTR(&contig); } /* * If the contiguous bits are available, add them to the new job * nodemask, and remove them from the avail mask. Adjust the * remaining node count, and start the next hunt for the remaining * nodes. */ if (i < count) { BITFIELD_SETM(&mask, &contig); BITFIELD_CLRM(&avail, &contig); found += n; remain -= n; break; /* for(n) loop */ } } /* Check for something going wrong. */ if (n == 0) { DBPRT(("%s: couldn't find any contiguous bits (even one!)\n", id)); break; /* while(remain) loop */ } } /* * If no bits remain to be allocated, copy the new mask into the provided * space, and return the number of bits requested. */ if (!remain && (found == request)) { BITFIELD_CPY(maskp, &mask); DBPRT(("%s: mask %s\n", id, schd_format_nodemask(&queue->queuemask, maskp))); return found; } return 0; }