void chunk_queue_insert(chunk_queue_ptr cq, chunk_ptr item) {
check_err(pthread_mutex_lock(&cq->mutex), "chunk_queue_insert");
if (cq->length == 0) {
/* Going from empty to nonempty */
cq->elements = calloc_or_fail(CQ_ALLOC, sizeof(chunk_ptr),
"chunk_queue_insert");
cq->alloc_length = CQ_ALLOC;
} else if (cq->length == cq->alloc_length) {
/* Must expand queue. Reset so that head is at 0 */
cq->alloc_length *= 2;
chunk_ptr *nelements = calloc_or_fail(cq->alloc_length, sizeof(chunk_ptr),
"chunk_queue_insert");
size_t i;
size_t n = cq->length;
for (i = 0; i < n; i++) {
nelements[i] = cq->elements[cq->head++];
if (cq->head >= cq->length) {
cq->head = 0;
}
}
free_array(cq->elements, cq->alloc_length, sizeof(chunk_ptr));
cq->elements = nelements;
cq->tail = n-1;
}
cq->tail++;
if (cq->tail >= cq->length) {
cq->tail = 0;
}
cq->elements[cq->tail] = item;
cq->length++;
check_err(pthread_cond_signal(&cq->cvar), "chunk_queue_insert");
check_err(pthread_mutex_unlock(&cq->mutex), "chunk_queue_insert");
}
/* Convert a set of literals into a CUDD cube */
static DdNode *cudd_lit_cube(shadow_mgr mgr, set_ptr lits) {
size_t nele = lits->nelements;
DdNode **vars = calloc_or_fail(nele, sizeof(DdNode *), "cudd_lit_cube");
int *phase = calloc_or_fail(nele, sizeof(int), "cudd_lit_cube");
word_t wr;
int i = 0;
set_iterstart(lits);
while (set_iternext(lits, &wr)) {
ref_t r = (ref_t) wr;
int pos = 1;
if (REF_GET_NEG(r)) {
r = REF_NEGATE(r);
pos = 0;
}
DdNode *n = get_ddnode(mgr, r);
vars[i] = n;
phase[i] = pos;
i++;
}
DdNode *cube = Cudd_bddComputeCube(mgr->bdd_manager, vars, phase, nele);
reference_dd(mgr, cube);
free_array(vars, nele, sizeof(DdNode *));
free_array(phase, nele, sizeof(int));
return cube;
}
/* Accumulate worker messages with statistics */
static void add_stat_message(chunk_ptr msg) {
size_t *stat_summary = NULL;
stat_messages[stat_message_cnt++] = msg;
/* See if we've accumulated a full set */
if (stat_message_cnt >= worker_cnt) {
size_t nstat = stat_messages[0]->length - 1;
if (flush_requestor_fd >= 0)
stat_summary = calloc_or_fail(nstat * 3, sizeof(size_t),
"add_stat_message");
/* Accumulate and print */
#if RPT >= 1
report(1, "Worker statistics:");
#endif
size_t i, w;
for (i = 0; i < nstat; i++) {
chunk_ptr msg = stat_messages[0];
word_t minval, maxval, sumval;
minval = maxval = sumval = chunk_get_word(msg, i+1);
for (w = 1; w < worker_cnt; w++) {
chunk_ptr msg = stat_messages[w];
word_t val = chunk_get_word(msg, i+1);
if (val < minval)
minval = val;
if (val > maxval)
maxval = val;
sumval += val;
}
if (stat_summary) {
stat_summary[3*i + 0] = minval;
stat_summary[3*i + 1] = maxval;
stat_summary[3*i + 2] = sumval;
}
#if RPT >= 1
report(1,
"Parameter %d\tMin: %" PRIu64 "\tMax: %" PRIu64 "\tAvg: %.2f\tSum: %" PRIu64,
(int) i, minval, maxval, (double) sumval/worker_cnt, sumval);
#endif
}
if (flush_requestor_fd >= 0) {
chunk_ptr msg = msg_new_stat(worker_cnt, nstat*3, stat_summary);
if (chunk_write(flush_requestor_fd, msg)) {
#if RPT >= 5
report(5, "Sent statistical summary to client at fd %d",
flush_requestor_fd);
#endif
} else {
err(false, "Failed to send statistical summary to client at fd %d",
flush_requestor_fd);
}
chunk_free(msg);
free_array(stat_summary, nstat*3, sizeof(size_t));
}
for (w = 0; w < worker_cnt; w++) {
chunk_free(stat_messages[w]);
stat_messages[w] = NULL;
}
stat_message_cnt = 0;
flush_requestor_fd = -1;
}
}
static void init_controller(int port, int nrouters, int nworkers) {
if (!new_server(port, &listen_fd, NULL))
err(true, "Cannot set up server on port %u", port);
#if RPT >= 2
report(2, "Listening socket has descriptor %d", listen_fd);
#endif
router_addr_set = word_set_new();
new_conn_map = word_keyvalue_new();
router_fd_set = word_set_new();
worker_fd_set = word_set_new();
client_fd_set = word_set_new();
init_cmd();
add_cmd("status", do_controller_status_cmd,
" | Determine status of connected nodes");
add_cmd("flush", do_controller_flush_cmd,
" | Flush state of all agents");
add_cmd("collect", do_controller_collect_cmd,
" | Initiate garbage collection");
add_quit_helper(quit_controller);
need_routers = nrouters;
need_workers = nworkers;
worker_cnt = nworkers;
stat_message_cnt = 0;
flush_requestor_fd = -1;
stat_messages = calloc_or_fail(worker_cnt, sizeof(chunk_ptr),
"init_controller");
gc_state = GC_READY;
need_worker_cnt = 0;
need_client_fd_set = NULL;
defer_client_fd_set = NULL;
gc_generation = 0;
}
static keyvalue_table_ptr cudd_shift(shadow_mgr mgr, set_ptr roots,
keyvalue_table_ptr vmap) {
DdNode **vector = calloc_or_fail(mgr->nvars, sizeof(DdNode *), "cudd_shift");
size_t i;
/* Must build up vector of composition functions */
for (i = 0; i < mgr->nvars; i++) {
ref_t vref = shadow_get_variable(mgr, i);
word_t wv;
ref_t nvref = vref;
if (keyvalue_find(vmap, (word_t) vref, &wv))
nvref = (ref_t) wv;
DdNode *nv = get_ddnode(mgr, nvref);
vector[i] = nv;
}
keyvalue_table_ptr stable = word_keyvalue_new();
word_t w;
set_iterstart(roots);
while (set_iternext(roots, &w)) {
ref_t r = (ref_t) w;
DdNode *n = get_ddnode(mgr, r);
DdNode *ns = Cudd_bddVectorCompose(mgr->bdd_manager, n, vector);
reference_dd(mgr, ns);
keyvalue_insert(stable, w, (word_t) ns);
}
free_array(vector, mgr->nvars, sizeof(DdNode *));
return stable;
}
/* Parse a single command line and return the name/value
* name & value are both allocated inside the function
*/
int parse_env_line(
memmgr **mm, /* memory manager */
char *one_var,
char **name,
char **value)
{
char *tmp_char = NULL;
int pos_eq = 0;
int tmp_pos = 0;
int len_name = 0;
int len_value = 0;
int len_total = 0;
len_total = strlen(one_var);
tmp_char = strchr(one_var, '=');
/* check to make sure we got an '=' */
if (tmp_char == NULL)
return(PBSE_BAD_PARAMETER);
pos_eq = tmp_char - one_var;
len_name = pos_eq;
tmp_pos = pos_eq + 1;
calloc_or_fail(mm, name, len_name+1, "Allocating name for hash_map");
/* Remove preceeding spaces in an env var
* This will NOT affect a env var that is all spaces
*/
while (one_var[tmp_pos] == ' ')
{
tmp_pos++;
}
if (tmp_pos == len_total)
{
/* The value consists of ALL spaces, reset to = pos + 1 */
tmp_pos = pos_eq + 1;
}
len_value = len_total - tmp_pos;
if (len_value == 0)
{
*value = NULL;
}
else
{
/* Env values can contain nested commas which must be
* escaped to be preserved. */
copy_env_value(mm, one_var + tmp_pos, value);
strncpy(*name, one_var, len_name);
}
return(PBSE_NONE);
} /* END parse_env_line() */
/* Copies env variable into an allocated string
* and escapes nested characters if necessary.
*/
int copy_env_value(
memmgr **mm, /* memory manager */
char *cur_val,
char **value)
{
char *tmpPtr;
int escape_count = 0, len_value = 0;
char escape_chars[] = {'"', '\'', '\\', '\n', ',', '\0'};
len_value = strlen(cur_val);
tmpPtr = cur_val;
while (*tmpPtr)
{
int escape_index = 0;
for (; escape_chars[escape_index] != '\0'; escape_index++)
{
if (*tmpPtr == escape_chars[escape_index])
{
escape_count++;
break;
}
}
tmpPtr++;
}
calloc_or_fail(mm, value, len_value + escape_count + 1, "Allocating value for hash_map");
if (escape_count > 0)
{
char *tmpVal = *value;
tmpPtr = cur_val;
while(*tmpPtr)
{
int escape_index = 0;
for (; escape_chars[escape_index] != '\0'; escape_index++)
{
if (*tmpPtr == escape_chars[escape_index])
{
*tmpVal++ = '\\';
break;
}
}
*tmpVal++ = *tmpPtr++;
}
}
else
strncpy(*value, cur_val, len_value);
return (PBSE_NONE);
}
/* Use CUDD to compute number of BDD nodes to represent set of functions */
size_t cudd_size(shadow_mgr mgr, set_ptr roots) {
if (!mgr->do_cudd)
return 0;
size_t nele = roots->nelements;
DdNode **croots = calloc_or_fail(nele, sizeof(DdNode *), "cudd_size");
word_t wr;
int i = 0;
set_iterstart(roots);
while (set_iternext(roots, &wr)) {
ref_t r = (ref_t) wr;
DdNode *n = get_ddnode(mgr, r);
croots[i++] = n;
}
int cnt = Cudd_SharingSize(croots, nele);
free_array(croots, nele, sizeof(DdNode *));
return (size_t) cnt;
}
/* Uses CUDD to determine refs for all variable in support set of root functions */
static set_ptr cudd_support(shadow_mgr mgr, set_ptr roots) {
/* Create vector of nodes */
DdNode **vector = calloc_or_fail(roots->nelements, sizeof(DdNode *),
"cudd_support");
size_t i = 0;
word_t w;
set_iterstart(roots);
while (set_iternext(roots, &w)) {
ref_t r = (ref_t) w;
vector[i++] = get_ddnode(mgr, r);
}
int *indices = NULL;
int cnt = Cudd_VectorSupportIndices(mgr->bdd_manager, vector,
roots->nelements, &indices);
set_ptr sset = word_set_new();
for (i = 0; i < cnt; i++) {
ref_t rv = shadow_get_variable(mgr, indices[i]);
set_insert(sset, (word_t) rv);
}
if (indices)
free(indices);
free_array(vector, roots->nelements, sizeof(DdNode *));
return sset;
}
chunk_ptr chunk_read(int fd, bool* eofp)
{
if (fd > maxfd)
{
// on first call, we zero the buffer set
if (maxfd == 0)
{
FD_ZERO(&buf_set);
FD_ZERO(&in_set);
}
maxfd = fd;
}
buf_node* curr_node = NULL;
buf_node* temp_node = NULL;
//create new head
if (buf_list_head == NULL) {
buf_list_head = calloc_or_fail(sizeof(buf_node), 1,
"chunk_read create head");
buf_list_head->fd = fd;
#if RPT >= 5
report(5, "created a node for fd %d as head\n", fd);
#endif
buf_list_head->length = 0;
buf_list_head->location = 0;
buf_list_head->buf = calloc_or_fail(CHUNK_MAX_SIZE, 2,
"chunk_read create head buf");
curr_node = buf_list_head;
}
// search for the fd in the buffer list, if it exists
else {
temp_node = buf_list_head;
while (temp_node != NULL && curr_node == NULL) {
if (fd == temp_node->fd) {
curr_node = temp_node;
#if RPT >= 5
report(5, "found node for fd %d\n", fd);
#endif
}
temp_node = temp_node->next;
}
}
// if it doesn't exist, create the new fd buffer at the head of the list
if (curr_node == NULL) {
curr_node = calloc_or_fail(sizeof(buf_node), 1, "chunk_read create node");
curr_node->fd = fd;
curr_node->length = 0;
curr_node->location = 0;
curr_node->next = buf_list_head;
curr_node->buf = calloc_or_fail(CHUNK_MAX_SIZE, 2,
"chunk_read create head buf");
#if RPT >= 5
report(5, "created a node for fd %d at head\n", fd);
#endif
buf_list_head = curr_node;
}
// if we can copy to the beginning, then we copy to the beginning
// (if the read point is past the beginning, and if the end of
// the buffered data is past the midway point of the buffer)
if (curr_node->length + curr_node->location >= CHUNK_MAX_SIZE
&& curr_node->location > 0)
{
memmove(curr_node->buf, (char *)(curr_node->buf + curr_node->location),
curr_node->length);
curr_node->location = 0;
}
// read if possible - if there is space, if the inset contains it, and if we
// want to use buffering (otherwise we don't want random buffer refills)
if (((curr_node->length + curr_node->location) < CHUNK_MAX_SIZE)
&& bufferReadBool && !(!(FD_ISSET(fd, &in_set))) )
{
#if RPT >= 5
report(5, "reading for %d\n", curr_node->fd);
#endif
ssize_t n = read(curr_node->fd,
curr_node->buf + curr_node->location + curr_node->length,
CHUNK_MAX_SIZE);
curr_node->length += n;
}
#if RPT >= 5
report(5, "about to get header for %d\n", fd);
#endif
// get header of chunk
size_t need_cnt = sizeof(chunk_t);
unsigned char buf[CHUNK_MAX_SIZE];
unsigned char* buf_ptr = (unsigned char*)buf;
ssize_t n = buf_read(curr_node, eofp, buf_ptr, need_cnt);
//ssize_t n = read(curr_node->fd, buf, need_cnt);
if (n <= 0)
{
return NULL;
}
#if RPT >= 5
report(5, "about to get rest of chunk for fd %d\n", fd);
#endif
// get rest of chunk
chunk_ptr creadp = (chunk_ptr) buf_ptr;
size_t len = creadp->length;
#if RPT >= 5
report(5, "len needed: %d", len);
#endif
if (len > 1) {
need_cnt = WORD_BYTES * (len - 1);
#if RPT >= 5
report(5, "head buf pointer at %p", buf_ptr);
#endif
buf_ptr = (unsigned char *)(buf_ptr + n);
#if RPT >= 5
report(5, "moved pointer to %p for rest", buf_ptr);
#endif
ssize_t n = buf_read(curr_node, eofp, buf_ptr, need_cnt);
//ssize_t n = read(curr_node->fd, buf_ptr, need_cnt);
if (n < 0) {
chunk_error("Failed read", NULL);
if (eofp)
*eofp = false;
return NULL;
}
}
#if RPT >= 5
report(5, "exiting chunk_read_buffered_builtin!\n");
#endif
if (eofp)
*eofp = false;
return chunk_clone(creadp);
}
void init_agent(bool iscli, char *controller_name, unsigned controller_port,
bool try_self_route, bool try_local_router) {
operator_table = word_keyvalue_new();
deferred_operand_table = word_keyvalue_new();
size_t i;
for (i = 0; i < NSTATA; i++)
agent_stat_counter[i] = 0;
chunk_ptr msg;
bool eof;
isclient = iscli;
self_route = try_self_route;
controller_fd = open_clientfd(controller_name, controller_port);
if (controller_fd < 0)
err(true,
"Cannot create connection to controller at %s:%u",
controller_name, controller_port);
else {
#if RPT >= 2
report(2, "Connection to controller has descriptor %d", controller_fd);
#endif
}
msg = isclient ? msg_new_register_client() : msg_new_register_worker();
bool sok = chunk_write(controller_fd, msg);
#if RPT >= 3
report(3, "Sent %s registration to controller",
isclient ? "client" : "worker");
#endif
chunk_free(msg);
if (!sok)
err(true, "Could not send registration message to controller");
/* Get acknowledgement from controller */
bool first = true;
/* Anticipated number of routers
(Will be updated when get first message from controller) */
nrouters = 1;
chunk_ptr amsg = NULL;
unsigned ridx = 0;;
while (ridx < nrouters) {
msg = chunk_read_unbuffered(controller_fd, &eof);
if (eof) {
err(true,
"Unexpected EOF from controller while getting router map");
}
word_t h = chunk_get_word(msg, 0);
unsigned code = msg_get_header_code(h);
switch (code) {
case MSG_ACK_AGENT:
if (first) {
own_agent = msg_get_header_agent(h);
amsg = msg_new_register_agent(own_agent);
nworkers = msg_get_header_workercount(h);
nrouters = msg_get_header_wordcount(h);
router_fd_array = calloc_or_fail(nrouters, sizeof(int),
"init_agent");
#if RPT >= 3
report(3,
"Ack from controller. Agent Id %u. %d workers. %d routers.",
own_agent, nworkers, nrouters);
#endif
first = false;
}
int i;
for (i = 1; i < msg->length; i++) {
word_t h = chunk_get_word(msg, i);
int fd;
unsigned ip = msg_get_header_ip(h);
unsigned port = msg_get_header_port(h);
#if RPT >= 4
report(4, "Attempting to add router %u with ip 0x%x, port %d",
ridx, ip, port);
#endif
fd = open_clientfd_ip(ip, port);
if (fd < 0) {
err(true, "Couldn't add router with ip 0x%x, port %d",
ip, port);
} else {
router_fd_array[ridx++] = fd;
#if RPT >= 3
report(3, "Added router %u with ip 0x%x, port %d, fd %d",
ridx, ip, port, fd);
#endif
if (!chunk_write(fd, amsg)) {
err(true,
"Couldn't send agent registration message to router with ip 0x%x, port %u",
ip, port);
}
if (try_local_router && local_router_fd == -1 &&
match_self_ip(ip))
{
local_router_fd = fd;
#if RPT >= 5
report(5,
"Router with fd %d designated as local router and prioritized for sending packets",
fd);
#endif
}
}
}
chunk_free(msg);
break;
case MSG_NACK:
err(true, "Connection request refused.");
break;
default:
err(false,
"Unexpected message code %u while getting router information", code);
chunk_free(msg);
break;
}
}
chunk_free(amsg);
#if RPT >= 2
report(2, "All %d routers connected", nrouters);
#endif
if (isclient) {
add_quit_helper(quit_agent);
add_cmd("kill", do_agent_kill,
" | Shutdown system");
add_cmd("flush", do_agent_flush,
" | Flush system");
add_cmd("collect", do_agent_gc,
" | Initiate garbage collection");
} else {
/* Worker must notify controller that it's ready */
chunk_ptr rmsg = msg_new_worker_ready(own_agent);
if (chunk_write(controller_fd, rmsg)) {
#if RPT >= 3
report(3, "Notified controller that worker is ready");
#endif
} else {
err(true, "Couldn't notify controller that worker is ready");
}
chunk_free(rmsg);
}
}
/* This takes comma delimited list of name=value pairs.
* If value is empty (i.e. name=,name1=...) the the value is pulled from the
* env
* return 1 on success, 0 on failure
*/
void parse_variable_list(
memmgr **mm, /* memory manager */
job_data **dest_hash, /* This is the dest hashmap for vars found */
job_data *user_env, /* This is the source hashmap */
int var_type, /* Type for vars not pulled from the source hash */
int op_type, /* Op for vars not pulled from the source hash */
char *the_list) /* name=value,name1=value1,etc to be parsed */
{
int alloc_size = 0;
char *s = NULL;
char *e = NULL;
char *delim = NULL;
char *name = NULL;
char *tmp_name = NULL;
char *val = NULL;
job_data *hash_var = NULL;
s = the_list;
while (s)
{
delim = strpbrk(s, "=,");
/* There is no = or , in the following string */
/* The string is improperly formatted (, found when = should have been) */
/* The start character is a , or = */
if ((delim == NULL) || (*delim == ',') || (delim == s))
break;
e = strchr(delim+1, ',');
/* This is last value */
if (!e)
e = strchr(delim+1, '\0');
/* Get the variable from the src hash */
/* Set the variable from the incoming data */
alloc_size = delim - s;
/* the +8 is for prepending the value of pbs_var_ to the value
* This is used and removed in build_var_list later */
calloc_or_fail(mm, &name, alloc_size+8, "parse_variable_list name");
memcpy(name, "pbs_var_", 8);
memcpy(name+8, s, alloc_size);
if ((e - delim) == 1)
{
calloc_or_fail(mm, &tmp_name, alloc_size, "parse_variable_list name");
memcpy(tmp_name, s, alloc_size);
if (hash_find(user_env, tmp_name, &hash_var))
{
hash_add_or_exit(mm, dest_hash, name, hash_var->value, hash_var->var_type);
}
else
{
hash_add_or_exit(mm, dest_hash, name, "", CMDLINE_DATA);
}
}
else
{
delim++; /* Move past the = */
alloc_size = e - delim;
calloc_or_fail(mm, &val, alloc_size, "parse_variable_list val");
strncpy(val, delim, alloc_size);
hash_add_or_exit(mm, dest_hash, name, val, var_type);
}
if (*e == '\0')
s = NULL; /* End of string previously found */
else
s = e + 1; /* Set the start to one char beyond the , */
}
} /* END parse_variable_list() */
