// Receive ack message
int receive_ack_and_finit(int num)
{
int sockfd = sock_first;
struct sockaddr_in addr = first_addr;
char* nums = "first";
if(num == LAST)
{
sockfd = sock_last;
addr = last_addr;
nums = "last";
}
if(setsockopt(sockfd, SOL_SOCKET, SO_RCVTIMEO, &error_timeout, sizeof(struct timeval)) < 0)
{
print_error("Failed to set error timeout");
}
else
{
print_success("Error timeout set");
}
unsigned char buf[MAX_DATA];
unsigned addr_len = sizeof(addr);
print_info("Waiting for ack from %d...", num);
int len = recvfrom(sockfd, buf, MAX_DATA, 0, (struct sockaddr*)&addr, &addr_len);
union msg received_msg;
int msg_type = unpack_msg(buf, &received_msg);
if(len < 0 || msg_type != ACK_MSG)
{
print_error("Failed to receive ack msg from %s sensor", nums);
print_info("Received %d type instead", msg_type);
print_success("Closing socket");
close(sockfd);
return -1;
}
print_success("Received ack msg from %d", num);
len = recvfrom(sockfd, buf, MAX_DATA, 0, (struct sockaddr*)&addr, &addr_len);
msg_type = unpack_msg(buf, &received_msg);
if(len < 0 || msg_type != FINIT_MSG)
{
print_error("Failed to receive finit msg from %s sensor", nums);
print_info("Received %d type instead", msg_type);
print_success("Closing socket");
close(sockfd);
return -1;
}
print_success("Received finit msg from %d", num);
if(setsockopt(sockfd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(struct timeval)) < 0)
{
print_error("Failed to return to normal timeout");
}
else
{
print_success("Returned to normal timeout");
}
return 0;
}
static int process_msg(struct ipc *ipc, void *buf, size_t len)
{
ipc_handler_t handler;
uint32_t func_id = 0;
char out_arg[1024];
size_t out_len;
size_t ret_len;
struct ipc_packet *pkt = (struct ipc_packet *)buf;
if (-1 == unpack_msg(pkt, &func_id, &out_arg, &out_len)) {
logd("unpack_msg failed!\n");
return -1;
}
if (find_ipc_handler(func_id, &handler) == 0 ) {
handler.cb(ipc, out_arg, out_len, _arg_buf, &ret_len);//direct call cb is not good, will be change to workq
} else {
logi("no callback for this MSG ID in process_msg\n");
}
if (ret_len > 0) {
if (0 > pack_msg(pkt, func_id, _arg_buf, ret_len)) {
loge("pack_msg failed!\n");
return -1;
}
ipc->ops->send(ipc, pkt, sizeof(ipc_packet_t) + ret_len);
}
return 0;
}
static void on_read(int fd, void *arg)
{
struct rpc *r = (struct rpc *)arg;
char out_arg[1024];
struct iovec *recv_buf;
uint32_t msg_id;
size_t out_len;
memset(&r->recv_pkt, 0, sizeof(struct rpc_packet));
recv_buf = rpc_recv_buf(r);
if (!recv_buf) {
loge("rpc_recv_buf failed!\n");
return;
}
struct rpc_packet *pkt = &r->recv_pkt;
unpack_msg(pkt, &msg_id, &out_arg, &out_len);
logi("msg_id = %08x\n", msg_id);
if (r->state == rpc_inited) {
r->send_pkt.header.uuid_src = *(uint32_t *)pkt->payload;
thread_sem_signal(r->dispatch_thread);
r->state = rpc_connected;
} else if (r->state == rpc_connected) {
struct iovec buf;
buf.iov_len = pkt->header.payload_len;
buf.iov_base = pkt->payload;
process_msg(r, &buf);
//free(pkt->payload);
thread_sem_signal(r->dispatch_thread);
}
}
// Thread for receving reconf
void* reconf_fun(void* arg)
{
int sockfd = socket(AF_UNIX, SOCK_STREAM, 0);
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
strcpy(addr.sun_path, "reconf_socket");
unlink("reconf_socket");
if(bind(sockfd, (struct sockaddr*)&addr, sizeof(addr)) < 0)
{
print_error("Failed to create unix socket, %s %d\n", strerror(errno), errno);
}
print_success("Created socket");
int sock;
if (listen(sockfd, 5) < 0)
print_error("Failed to listen on unix socket\n");
if ((sock = accept(sockfd, NULL, NULL)) < 0)
print_error("Failed to accept unix socket\n");
else
print_success("Unix socket accepted");
char buf[1] = {' '};
while(1)
{
if(read(sock, buf, sizeof(buf)) > 0)
{
if(buf[0] == 'r' && mode == RING)
{
//print_info("I got reconf!!\n");
mode = RECONF;
send_reconf_msg();
while(1)
{
//print_info("Waiting for reconf to come back\n");
unsigned char buf[MAX_DATA];
unsigned last_addr_len = sizeof(last_addr);
int len = recvfrom(sock_last, buf, MAX_DATA, 0, (struct sockaddr*)&last_addr, &last_addr_len);
if(len < 1)
continue;
union msg received_msg;
int msg_type = unpack_msg(buf, &received_msg);
//print_info("Got message %d\n", msg_type);
if(msg_type == RECONF_MSG)
break;
}
print_success("Now you can reconfigure network!");
}
else if(buf[0] == 'i' && mode == RECONF)
{
send_init_msg();
mode = RING;
}
}
else
sleep(1);
}
return NULL;
}
// Loop for communication with last sensor
void last_loop()
{
print_info("Last loop");
while(1)
{
if(mode == RECONF)
{
sleep(1);
continue;
}
print_info("Waiting for data from last...");
unsigned char buf[MAX_DATA];
unsigned last_addr_len = sizeof(last_addr);
int len = recvfrom(sock_last, buf, MAX_DATA, 0, (struct sockaddr*)&last_addr, &last_addr_len);
if(len < 0)
{
mode = DOUBLE_LIST;
resolving_error = 1;
print_warning("Didn't receive data message from last sensor");
print_success("Mode changed to double-list");
if(send_error_msg(FIRST) < 0 || receive_ack_and_finit(FIRST) < 0)
{
close_first = 1;
}
if(send_error_msg(LAST) < 0 || receive_ack_and_finit(LAST) < 0)
{
resolving_error = 0;
return;
}
resolving_error = 0;
}
else
{
union msg received_msg;
int msg_type = unpack_msg(buf, &received_msg);
switch(msg_type)
{
case INIT_MSG:
take_init_msg(received_msg.init);
break;
case DATA_MSG:
take_data_msg(received_msg.data);
break;
default:
print_warning("Received unknown bytes");
}
cleanup_msg(&received_msg);
}
usleep(SERVER_TIMEOUT*1000);
if(mode == DOUBLE_LIST)
{
print_info("Sending data to last sensor...");
send_data_msg(LAST);
}
}
}
extern int slurm_persist_msg_unpack(slurm_persist_conn_t *persist_conn,
persist_msg_t *resp_msg, Buf buffer)
{
int rc;
xassert(persist_conn);
xassert(resp_msg);
if (persist_conn->flags & PERSIST_FLAG_DBD) {
rc = unpack_slurmdbd_msg((slurmdbd_msg_t *)resp_msg,
persist_conn->version,
buffer);
} else {
slurm_msg_t msg;
slurm_msg_t_init(&msg);
msg.protocol_version = persist_conn->version;
safe_unpack16(&msg.msg_type, buffer);
rc = unpack_msg(&msg, buffer);
resp_msg->msg_type = msg.msg_type;
resp_msg->data = msg.data;
}
/* Here we transfer the auth_cred to the persist_conn just in case in the
* future we need to use it in some way to verify things for messages
* that don't have on that will follow on the connection.
*/
if (resp_msg->msg_type == REQUEST_PERSIST_INIT) {
slurm_msg_t *msg = resp_msg->data;
if (persist_conn->auth_cred)
g_slurm_auth_destroy(persist_conn->auth_cred);
persist_conn->auth_cred = msg->auth_cred;
msg->auth_cred = NULL;
}
return rc;
unpack_error:
return SLURM_ERROR;
}
static void on_return(struct ipc *ipc, void *buf, size_t len)
{
uint32_t func_id;
size_t out_len;
struct ipc_packet *pkt = (struct ipc_packet *)buf;
memset(ipc->resp_buf, 0, MAX_IPC_RESP_BUF_LEN);
if (-1 == unpack_msg(pkt, &func_id, ipc->resp_buf, &out_len)) {
logd("unpack_msg failed!\n");
return;
}
if (-1 == pop_async_cmd(ipc, func_id)) {
logd("msg received is not the response of cmd_id!\n");
return;
}
logd("buf=%s\n", ipc->resp_buf);
ipc->resp_len = out_len;
sem_post(&ipc->sem);
}
// Loop for communication with first sensor
void* first_loop(void* arg)
{
print_success("Loop thread started");
send_init_msg();
while(1)
{
if(close_first)
return NULL;
if(mode == RECONF)
{
sleep(1);
continue;
}
if(resolving_error)
{
usleep(SERVER_TIMEOUT*1000);
continue;
}
send_data_msg(FIRST);
if(mode == DOUBLE_LIST)
{
print_info("Waiting for data from first...");
unsigned char buf[MAX_DATA];
unsigned first_addr_len = sizeof(first_addr);
int len = recvfrom(sock_first, buf, MAX_DATA, 0, (struct sockaddr*)&first_addr, &first_addr_len);
if(len < 0)
{
print_error("Second problem with network(not received msg from first sensor), terminating....");
exit(-1);
}
union msg received_msg;
int msg_type = unpack_msg(buf, &received_msg);
if(msg_type == DATA_MSG)
take_data_msg(received_msg.data);
else
print_warning("Received unknown bytes");
}
usleep(SERVER_TIMEOUT*1000);
}
return NULL;
}
/*
* This function handles the initialization information from slurmd
* sent by _send_slurmstepd_init() in src/slurmd/slurmd/req.c.
*/
static int
_init_from_slurmd(int sock, char **argv,
slurm_addr_t **_cli, slurm_addr_t **_self, slurm_msg_t **_msg,
int *_ngids, gid_t **_gids)
{
char *incoming_buffer = NULL;
Buf buffer;
int step_type;
int len;
slurm_addr_t *cli = NULL;
slurm_addr_t *self = NULL;
slurm_msg_t *msg = NULL;
int ngids = 0;
gid_t *gids = NULL;
uint16_t port;
char buf[16];
log_options_t lopts = LOG_OPTS_INITIALIZER;
log_init(argv[0], lopts, LOG_DAEMON, NULL);
/* receive job type from slurmd */
safe_read(sock, &step_type, sizeof(int));
debug3("step_type = %d", step_type);
/* receive reverse-tree info from slurmd */
pthread_mutex_lock(&step_complete.lock);
safe_read(sock, &step_complete.rank, sizeof(int));
safe_read(sock, &step_complete.parent_rank, sizeof(int));
safe_read(sock, &step_complete.children, sizeof(int));
safe_read(sock, &step_complete.depth, sizeof(int));
safe_read(sock, &step_complete.max_depth, sizeof(int));
safe_read(sock, &step_complete.parent_addr, sizeof(slurm_addr_t));
step_complete.bits = bit_alloc(step_complete.children);
step_complete.jobacct = jobacct_gather_g_create(NULL);
pthread_mutex_unlock(&step_complete.lock);
/* receive conf from slurmd */
if ((conf = read_slurmd_conf_lite (sock)) == NULL)
fatal("Failed to read conf from slurmd");
log_alter(conf->log_opts, 0, conf->logfile);
debug2("debug level is %d.", conf->debug_level);
/* acct info */
jobacct_gather_g_startpoll(conf->job_acct_gather_freq);
switch_g_slurmd_step_init();
slurm_get_ip_str(&step_complete.parent_addr, &port, buf, 16);
debug3("slurmstepd rank %d, parent address = %s, port = %u",
step_complete.rank, buf, port);
/* receive cli from slurmd */
safe_read(sock, &len, sizeof(int));
incoming_buffer = xmalloc(sizeof(char) * len);
safe_read(sock, incoming_buffer, len);
buffer = create_buf(incoming_buffer,len);
cli = xmalloc(sizeof(slurm_addr_t));
if(slurm_unpack_slurm_addr_no_alloc(cli, buffer) == SLURM_ERROR)
fatal("slurmstepd: problem with unpack of slurmd_conf");
free_buf(buffer);
/* receive self from slurmd */
safe_read(sock, &len, sizeof(int));
if (len > 0) {
/* receive packed self from main slurmd */
incoming_buffer = xmalloc(sizeof(char) * len);
safe_read(sock, incoming_buffer, len);
buffer = create_buf(incoming_buffer,len);
self = xmalloc(sizeof(slurm_addr_t));
if (slurm_unpack_slurm_addr_no_alloc(self, buffer)
== SLURM_ERROR) {
fatal("slurmstepd: problem with unpack of "
"slurmd_conf");
}
free_buf(buffer);
}
/* Receive GRES information from slurmd */
gres_plugin_recv_stepd(sock);
/* receive req from slurmd */
safe_read(sock, &len, sizeof(int));
incoming_buffer = xmalloc(sizeof(char) * len);
safe_read(sock, incoming_buffer, len);
buffer = create_buf(incoming_buffer,len);
msg = xmalloc(sizeof(slurm_msg_t));
slurm_msg_t_init(msg);
switch(step_type) {
case LAUNCH_BATCH_JOB:
msg->msg_type = REQUEST_BATCH_JOB_LAUNCH;
break;
case LAUNCH_TASKS:
msg->msg_type = REQUEST_LAUNCH_TASKS;
break;
default:
fatal("Unrecognized launch RPC");
break;
}
if(unpack_msg(msg, buffer) == SLURM_ERROR)
fatal("slurmstepd: we didn't unpack the request correctly");
free_buf(buffer);
/* receive cached group ids array for the relevant uid */
safe_read(sock, &ngids, sizeof(int));
if (ngids > 0) {
int i;
uint32_t tmp32;
gids = (gid_t *)xmalloc(sizeof(gid_t) * ngids);
for (i = 0; i < ngids; i++) {
safe_read(sock, &tmp32, sizeof(uint32_t));
gids[i] = (gid_t)tmp32;
debug2("got gid %d", gids[i]);
}
}
*_cli = cli;
*_self = self;
*_msg = msg;
*_ngids = ngids;
*_gids = gids;
return 1;
rwfail:
fatal("Error reading initialization data from slurmd");
exit(1);
}
extern int unpack_slurmdbd_msg(slurmdbd_msg_t *resp,
uint16_t rpc_version, Buf buffer)
{
int rc = SLURM_SUCCESS;
slurm_msg_t msg;
safe_unpack16(&resp->msg_type, buffer);
if (rpc_version < SLURM_MIN_PROTOCOL_VERSION) {
error("slurmdbd: Invalid message version=%hu, type:%hu",
rpc_version, resp->msg_type);
return SLURM_ERROR;
}
switch (resp->msg_type) {
case PERSIST_RC:
slurm_msg_t_init(&msg);
msg.protocol_version = rpc_version;
msg.msg_type = resp->msg_type;
rc = unpack_msg(&msg, buffer);
resp->data = msg.data;
break;
case REQUEST_PERSIST_INIT:
resp->data = xmalloc(sizeof(slurm_msg_t));
slurm_msg_t_init(resp->data);
rc = slurm_unpack_received_msg(
(slurm_msg_t *)resp->data, 0, buffer);
break;
case DBD_ADD_ACCOUNTS:
case DBD_ADD_TRES:
case DBD_ADD_ASSOCS:
case DBD_ADD_CLUSTERS:
case DBD_ADD_FEDERATIONS:
case DBD_ADD_RES:
case DBD_ADD_USERS:
case DBD_GOT_ACCOUNTS:
case DBD_GOT_TRES:
case DBD_GOT_ASSOCS:
case DBD_GOT_CLUSTERS:
case DBD_GOT_EVENTS:
case DBD_GOT_FEDERATIONS:
case DBD_GOT_JOBS:
case DBD_GOT_LIST:
case DBD_GOT_PROBS:
case DBD_ADD_QOS:
case DBD_GOT_QOS:
case DBD_GOT_RESVS:
case DBD_GOT_RES:
case DBD_ADD_WCKEYS:
case DBD_GOT_WCKEYS:
case DBD_GOT_TXN:
case DBD_GOT_USERS:
case DBD_GOT_CONFIG:
case DBD_SEND_MULT_JOB_START:
case DBD_GOT_MULT_JOB_START:
case DBD_SEND_MULT_MSG:
case DBD_GOT_MULT_MSG:
case DBD_FIX_RUNAWAY_JOB:
rc = slurmdbd_unpack_list_msg(
(dbd_list_msg_t **)&resp->data, rpc_version,
resp->msg_type, buffer);
break;
case DBD_ADD_ACCOUNT_COORDS:
case DBD_REMOVE_ACCOUNT_COORDS:
rc = _unpack_acct_coord_msg(
(dbd_acct_coord_msg_t **)&resp->data,
rpc_version, buffer);
break;
case DBD_ARCHIVE_LOAD:
rc = slurmdb_unpack_archive_rec(
&resp->data, rpc_version, buffer);
break;
case DBD_CLUSTER_TRES:
case DBD_FLUSH_JOBS:
rc = _unpack_cluster_tres_msg(
(dbd_cluster_tres_msg_t **)&resp->data,
rpc_version, buffer);
break;
case DBD_GET_ACCOUNTS:
case DBD_GET_TRES:
case DBD_GET_ASSOCS:
case DBD_GET_CLUSTERS:
case DBD_GET_EVENTS:
case DBD_GET_FEDERATIONS:
case DBD_GET_JOBS_COND:
case DBD_GET_PROBS:
case DBD_GET_QOS:
case DBD_GET_RESVS:
case DBD_GET_RES:
case DBD_GET_TXN:
case DBD_GET_USERS:
case DBD_GET_WCKEYS:
case DBD_REMOVE_ACCOUNTS:
case DBD_REMOVE_ASSOCS:
case DBD_REMOVE_CLUSTERS:
case DBD_REMOVE_FEDERATIONS:
case DBD_REMOVE_QOS:
case DBD_REMOVE_RES:
case DBD_REMOVE_WCKEYS:
case DBD_REMOVE_USERS:
case DBD_ARCHIVE_DUMP:
rc = _unpack_cond_msg(
(dbd_cond_msg_t **)&resp->data, rpc_version,
resp->msg_type, buffer);
break;
case DBD_GET_ASSOC_USAGE:
case DBD_GOT_ASSOC_USAGE:
case DBD_GET_CLUSTER_USAGE:
case DBD_GOT_CLUSTER_USAGE:
case DBD_GET_WCKEY_USAGE:
case DBD_GOT_WCKEY_USAGE:
rc = slurmdbd_unpack_usage_msg(
(dbd_usage_msg_t **)&resp->data, rpc_version,
resp->msg_type, buffer);
break;
case DBD_FINI:
rc = slurmdbd_unpack_fini_msg((dbd_fini_msg_t **)&resp->data,
rpc_version,
buffer);
break;
case DBD_JOB_COMPLETE:
rc = _unpack_job_complete_msg(
(dbd_job_comp_msg_t **)&resp->data,
rpc_version, buffer);
break;
case DBD_JOB_START:
rc = _unpack_job_start_msg(
&resp->data, rpc_version, buffer);
break;
case DBD_ID_RC:
rc = slurmdbd_unpack_id_rc_msg(
&resp->data, rpc_version, buffer);
break;
case DBD_JOB_SUSPEND:
rc = _unpack_job_suspend_msg(
(dbd_job_suspend_msg_t **)&resp->data, rpc_version,
buffer);
break;
case DBD_MODIFY_ACCOUNTS:
case DBD_MODIFY_ASSOCS:
case DBD_MODIFY_CLUSTERS:
case DBD_MODIFY_FEDERATIONS:
case DBD_MODIFY_JOB:
case DBD_MODIFY_QOS:
case DBD_MODIFY_RES:
case DBD_MODIFY_USERS:
rc = _unpack_modify_msg(
(dbd_modify_msg_t **)&resp->data,
rpc_version,
resp->msg_type,
buffer);
break;
case DBD_NODE_STATE:
rc = _unpack_node_state_msg(
(dbd_node_state_msg_t **)&resp->data, rpc_version,
buffer);
break;
case DBD_STEP_COMPLETE:
rc = _unpack_step_complete_msg(
(dbd_step_comp_msg_t **)&resp->data,
rpc_version, buffer);
break;
case DBD_STEP_START:
rc = _unpack_step_start_msg(
(dbd_step_start_msg_t **)&resp->data,
rpc_version, buffer);
break;
case DBD_REGISTER_CTLD:
rc = _unpack_register_ctld_msg(
(dbd_register_ctld_msg_t **)&resp->data,
rpc_version, buffer);
break;
case DBD_ROLL_USAGE:
rc = _unpack_roll_usage_msg(
(dbd_roll_usage_msg_t **)&resp->data, rpc_version,
buffer);
break;
case DBD_ADD_RESV:
case DBD_REMOVE_RESV:
case DBD_MODIFY_RESV:
rc = _unpack_rec_msg(
(dbd_rec_msg_t **)&resp->data, rpc_version,
resp->msg_type, buffer);
break;
case DBD_GET_CONFIG:
rc = _unpack_config_name(
(char **)&resp->data, rpc_version, buffer);
break;
case DBD_RECONFIG:
case DBD_GET_STATS:
case DBD_CLEAR_STATS:
case DBD_SHUTDOWN:
/* No message to unpack */
break;
case DBD_GOT_STATS:
rc = slurmdb_unpack_stats_msg(
(void **)&resp->data, rpc_version, buffer);
break;
default:
error("slurmdbd: Invalid message type unpack %u(%s)",
resp->msg_type,
slurmdbd_msg_type_2_str(resp->msg_type, 1));
return SLURM_ERROR;
}
return rc;
unpack_error:
return SLURM_ERROR;
}
int pomme_client_get_data(uuid_t id,
size_t off,
size_t len,
int handle,
void *buffer,
int *r_len)
{
assert( buffer !=NULL );
assert( r_len != NULL );
int ret = 0,flags=0;
pomme_protocol_t pro,rpro;
memset( &pro, 0, sizeof(pro));
memset( &rpro, 0, sizeof(rpro));
pro.op = get_data;
pro.total_len = len;
pro.offset = off;
uuid_copy(pro.id, id);
pomme_pack_t *buf = NULL;
if( ( ret = pack_msg(&pro, &buf) ) < 0 )
{
debug("pack msg failure");
goto err;
}
if( ( ret = pomme_send(handle, buf->data,
pomme_msg_get_len((&pro)),flags )) < 0)
{
debug("send data fail");
goto err;
}
/*
* recv data package
*/
unsigned char t_buffer[POMME_PACKAGE_SIZE];
size_t t_len = 0;
if( ( ret = pomme_recv(handle, t_buffer,
POMME_PACKAGE_SIZE,
&t_len, flags) ) < 0 )
{
debug("recv first fail");
goto err;
}
pomme_pack_t *p_buffer = NULL;
if( (ret = pomme_pack_create(&p_buffer,t_buffer,
t_len)) < 0 )
{
debug("create buffer error");
goto err;
}
if( (ret = unpack_msg( &rpro, p_buffer) ) < 0 )
{
debug("unpack msg fail");
goto data_err;
}
pomme_print_proto(&rpro,NULL);
if( rpro.op != put_data )
{
debug("wrong operation");
goto data_err;
}
if( rpro.total_len > len )
{
debug("too much data recved");
goto data_err;
}
size_t tr_len = rpro.len;
memcpy(buffer,t_buffer, tr_len);
while( tr_len < rpro.total_len )
{
size_t tmp = 0;
ret = pomme_recv(handle, buffer+tr_len,
rpro.total_len - tr_len,
&tmp, flags);
if( ret < 0 )
{
debug("recv err,data total get:%d",tr_len);
break;
}
tr_len += tmp;
}
*r_len = tr_len;
debug("data len %d",tr_len);
data_err:
pomme_pack_distroy(&p_buffer);
err:
pomme_pack_distroy(&buf);
return ret;
}
