int main(int argc, char* argv[])
{
int rc, idx;
uint32_t addr, netmask, netaddr;
struct sockaddr_in inaddr;
char **aliases=NULL;
if (0 > (rc = opal_init(&argc, &argv))) {
fprintf(stderr, "orte_interface: couldn't init opal - error code %d\n", rc);
return rc;
}
if (2 == argc) {
rc = opal_iftupletoaddr(argv[1], &netaddr, &netmask);
fprintf(stderr, "netaddr %03d.%03d.%03d.%03d netmask %03d.%03d.%03d.%03d rc %d\n",
OPAL_IF_FORMAT_ADDR(netaddr), OPAL_IF_FORMAT_ADDR(netmask), rc);
/* search for a matching interface - take the first one within the returned scope */
idx = opal_ifbegin();
while (0 < idx) {
/* ignore the loopback interface */
if (opal_ifisloopback(idx)) {
fprintf(stderr, "LOOPBACK IGNORED\n");
idx = opal_ifnext(idx);
continue;
}
if (0 != (rc = opal_ifindextoaddr(idx, (struct sockaddr*)&inaddr, sizeof(inaddr)))) {
break;
}
addr = ntohl(inaddr.sin_addr.s_addr);
fprintf(stderr, "checking netaddr %03d.%03d.%03d.%03d addr %03d.%03d.%03d.%03d netmask %03d.%03d.%03d.%03d rc %d\n",
OPAL_IF_FORMAT_ADDR(netaddr), OPAL_IF_FORMAT_ADDR(addr), OPAL_IF_FORMAT_ADDR(netmask), rc);
if (netaddr == (addr & netmask)) {
fprintf(stderr, "MATCH FOUND\n");
}
idx = opal_ifnext(idx);
}
}
/* check the aliases */
opal_ifgetaliases(&aliases);
idx = 0;
while (NULL != aliases[idx]) {
fprintf(stderr, "alias: %s\n", aliases[idx]);
idx++;
}
opal_finalize();
return 0;
}
static int setup_channel(rmcast_base_channel_t *chan, uint8_t direction)
{
int rc;
int xmitsd, recvsd;
if (0 <= chan->xmit && 0 <= chan->recv) {
/* already setup */
OPAL_OUTPUT_VERBOSE((2, orte_rmcast_base.rmcast_output,
"%s setup:channel %d already setup",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
chan->channel));
return ORTE_SUCCESS;
}
/* setup the IPv4 addr info */
chan->addr.sin_family = AF_INET;
chan->addr.sin_addr.s_addr = htonl(chan->network);
chan->addr.sin_port = htons(chan->port);
OPAL_OUTPUT_VERBOSE((2, orte_rmcast_base.rmcast_output,
"%s setup:channel addr %03d.%03d.%03d.%03d port %d for %s:%s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
OPAL_IF_FORMAT_ADDR(chan->network), (int)chan->port,
(ORTE_RMCAST_RECV & direction) ? " RECV" : " ",
(ORTE_RMCAST_XMIT & direction) ? " XMIT" : " "));
if (0 > chan->xmit && (ORTE_RMCAST_XMIT & direction)) {
/* create a xmit socket */
if (ORTE_SUCCESS != (rc = setup_socket(&xmitsd, chan, false))) {
ORTE_ERROR_LOG(rc);
return rc;
}
chan->xmit = xmitsd;
}
if (0 > chan->recv && (ORTE_RMCAST_RECV & direction)) {
/* create a recv socket */
if (ORTE_SUCCESS != (rc = setup_socket(&recvsd, chan, true))) {
ORTE_ERROR_LOG(rc);
return rc;
}
chan->recv = recvsd;
/* setup an event to catch messages */
OPAL_OUTPUT_VERBOSE((2, orte_rmcast_base.rmcast_output,
"%s setup:channel activating recv event on fd %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),(int)chan->recv));
opal_event_set(opal_event_base, &chan->recv_ev, chan->recv,
OPAL_EV_READ|OPAL_EV_PERSIST, recv_handler, chan);
opal_event_add(&chan->recv_ev, 0);
}
return ORTE_SUCCESS;
}
static int send_data(rmcast_base_send_t *snd, orte_rmcast_channel_t channel)
{
char *bytes=NULL;
int32_t sz;
int rc;
opal_buffer_t *buf=NULL;
rmcast_base_channel_t *chan;
rmcast_send_log_t *log, *lg;
if (!comm_enabled) {
return ORTE_ERR_COMM_DISABLED;
}
OPAL_OUTPUT_VERBOSE((2, orte_rmcast_base.rmcast_output,
"%s transmitting data for channel %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), channel));
/* setup the message for xmission */
if (ORTE_SUCCESS != (rc = orte_rmcast_base_queue_xmit(snd, channel, &buf, &chan))) {
ORTE_ERROR_LOG(rc);
goto CLEANUP;
}
/* store the working buf in the send ring buffer in case we
* need to retransmit it later
*/
log = OBJ_NEW(rmcast_send_log_t);
log->channel = chan->channel;
log->seq_num = chan->seq_num;
opal_dss.copy_payload(log->buf, buf);
if (NULL != (lg = (rmcast_send_log_t*)opal_ring_buffer_push(&chan->cache, log))) {
/* release the old message */
OPAL_OUTPUT_VERBOSE((5, orte_rmcast_base.rmcast_output,
"%s releasing message %d channel %d from log",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
lg->seq_num, lg->channel));
OBJ_RELEASE(lg);
}
/* unload the working buf to obtain the payload */
if (ORTE_SUCCESS != (rc = opal_dss.unload(buf, (void**)&bytes, &sz))) {
ORTE_ERROR_LOG(rc);
goto CLEANUP;
}
OPAL_OUTPUT_VERBOSE((2, orte_rmcast_base.rmcast_output,
"%s rmcast:udp multicasting %d bytes to network %03d.%03d.%03d.%03d port %d tag %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), sz,
OPAL_IF_FORMAT_ADDR(chan->network), (int)chan->port, (int)snd->tag));
if (sz != (rc = sendto(chan->xmit, bytes, sz, 0,
(struct sockaddr *)&(chan->addr), sizeof(struct sockaddr_in)))) {
/* didn't get the message out */
opal_output(0, "%s failed to send message to multicast network %03d.%03d.%03d.%03d on\n\terror %s(%d)",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), OPAL_IF_FORMAT_ADDR(chan->network),
strerror(errno), errno);
rc = ORTE_ERR_COMM_FAILURE;
} else {
rc = ORTE_SUCCESS;
}
if (NULL != snd->buf) {
/* call the cbfunc if required */
if (NULL != snd->cbfunc_buffer) {
snd->cbfunc_buffer(rc, chan->channel, chan->seq_num, snd->tag,
ORTE_PROC_MY_NAME, snd->buf, snd->cbdata);
}
} else {
/* call the cbfunc if required */
if (NULL != snd->cbfunc_iovec) {
snd->cbfunc_iovec(rc, chan->channel, chan->seq_num, snd->tag, ORTE_PROC_MY_NAME,
snd->iovec_array, snd->iovec_count, snd->cbdata);
}
}
CLEANUP:
if (NULL != buf) {
OBJ_RELEASE(buf);
}
if (NULL != bytes) {
free(bytes);
}
return rc;
}
static int setup_socket(int *sd, rmcast_base_channel_t *chan, bool recvsocket)
{
uint8_t ttl = 1;
struct sockaddr_in inaddr;
struct ip_mreq req;
int addrlen;
int target_sd;
int flags;
OPAL_OUTPUT_VERBOSE((2, orte_rmcast_base.rmcast_output,
"setup:socket addr %03d.%03d.%03d.%03d port %d",
OPAL_IF_FORMAT_ADDR(chan->network), (int)chan->port));
target_sd = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP);
if(target_sd < 0) {
if (EAFNOSUPPORT != opal_socket_errno) {
opal_output(0,"rmcast:init: socket() failed: %s (%d)",
strerror(opal_socket_errno), opal_socket_errno);
}
return ORTE_ERR_IN_ERRNO;
}
/* set the multicast flags */
if ((setsockopt(target_sd, IPPROTO_IP, IP_MULTICAST_TTL,
(void *)&ttl, sizeof(ttl))) < 0) {
opal_output(0,"rmcast:init: socketopt() failed on MULTICAST_TTL: %s (%d)",
strerror(opal_socket_errno), opal_socket_errno);
return ORTE_ERR_IN_ERRNO;
}
/* enable port sharing */
flags = 1;
if (setsockopt (target_sd, SOL_SOCKET, SO_REUSEADDR, (const char *)&flags, sizeof(flags)) < 0) {
opal_output(0, "rmcast:udp: unable to set the "
"SO_REUSEADDR option (%s:%d)\n",
strerror(opal_socket_errno), opal_socket_errno);
CLOSE_THE_SOCKET(target_sd);
return ORTE_ERROR;
}
/* if this is the recv side... */
if (recvsocket) {
memset(&inaddr, 0, sizeof(inaddr));
inaddr.sin_family = AF_INET;
inaddr.sin_addr.s_addr = htonl(chan->network);
inaddr.sin_port = htons(chan->port);
addrlen = sizeof(struct sockaddr_in);
/* bind the socket */
if (bind(target_sd, (struct sockaddr*)&inaddr, addrlen) < 0) {
opal_output(0, "%s rmcast:init: bind() failed for addr %03d.%03d.%03d.%03d port %d\n\tError: %s (%d)",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), OPAL_IF_FORMAT_ADDR(chan->network), (int)chan->port,
strerror(opal_socket_errno), opal_socket_errno);
CLOSE_THE_SOCKET(target_sd);
return ORTE_ERROR;
}
/* set membership on the multicast interface */
memset(&req, 0, sizeof (req));
req.imr_multiaddr.s_addr = htonl(chan->network);
req.imr_interface.s_addr = htonl(chan->interface);
OPAL_OUTPUT_VERBOSE((2, orte_rmcast_base.rmcast_output,
"setup:socket:membership addr %03d.%03d.%03d.%03d interface %03d.%03d.%03d.%03d",
OPAL_IF_FORMAT_ADDR(chan->network), OPAL_IF_FORMAT_ADDR(chan->interface)));
if ((setsockopt(target_sd, IPPROTO_IP, IP_ADD_MEMBERSHIP,
(void *)&req, sizeof (req))) < 0) {
opal_output(0, "%s rmcast:init: setsockopt() failed on ADD_MEMBERSHIP\n"
"\tfor multicast network %03d.%03d.%03d.%03d interface %03d.%03d.%03d.%03d\n\tError: %s (%d)",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
OPAL_IF_FORMAT_ADDR(chan->network), OPAL_IF_FORMAT_ADDR(chan->interface),
strerror(opal_socket_errno), opal_socket_errno);
CLOSE_THE_SOCKET(target_sd);
return ORTE_ERROR;
}
/* set the recvbuf size */
if ((setsockopt(target_sd, SOL_SOCKET, SO_RCVBUF,
&orte_rmcast_udp_rcvbuf_size, sizeof(orte_rmcast_udp_rcvbuf_size))) < 0) {
opal_output(0, "%s rmcast:init: setsockopt() failed on SO_RCVBUF\n"
"\tfor multicast network %03d.%03d.%03d.%03d interface %03d.%03d.%03d.%03d\n\tError: %s (%d)",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
OPAL_IF_FORMAT_ADDR(chan->network), OPAL_IF_FORMAT_ADDR(chan->interface),
strerror(opal_socket_errno), opal_socket_errno);
CLOSE_THE_SOCKET(target_sd);
return ORTE_ERROR;
}
} else {
/* on the xmit side, need to set the interface */
memset(&inaddr, 0, sizeof(inaddr));
inaddr.sin_addr.s_addr = htonl(chan->interface);
addrlen = sizeof(struct sockaddr_in);
OPAL_OUTPUT_VERBOSE((2, orte_rmcast_base.rmcast_output,
"setup:socket:xmit interface %03d.%03d.%03d.%03d",
OPAL_IF_FORMAT_ADDR(chan->interface)));
if ((setsockopt(target_sd, IPPROTO_IP, IP_MULTICAST_IF,
(void *)&inaddr, addrlen)) < 0) {
opal_output(0, "%s rmcast:init: setsockopt() failed on MULTICAST_IF\n"
"\tfor multicast network %03d.%03d.%03d.%03d interface %03d.%03d.%03d.%03d\n\tError: %s (%d)",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
OPAL_IF_FORMAT_ADDR(chan->network), OPAL_IF_FORMAT_ADDR(chan->interface),
strerror(opal_socket_errno), opal_socket_errno);
CLOSE_THE_SOCKET(target_sd);
return ORTE_ERROR;
}
/* set the sendbuf size */
if ((setsockopt(target_sd, SOL_SOCKET, SO_SNDBUF,
&orte_rmcast_udp_sndbuf_size, sizeof(orte_rmcast_udp_sndbuf_size))) < 0) {
opal_output(0, "%s rmcast:init: setsockopt() failed on SO_SNDBUF\n"
"\tfor multicast network %03d.%03d.%03d.%03d interface %03d.%03d.%03d.%03d\n\tError: %s (%d)",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
OPAL_IF_FORMAT_ADDR(chan->network), OPAL_IF_FORMAT_ADDR(chan->interface),
strerror(opal_socket_errno), opal_socket_errno);
CLOSE_THE_SOCKET(target_sd);
return ORTE_ERROR;
}
}
/* set socket up to be non-blocking */
if((flags = fcntl(target_sd, F_GETFL, 0)) < 0) {
opal_output(0, "rmcast:init: fcntl(F_GETFL) failed: %s (%d)",
strerror(opal_socket_errno), opal_socket_errno);
return ORTE_ERROR;
} else {
flags |= O_NONBLOCK;
if(fcntl(target_sd, F_SETFL, flags) < 0) {
opal_output(0, "rmcast:init: fcntl(F_SETFL) failed: %s (%d)",
strerror(opal_socket_errno), opal_socket_errno);
return ORTE_ERROR;
}
}
/* return the socket */
*sd = target_sd;
return ORTE_SUCCESS;
}
static int open_channel(orte_rmcast_channel_t channel, char *name,
char *network, int port, char *interface, uint8_t direction)
{
opal_list_item_t *item;
rmcast_base_channel_t *nchan, *chan;
uint32_t netaddr=0, netmask=0, intr=0;
int rc;
unsigned int i, n, start, end, range;
bool port_assigned;
OPAL_OUTPUT_VERBOSE((2, orte_rmcast_base.rmcast_output,
"%s opening channel %d for %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), channel, name));
/* parse the network, if provided */
if (NULL != network) {
if (ORTE_SUCCESS != (rc = opal_iftupletoaddr(network, &netaddr, &netmask))) {
orte_show_help("help-rmcast-base.txt", "invalid-net-mask", true, network, ORTE_ERROR_NAME(rc));
return ORTE_ERR_SILENT;
}
}
/* parse the interface, if provided */
if (NULL != interface) {
if (ORTE_SUCCESS != (rc = opal_iftupletoaddr(interface, &intr, NULL))) {
orte_show_help("help-rmcast-base.txt", "invalid-net-mask", true, interface, ORTE_ERROR_NAME(rc));
return ORTE_ERR_SILENT;
}
}
/* see if this name has already been assigned a channel on the specified network */
OPAL_OUTPUT_VERBOSE((7, orte_rmcast_base.rmcast_output,
"%s open_channel: searching for %s:%d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), name, channel));
chan = NULL;
ORTE_ACQUIRE_THREAD(&orte_rmcast_base.main_ctl);
for (item = opal_list_get_first(&orte_rmcast_base.channels);
item != opal_list_get_end(&orte_rmcast_base.channels);
item = opal_list_get_next(item)) {
nchan = (rmcast_base_channel_t*)item;
OPAL_OUTPUT_VERBOSE((7, orte_rmcast_base.rmcast_output,
"%s open_channel: channel %s:%d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
nchan->name, channel));
if (nchan->channel == channel ||
0 == strcasecmp(nchan->name, name)) {
chan = nchan;
break;
}
}
if (NULL != chan) {
/* already exists - check that the requested
* sockets are setup
*/
OPAL_OUTPUT_VERBOSE((2, orte_rmcast_base.rmcast_output,
"%s rmcast:udp using existing channel %s:%d network %03d.%03d.%03d.%03d port %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
chan->name, chan->channel,
OPAL_IF_FORMAT_ADDR(chan->network),
(int)chan->port));
if (ORTE_SUCCESS != (rc = setup_channel(chan, direction))) {
ORTE_ERROR_LOG(rc);
ORTE_RELEASE_THREAD(&orte_rmcast_base.main_ctl);
return rc;
}
ORTE_RELEASE_THREAD(&orte_rmcast_base.main_ctl);
return ORTE_SUCCESS;
}
/* we didn't find an existing match, so create a new channel */
OPAL_OUTPUT_VERBOSE((2, orte_rmcast_base.rmcast_output,
"%s creating new channel %s for %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
orte_rmcast_base_print_channel(channel), name));
chan = OBJ_NEW(rmcast_base_channel_t);
chan->name = strdup(name);
chan->channel = channel;
/* if we were not given a network, use the default */
if (NULL == network) {
chan->network = orte_rmcast_base.xmit_network;
} else {
chan->network = netaddr;
}
/* if we were not given an interface, use the default */
if (NULL == interface) {
chan->interface = orte_rmcast_base.interface;
} else {
chan->interface = intr;
}
/* if we were not given a port, use a default one */
if (port < 0) {
/* cycle thru the port ranges until we find the
* port corresponding to this channel number
*/
n=0;
port_assigned = false;
for (i=0; NULL != orte_rmcast_base.ports.start[i]; i++) {
/* how many ports are in this range? */
start = strtol(orte_rmcast_base.ports.start[i], NULL, 10);
end = strtol(orte_rmcast_base.ports.end[i], NULL, 10);
range = end - start + 1;
if (chan->channel < (n + range)) {
/* take the corresponding port */
chan->port = start + (chan->channel - n);
port_assigned = true;
break;
}
n += range;
}
if (!port_assigned) {
opal_output(0, "%s CANNOT ASSIGN PORT TO CHANNEL %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
orte_rmcast_base_print_channel(chan->channel));
return ORTE_ERROR;
}
} else {
chan->port = port;
}
opal_list_append(&orte_rmcast_base.channels, &chan->item);
ORTE_RELEASE_THREAD(&orte_rmcast_base.main_ctl);
/* if this is my input, set that value */
if (ORTE_RMCAST_MY_INPUT & direction) {
orte_rmcast_base.my_input_channel = chan;
}
/* if this is my output, set that value */
if (ORTE_RMCAST_MY_OUTPUT & direction) {
orte_rmcast_base.my_output_channel = chan;
}
OPAL_OUTPUT_VERBOSE((2, orte_rmcast_base.rmcast_output,
"%s rmcast:udp opening new channel %s:%s network %03d.%03d.%03d.%03d port %d for%s%s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
chan->name, orte_rmcast_base_print_channel(chan->channel),
OPAL_IF_FORMAT_ADDR(chan->network),
(int)chan->port,
(ORTE_RMCAST_RECV & direction) ? " RECV" : " ",
(ORTE_RMCAST_XMIT & direction) ? " XMIT" : " "));
if (ORTE_SUCCESS != (rc = setup_channel(chan, direction))) {
ORTE_ERROR_LOG(rc);
return rc;
}
return ORTE_SUCCESS;
}
int main(int argc, char* argv[])
{
int rc, idx;
uint32_t addr, netmask, netaddr;
struct sockaddr_in inaddr, *paddr;
char **aliases=NULL;
opal_if_t *intf;
if (0 > (rc = opal_init(&argc, &argv))) {
fprintf(stderr, "opal_interface: couldn't init opal - error code %d\n", rc);
return rc;
}
if (OPAL_SUCCESS != mca_base_framework_open(&opal_if_base_framework, 0)) {
fprintf(stderr, "opal_interface: couldn't get interfaces\n");
return 1;
}
for (intf = (opal_if_t*)opal_list_get_first(&opal_if_list);
intf != (opal_if_t*)opal_list_get_end(&opal_if_list);
intf = (opal_if_t*)opal_list_get_next(intf)) {
paddr = (struct sockaddr_in*) &intf->if_addr;
fprintf(stderr, "intf: %s AF: %s indx: %d kindx: %d\n", intf->if_name,
(AF_INET == paddr->sin_family) ? "v4" : "v6",
intf->if_index, intf->if_kernel_index);
}
if (2 == argc) {
rc = opal_iftupletoaddr(argv[1], &netaddr, &netmask);
fprintf(stderr, "netaddr %03d.%03d.%03d.%03d netmask %03d.%03d.%03d.%03d rc %d\n",
OPAL_IF_FORMAT_ADDR(netaddr), OPAL_IF_FORMAT_ADDR(netmask), rc);
/* search for a matching interface - take the first one within the returned scope */
idx = opal_ifbegin();
while (0 < idx) {
/* ignore the loopback interface */
if (opal_ifisloopback(idx)) {
fprintf(stderr, "LOOPBACK IGNORED\n");
idx = opal_ifnext(idx);
continue;
}
if (0 != (rc = opal_ifindextoaddr(idx, (struct sockaddr*)&inaddr, sizeof(inaddr)))) {
break;
}
addr = ntohl(inaddr.sin_addr.s_addr);
fprintf(stderr, "checking netaddr %03d.%03d.%03d.%03d addr %03d.%03d.%03d.%03d netmask %03d.%03d.%03d.%03d rc %d\n",
OPAL_IF_FORMAT_ADDR(netaddr), OPAL_IF_FORMAT_ADDR(addr), OPAL_IF_FORMAT_ADDR(netmask), rc);
if (netaddr == (addr & netmask)) {
fprintf(stderr, "MATCH FOUND\n");
}
idx = opal_ifnext(idx);
}
}
/* check the aliases */
opal_ifgetaliases(&aliases);
idx = 0;
while (NULL != aliases[idx]) {
fprintf(stderr, "alias: %s\n", aliases[idx]);
idx++;
}
opal_finalize();
return 0;
}
