static int virtual_switch_probe(void)
{
struct sockaddr_in servaddr, devaddr;
socklen_t slen = sizeof(servaddr);
socklen_t devlen = sizeof(devaddr);
uint8_t sbuf[64], rbuf[64];
int ret;
size_t count = 0;
//hsb_debug("probe virtual switch\n");
/* 1.create udp socket */
int sockfd = open_udp_clientfd();
/* 2.send broadcast packet, 192.168.2.255:19001 */
if (get_broadcast_address(sockfd, &servaddr.sin_addr)) {
close(sockfd);
return -1;
}
servaddr.sin_family = AF_INET;
servaddr.sin_port = htons(VIRTUAL_SWITCH_LISTEN_PORT);
set_broadcast(sockfd, true);
memset(sbuf, 0, sizeof(sbuf));
count = 8;
SET_CMD_FIELD(sbuf, 0, uint16_t, VS_CMD_DEVICE_DISCOVER);
SET_CMD_FIELD(sbuf, 2, uint16_t, count);
sendto(sockfd, sbuf, count, 0, (struct sockaddr *)&servaddr, slen);
/* 3.wait for response in 1 second */
struct timeval tv = { 3, 0 };
do {
ret = recvfrom_timeout(sockfd, rbuf, sizeof(rbuf), (struct sockaddr *)&devaddr, &devlen, &tv);
if (ret < count) {
hsb_critical("probe: get err pkt, len=%d\n", ret);
continue;
}
count = 32;
int cmd = GET_CMD_FIELD(rbuf, 0, uint16_t);
int len = GET_CMD_FIELD(rbuf, 2, uint16_t);
if (cmd != VS_CMD_DEVICE_DISCOVER_RESP || len != count) {
hsb_critical("probe: get err pkt, cmd=%x, len=%d\n", cmd, len);
continue;
}
_register_device(&devaddr.sin_addr, (VS_INFO_T *)(rbuf + 8));
} while (tv.tv_sec > 0 && tv.tv_usec > 0);
close(sockfd);
//hsb_debug("probe virtual switch done\n");
return 0;
}
rstatus_t
dnode_peer_handshake_announcing(void *rmsg)
{
rstatus_t status;
struct ring_msg *msg = rmsg;
struct server_pool *sp = msg->sp;
log_debug(LOG_VVERB, "dyn: handshaking peers");
struct array *peers = &sp->peers;
uint32_t i,nelem;
nelem = array_n(peers);
//we assume one mbuf is enough for now - will enhance with multiple mbufs later
struct mbuf *mbuf = mbuf_get();
if (mbuf == NULL) {
log_debug(LOG_VVERB, "Too bad, not enough memory!");
return DN_ENOMEM;
}
//annoucing myself by sending msg: 'dc$rack$token,started_ts,node_state,node_dns'
mbuf_write_string(mbuf, &sp->dc);
mbuf_write_char(mbuf, '$');
mbuf_write_string(mbuf, &sp->rack);
mbuf_write_char(mbuf, '$');
struct dyn_token *token = (struct dyn_token *) array_get(&sp->tokens, 0);
if (token == NULL) {
log_debug(LOG_VVERB, "Why? This should not be null!");
mbuf_put(mbuf);
return DN_ERROR;
}
mbuf_write_uint32(mbuf, token->mag[0]);
mbuf_write_char(mbuf, ',');
int64_t cur_ts = (int64_t)time(NULL);
mbuf_write_uint64(mbuf, cur_ts);
mbuf_write_char(mbuf, ',');
mbuf_write_uint8(mbuf, sp->ctx->dyn_state);
mbuf_write_char(mbuf, ',');
char *broadcast_addr = get_broadcast_address(sp);
mbuf_write_bytes(mbuf, broadcast_addr, dn_strlen(broadcast_addr));
//for each peer, send a registered msg
for (i = 0; i < nelem; i++) {
struct server *peer = (struct server *) array_get(peers, i);
if (peer->is_local)
continue;
log_debug(LOG_VVERB, "Gossiping to node '%.*s'", peer->name.len, peer->name.data);
struct conn * conn = dnode_peer_conn(peer);
if (conn == NULL) {
//running out of connection due to memory exhaust
log_debug(LOG_DEBUG, "Unable to obtain a connection object");
return DN_ERROR;
}
status = dnode_peer_connect(sp->ctx, peer, conn);
if (status != DN_OK ) {
dnode_peer_close(sp->ctx, conn);
log_debug(LOG_DEBUG, "Error happened in connecting on conn %d", conn->sd);
return DN_ERROR;
}
//conn->
dnode_peer_gossip_forward(sp->ctx, conn, sp->redis, mbuf);
//peer_gossip_forward1(sp->ctx, conn, sp->redis, &data);
}
//free this as nobody else will do
//mbuf_put(mbuf);
return DN_OK;
}
static void *_monitor_thread(void *arg)
{
int fd = open_udp_listenfd(VIRTUAL_SWITCH_BOX_LISTEN_PORT);
fd_set rset;
int ret;
struct timeval tv;
struct sockaddr_in mc_addr, dev_addr;
socklen_t mc_len = sizeof(mc_addr);
socklen_t dev_len = sizeof(dev_addr);
uint8_t sbuf[16], rbuf[16];
int cmd_len;
set_broadcast(fd, true);
get_broadcast_address(fd, &mc_addr.sin_addr);
mc_addr.sin_family = AF_INET;
mc_addr.sin_port = htons(VIRTUAL_SWITCH_LISTEN_PORT);
while (1) {
FD_ZERO(&rset);
FD_SET(fd, &rset);
tv.tv_sec = 2;
tv.tv_usec = 0;
ret = select(fd+1, &rset, NULL, NULL, &tv);
if (ret < 0)
continue;
if (0 == ret) { /* timeout, send keep alive */
_remove_timeout_dev();
memset(sbuf, 0, sizeof(sbuf));
cmd_len = 4;
SET_CMD_FIELD(sbuf, 0, uint16_t, VS_CMD_KEEP_ALIVE);
SET_CMD_FIELD(sbuf, 2, uint16_t, cmd_len);
sendto(fd, sbuf, cmd_len, 0, (struct sockaddr *)&mc_addr, mc_len);
continue;
}
cmd_len = 16;
/* get message */
dev_len = sizeof(struct sockaddr_in);
ret = recvfrom(fd, rbuf, cmd_len, 0, (struct sockaddr *)&dev_addr, &dev_len);
if (ret < 4)
continue;
uint16_t cmd = GET_CMD_FIELD(rbuf, 0, uint16_t);
uint16_t len = GET_CMD_FIELD(rbuf, 2, uint16_t);
if (len != ret) {
hsb_debug("error cmd: %d, %d\n", len, ret);
continue;
}
//hsb_debug("get a cmd: %x\n", cmd);
VS_DEV_T *pdev = _find_dev_by_ip(&dev_addr.sin_addr);
if (!pdev) {
probe_dev(virtual_switch_drv.id);
continue;
}
switch (cmd) {
case VS_CMD_KEEP_ALIVE:
{
break;
}
case VS_CMD_STATUS_CHANGED:
{
uint16_t id = GET_CMD_FIELD(rbuf, 4, uint16_t);
uint16_t val = GET_CMD_FIELD(rbuf, 6, uint16_t);
_status_updated(pdev->id, id, val);
break;
}
case VS_CMD_EVENT:
{
uint16_t id = GET_CMD_FIELD(rbuf, 4, uint16_t);
uint16_t param = GET_CMD_FIELD(rbuf, 6, uint16_t);
uint32_t param2 = GET_CMD_FIELD(rbuf, 8, uint32_t);
_event(pdev->id, id, param, param2);
break;
}
default:
break;
}
_refresh_device(pdev);
}
return NULL;
}
static rstatus_t
gossip_pool_each_init(void *elem, void *data)
{
rstatus_t status;
struct server_pool *sp = elem;
gn_pool.ctx = sp->ctx;
gn_pool.name = &sp->name;
gn_pool.idx = sp->idx;
gn_pool.g_interval = sp->g_interval;
//dictDisableResize();
gn_pool.dict_dc = dictCreate(&string_table_dict_type, NULL);
gossip_set_seeds_provider(&sp->seed_provider);
uint32_t n_dc = array_n(&sp->datacenters);
if (n_dc == 0)
return DN_OK;
if (n_dc > 0) {
status = array_init(&gn_pool.datacenters, n_dc, sizeof(struct gossip_dc));
if (status != DN_OK) {
return status;
}
}
//add racks and datacenters
uint32_t dc_cnt = array_n(&sp->datacenters);
uint32_t dc_index;
for(dc_index = 0; dc_index < dc_cnt; dc_index++) {
struct datacenter *dc = array_get(&sp->datacenters, dc_index);
uint32_t rack_cnt = array_n(&dc->racks);
uint32_t rack_index;
for(rack_index = 0; rack_index < rack_cnt; rack_index++) {
struct rack *rack = array_get(&dc->racks, rack_index);
if (dictFind(gn_pool.dict_dc, rack->dc) == NULL) {
struct gossip_dc *g_dc = array_push(&gn_pool.datacenters);
gossip_dc_init(g_dc, rack->dc);
dictAdd(gn_pool.dict_dc, &g_dc->name, g_dc);
}
struct gossip_dc *g_dc = dictFetchValue(gn_pool.dict_dc, rack->dc);
if (dictFind(g_dc->dict_rack, rack->name) == NULL) {
log_debug(LOG_VERB, "What?? No rack in Dict for rack : '%.*s'", g_dc->name);
struct gossip_rack *g_rack = array_push(&g_dc->racks);
gossip_rack_init(g_rack, rack->dc, rack->name);
dictAdd(g_dc->dict_rack, &g_rack->name, g_rack);
}
}
}
uint32_t i, nelem;
for (i = 0, nelem = array_n(&sp->peers); i < nelem; i++) {
struct server *peer = array_get(&sp->peers, i);
struct gossip_dc *g_dc = dictFetchValue(gn_pool.dict_dc, &peer->dc);
struct gossip_rack *g_rack = dictFetchValue(g_dc->dict_rack, &peer->rack);
struct node *gnode = array_push(&g_rack->nodes);
node_init(gnode);
string_copy(&gnode->dc, peer->dc.data, peer->dc.len);
string_copy(&gnode->rack, g_rack->name.data, g_rack->name.len);
string_copy(&gnode->name, peer->name.data, peer->name.len);
string_copy(&gnode->pname, peer->pname.data, peer->pname.len); //ignore the port for now
gnode->port = peer->port;
gnode->is_local = peer->is_local;
if (i == 0) { //Don't override its own state
gnode->state = sp->ctx->dyn_state; //likely it is JOINING state
gnode->ts = (uint64_t)time(NULL);
current_node = gnode;
char *b_address = get_broadcast_address(sp);
string_deinit(&gnode->name);
string_copy(&gnode->name, b_address, dn_strlen(b_address));
} else {
gnode->state = DOWN;
gnode->ts = 1010101; //make this to be a very aged ts
}
struct dyn_token *ptoken = array_get(&peer->tokens, 0);
copy_dyn_token(ptoken, &gnode->token);
//copy socket stuffs
g_rack->nnodes++;
//add into dicts
dictAdd(g_rack->dict_name_nodes, &gnode->name, gnode);
dictAdd(g_rack->dict_token_nodes, token_to_string(&gnode->token), gnode);
node_count++;
}
//gossip_debug();
status = gossip_start(sp);
if (status != DN_OK) {
goto error;
}
return DN_OK;
error:
gossip_destroy(sp);
return DN_OK;
}
