static int hash_presence_callback(const char *hexhash, unsigned int index, int code, void *context)
{
hash_presence_ctx_t *ctx = (hash_presence_ctx_t*)context;
sx_hashfs_t *h = ctx->h;
sx_nodelist_t *nodes;
sx_hash_t hash;
if (code != 200) {
if (code < 0)
WARN("Failed to query hash %.*s: %s", 40, hexhash, sx_hashfs_geterrmsg(h));
if (hex2bin(hexhash, SXI_SHA1_TEXT_LEN, hash.b, sizeof(hash.b))) {
WARN("hex2bin failed on %.*s", 40, hexhash);
return -1;
}
nodes = sx_hashfs_putfile_hashnodes(h, &hash);
if (!nodes) {
WARN("hashnodes failed");
return -1;
}
if(ctx->comma)
CGI_PUTC(',');
else
ctx->comma |= 1;
DEBUG("Requesting from user: #%.*s#", 40, hexhash);
send_qstring_hash(&hash);
CGI_PUTC(':');
/* Although there is no danger in doing so, nodes SHOULD NOT be randomized:
* hdist already does a pretty good job here */
send_nodes(nodes);
sx_nodelist_delete(nodes);
}
send_keepalive();
return 0;
}
static struct RequestData node_request(ServicesData servicesData, void *request) {
NodeRequest *req = request;
struct RequestData data;
switch(req->type) {
case NODESERVICE_TYPE_GIVE:
data = send_nodes(req->type, req->ip4array);
break;
case NODESERVICE_TYPE_TAKE:
data = send_nodes(req->type, req->ip4array);
break;
default:
DERROR("INFO SERVICE: %s\n", " unknown operation");
data.len = -1;
data.buffer = NULL;
break;
}
return data;
}
void fcgi_send_file(void) {
sx_hashfs_file_t filedata;
const sx_hash_t *hash;
sx_nodelist_t *nodes;
sx_hash_t etag;
int comma = 0;
rc_ty s = sx_hashfs_getfile_begin(hashfs, volume, path, get_arg("rev"), &filedata, &etag);
if(s != OK)
quit_errnum(s == ENOENT ? 404 : 500);
if(is_object_fresh(&etag, 'F', filedata.created_at)) {
sx_hashfs_getfile_end(hashfs);
return;
}
CGI_PRINTF("Content-type: application/json\r\n\r\n{\"blockSize\":%d,\"fileSize\":", filedata.block_size);
CGI_PUTLL(filedata.file_size);
CGI_PRINTF(",\"createdAt\":%u,\"fileRevision\":\"%s\",\"fileData\":[", filedata.created_at, filedata.revision);
while((s = sx_hashfs_getfile_block(hashfs, &hash, &nodes)) == OK) {
if(comma)
CGI_PUTC(',');
else
comma |= 1;
CGI_PUTC('{');
send_qstring_hash(hash);
CGI_PUTC(':');
/* Nodes are in NL_PREVNEXT order and MUST NOT be randomized
* (see comments in sx_hashfs_getfile_block) */
send_nodes(nodes);
CGI_PUTC('}');
sx_nodelist_delete(nodes);
}
sx_hashfs_getfile_end(hashfs);
CGI_PUTS("]");
if(s != ITER_NO_MORE)
quit_itererr("Failed to list file blocks", s);
CGI_PUTS("}");
}
// do neighbor discovery check for AOI neighbors
//void
//vast_dc::check_neighbor_discovery ()
void vast_dc::post_processmsg ()
{
//
// new neighbor notification check
//
vector<id_t> new_list; // list of new, unknown nodes
vector<id_t> hello_list; // list of new nodes to connect
vector<id_t> remove_list; // list of nodes to remove from the notified list
map<id_t, Msg_NODE>::iterator it = _notified_nodes.begin ();
// loop through each notified neighbor and see if it's unknown
for (; it != _notified_nodes.end (); ++it)
{
Msg_NODE &newnode = it->second;
Node &node = newnode.node;
// check if it's already known
// TODO: update_node should have time-stamp check
if (is_neighbor (node.id) == true)
{
_NM_known++;
// NOTE: do not update, as an older value might replace what's already
// on record, causing larger drift distance
//update_node (node);
// decrease counter, assuming notification means relevance
if (_count_drop[node.id] > 0)
_count_drop[node.id]--;
// remove this known neighbor from list
remove_list.push_back (node.id);
}
// if the notified node is already outdated, also ignore it
else if ((/*_time*/ _net->get_curr_timestamp () - newnode.node.time) > MAX_DROP_COUNT)
{
remove_list.push_back (node.id);
}
else
{
_voronoi->insert (node.id, node.pos);
new_list.push_back (node.id);
}
}
// loop through newly inserted nodes and check for relevance
int n = new_list.size ();
int i;
for (i=0; i<n; ++i)
{
//
// we will not connect to a new node if:
// 1) the node is irrelevant (outside of AOI)
// 2) connection limit has reached
//
// NOTE that we've added '1' units to the overlap test to avoid roundoff errors
if (_voronoi->is_enclosing (new_list[i]) == true ||
_voronoi->overlaps (new_list[i], _self.pos, (aoi_t)(_self.aoi + _detection_buffer + 1)))
{
/*
if (over_connected () == true)
{
// shrink the AOI to exclude the new node
adjust_aoi (&node.pos);
continue;
}
*/
hello_list.push_back (new_list[i]);
}
}
// clear up voronoi first
for (i=0; i<n; ++i)
_voronoi->remove (new_list[i]);
// send HELLO & EN messages to newly discovered nodes
n = hello_list.size ();
Msg_NODE hello (_self);//, _net->getaddr (_self.id)); // HELLO message to be sent
id_t target;
for (i=0; i<n; ++i)
{
target = hello_list[i];
Msg_NODE &newnode = _notified_nodes[target];
/*
#ifdef DEBUG_DETAIL
char ip[16];
newnode.addr.publicIP.get_string (ip);
printf ("[%d] learn about new node [%d] (%s:%d)\n", (int)_self.id, (int)newnode.node.id, ip, (int)newnode.addr.publicIP.port);
#endif
*/
if (insert_node (newnode.node/*, newnode.addr*/) == true)
{
// send HELLO
_net->sendmsg (target, DC_HELLO, (char *)&hello, sizeof (Msg_NODE), true, true);
// send EN
vector<id_t> &en_list = _voronoi->get_en (target);
send_ID (target, DC_EN, en_list);
}
// remove this node from those being considered
remove_list.push_back (target);
}
// remove notified nodes that are outdated or already known
n = remove_list.size ();
for (i=0; i<n; ++i)
_notified_nodes.erase (remove_list[i]);
// NOTE: simply clear out notified nodes gives lower Topology Consistency
//_notified_nodes.clear ();
//
// neighbor discovery check
//
vector<id_t> notify_list;
id_t from_id;
#ifdef CHECK_REQNODE_ONLY
map<id_t, int>::iterator it2 = _req_nodes.begin ();
for (;it2 != _req_nodes.end (); ++it2)
{
from_id = it2->first;
// a node requesting for new neighbor check
// might have been disconnected by now,
// as we process the request in batch
if (is_neighbor (from_id) == false)
continue;
#else
// we check for all known neighbors except myself
int num = _neighbors.size ();
int idx;
for (idx=1; idx<num; ++idx)
{
from_id = _neighbors[idx]->id;
#endif
// TODO: clear or new would be faster?
notify_list.clear ();
map<id_t, int> *known_list = new map<id_t, int>;
map<id_t, int>::iterator it;
id_t id;
int state, known_state;
#ifdef CHECK_EN_ONLY
// check for only enclosing neighbor
vector<id_t> &en_list = _voronoi->get_en (_self.id);
int n = en_list.size ();
#else
// loop through every known neighbor except myself
int n = _neighbors.size ();
#endif
for (int i=0; i<n; ++i)
{
#ifdef CHECK_EN_ONLY
id = en_list[i];
#else
id = _neighbors[i]->id;
#endif
if (id == _self.id || id == from_id)
continue;
// do a simple test to see if this enclosing neighbor is
// on the right side of me if I face the moving node directly
// only notify the moving node for these 'right-hand-side' neighbors
//if (right_of (id, from_id) == false)
// continue;
state = 0;
known_state = 0;
if (_voronoi->overlaps (id, _id2node[from_id].pos, (aoi_t)(_id2node[from_id].aoi + _detection_buffer + 1)) == true)
state = state | STATE_OVERLAPPED;
if (_voronoi->is_enclosing (id, from_id) == true)
state = state | STATE_ENCLOSED;
// notify case1: new overlap by moving node's AOI
// notify case2: new EN for the moving node
if (state != 0)
{
if ((it = _neighbor_states[from_id]->find (id)) != _neighbor_states[from_id]->end ())
known_state = it->second;
// note: what we want to achieve is:
// 1. notify just once when new EN is found
// 2. notify for new overlap, even if the node is known to be an EN
if ((known_state & STATE_OVERLAPPED) == 0 &&
((state & STATE_OVERLAPPED) > 0 || ((known_state & STATE_ENCLOSED) == 0 && (state & STATE_ENCLOSED) > 0)))
notify_list.push_back (id);
// store the state of this particular neighbor
known_list->insert (map<id_t, int>::value_type (id, state));
}
}
//_neighbor_states[from_id]->clear ();
delete _neighbor_states[from_id];
_neighbor_states[from_id] = known_list;
// notify moving nodes of new neighbors
if (_req_nodes.find (from_id) != _req_nodes.end ())
send_nodes (from_id, notify_list);
}
_req_nodes.clear ();
//=================================================================================================================
// send out all pending reliable messages
_net->flush ();
}
/*
int
vast_dc::recv (id_t target, char *buffer)
{
if (_joined == false)
{
printf ("[%d] attempt to receive before joined successfully\n", _self.id);
return 0;
}
if (_id2msg.find (target) == _id2msg.end ())
return 0;
netmsg *queue = _id2msg[target];
netmsg *msg;
queue = queue->getnext (&msg);
// update the mapping to queue
if (queue == NULL)
_id2msg.erase (target);
else
_id2msg[target] = queue;
// copy the msg into buffer
int size = msg->size;
memcpy (buffer, msg->msg, size);
delete msg;
return size;
}
*/
//
// private methods
//
bool
vast_dc::insert_node (Node &node, Addr *addr)
{
// check for redundency
if (is_neighbor (node.id))
return false;
// avoid self-connection
// note: connection may already exist with remote node, in which case the
// insert_node process will continue (instead of aborting..)
if (node.id != _self.id && !(_net->is_connected (node.id)))
{
int result = (addr != NULL ? _net->connect (*addr) : _net->connect (node.id));
if (result == (-1))
return false;
}
_voronoi->insert (node.id, node.pos);
_id2node[node.id] = node;
_id2vel[node.id] = 0;
_neighbors.push_back (&_id2node[node.id]);
_neighbor_states[node.id] = new map<id_t, int>;
_count_drop[node.id] = 0;
return true;
}
/*
// overloaded version for the node's address is prefetched
bool
vast_dc::insert_node (Node &node, Addr &addr)
{
// check for redundency
if (is_neighbor (node.id))
return false;
if ( node.id != _self.id && !(_net->is_connected (node.id)) )
if (_net->connect (addr) == (-1))
return false;
return _post_insert_node (node);
}
bool
vast_dc::insert_node (Node &node)
{
// check for redundency
if (is_neighbor (node.id))
return false;
// avoid self-connection
// note: connection may already exist with remote node, in which case the
// insert_node process will continue (instead of aborting..)
if ( node.id != _self.id && !(_net->is_connected (node.id)) )
if (_net->connect (node.id) == (-1))
return false;
return _post_insert_node (node);
}
bool
vast_dc::_post_insert_node (Node &node)
{
_voronoi->insert (node.id, node.pos);
_id2node[node.id] = node;
_id2vel[node.id] = 0;
_neighbors.push_back (&_id2node[node.id]);
_neighbor_states[node.id] = new map<id_t, int>;
_count_drop[node.id] = 0;
return true;
}
*/
bool
vast_dc::delete_node (id_t id, bool disconnect)
{
// NOTE: it's possible to remove self or EN, use carefully.
// we don't check for that to allow force-remove in
// the case of disconnection by remote node, or
// when leaving the overlay (removal of self)
if (is_neighbor (id) == false)
return false;
#ifdef DEBUG_DETAIL
printf ("[%lu] disconnecting [%lu]\n", _self.id, id);
#endif
if (disconnect == true)
_net->disconnect (id);
_voronoi->remove (id);
vector<Node *>::iterator it = _neighbors.begin();
for (; it != _neighbors.end (); it++)
{
if ((*it)->id == id)
{
_neighbors.erase (it);
break;
}
}
_id2node.erase (id);
_id2vel.erase (id);
delete _neighbor_states[id];
_neighbor_states.erase (id);
_count_drop.erase (id);
return true;
}
bool
vast_dc::update_node (Node &node)
{
if (is_neighbor (node.id) == false)
return false;
// only update the node if it's at a later time
if (_id2node[node.id].time <= node.time)
{
// calculate displacement from last position
double dis = _id2node[node.id].pos.dist (node.pos);
_voronoi->update (node.id, node.pos);
_id2node[node.id].update (node);
_id2vel[node.id] = dis;
}
return true;
}
// send node infos (NODE message) to a particular node
// 'list' is a list of indexes to _neighbors
void
vast_dc::send_nodes (id_t target, vector<id_t> &list, bool reliable)
{
int n = list.size ();
if (n == 0)
return;
// TODO: sort the notify list by distance from target's center
_buf[0] = (unsigned char)n;
char *p = _buf+1;
#ifdef DEBUG_DETAIL
printf (" aoi: %d ", (aoi_t)(_id2node[target].aoi + _detection_buffer));
#endif
VAST::Msg_NODE node;
for (int i=0; i<n; ++i, p += sizeof (Msg_NODE))
{
node.set (_id2node[list[i]]);//, _net->getaddr (list[i]));
/*
#ifdef DEBUG_DETAIL
char ip[16];
node.addr.publicIP.get_string (ip);
printf ("[%d] (%s:%d) ", (int)list[i], ip, (int)node.addr.publicIP.port);
#endif
*/
memcpy (p, (char *)&node, sizeof (Msg_NODE));
}
#ifdef DEBUG_DETAIL
printf ("\n");
#endif
// check whether to send the NODE via TCP or UDP
_net->sendmsg (target, DC_NODE, _buf, 1 + n * sizeof (Msg_NODE), reliable, true);
}
// process a single message in queue
bool
vast_dc::handlemsg (VAST::id_t from_id, msgtype_t msgtype, timestamp_t recvtime, char *msg, int size)
{
#ifdef DEBUG_DETAIL
printf ("%4d [%3d] processmsg from: %3d type: (%2d)%-12s size:%3d\n", (int)_net->get_curr_timestamp (), (int)_self.id, (int)from_id,
msgtype, (msgtype>=10 && msgtype<=DC_UNKNOWN)?VAST_DC_MESSAGE[msgtype-10]:"UNKNOWN", size);
#endif
// should do like this way?
if (_joined == S_INIT)
{
// if I'm not suppose to join, any VAST message will incur disconnect the node
// protential BUG: any side effects?
if (msgtype >= DC_QUERY && msgtype < DC_UNKNOWN)
{
// if a query message, send it back to prevent other nodes fail to join
if (msgtype == DC_QUERY)
_net->sendmsg (from_id, msgtype, msg, size);
if (_net->is_connected (from_id))
_net->disconnect (from_id);
}
return false;
}
else if (_joined == S_JOINING && msgtype != DC_NODE)
return false;
switch ((VAST_DC_Message)msgtype)
{
case DC_QUERY:
if (size == sizeof (Msg_QUERY))
{
// to prevent gateway server from over-connection
if (_self.id == NET_ID_GATEWAY)
_net->disconnect (from_id);
Msg_QUERY n (msg);
Node &joiner = n.node;
VAST::id_t closest_id = _voronoi->closest_to (joiner.pos);
// forward the request if a more approprite node exists
if (_voronoi->contains (_self.id, joiner.pos) == false &&
closest_id != _self.id &&
closest_id != from_id)
{
_net->sendmsg (closest_id, DC_QUERY, msg, size, true, true);
}
else
{
// I am the acceptor, send back initial neighbor list
vector<VAST::id_t> list;
// insert first so we can properly find joiner's EN
// TODO: what if joiner exceeds connection limit?
// should remove closest to AOI
//
insert_node (joiner, &n.addr);
int size = _neighbors.size ();
for (int i=0; i<size; ++i)
{
if (_neighbors[i]->id == joiner.id)
continue;
// send only relevant neighbors
if (_voronoi->overlaps (_neighbors[i]->id, joiner.pos, joiner.aoi) ||
_voronoi->is_enclosing (_neighbors[i]->id, joiner.id))
list.push_back (_neighbors[i]->id);
}
send_nodes (joiner.id, list, true);
}
}
break;
case DC_HELLO:
if (size == sizeof (Msg_NODE))
{
Msg_NODE newnode(msg);
if (is_neighbor (from_id))
update_node (newnode.node);
else
{
// accept all HELLO request
// we assume over-connections can be taken care
// later by adjust_aoi mechanism
insert_node (newnode.node);//, newnode.addr);
}
}
break;
case DC_EN:
if ((size-1) % sizeof(VAST::id_t) == 0)
{
// ignore EN request if not properly connected
if (is_neighbor (from_id) == false)
break;
int n = (int)msg[0];
char *p = msg+1;
// BUG: potential memory leak for the allocation of map?
map<VAST::id_t, int> *list = new map<VAST::id_t, int>; // list of EN received
vector<VAST::id_t> missing; // list of missing EN ids
vector<VAST::id_t> &en_list = _voronoi->get_en (_self.id); // list of my own EN
int i;
// create a searchable list of EN (BUG, mem leak for insert?)
for (i=0; i<n; ++i, p += sizeof (VAST::id_t))
list->insert (map<VAST::id_t, int>::value_type (*(VAST::id_t *)p, STATE_OVERLAPPED));
// store as initial known list (clear necessary to prevent memory leak?)
// csc 20080305: unnecessary, destructor should do it while deleting
//_neighbor_states[from_id]->clear ();
delete _neighbor_states[from_id];
_neighbor_states[from_id] = list;
int en_size = en_list.size ();
for (i=0; i<en_size; ++i)
{
// send only relevant missing neighbors
if (list->find (en_list[i]) == list->end () &&
en_list[i] != from_id &&
_voronoi->overlaps (en_list[i], _id2node[from_id].pos, _id2node[from_id].aoi))
missing.push_back (en_list[i]);
}
send_nodes (from_id, missing);
}
break;
case DC_MOVE:
case DC_MOVE_B:
if (size == sizeof (Node))
{
Node *node = (Node *)msg;
// if the remote node has just been disconnected,
// then no need to process MOVE message in queue
if (update_node (*node) == false)
break;
// records nodes requesting neighbor discovery check
if (msgtype == DC_MOVE_B)
_req_nodes[from_id] = 1;
//req_nodes.push_back (from_id);
/*
// prevent incorrect judgement when later I become a BN
else
_neighbor_states[from_id]->clear ();
*/
}
break;
case DC_NODE:
if ((size-1) % sizeof (Msg_NODE) == 0)
{
int n = (int)msg[0];
char *p = msg+1;
Msg_NODE newnode; // new node discovered
if (_joined < S_JOINING)
break;
// TODO: find a cleaner way than to reset everytime
_joined = S_JOINED;
// store each node notified, we'll process them later in batch
int i;
bool store;
map<VAST::id_t, Msg_NODE>::iterator it;
for (i=0; i<n; ++i, p += sizeof (Msg_NODE))
{
_NM_total++;
memcpy (&newnode, p, sizeof (Msg_NODE));
store = true;
it = _notified_nodes.find (newnode.node.id);
if (it != _notified_nodes.end ())
{
_NM_known++;
// potential BUG:
// NOTE: we're using time to decide if using a newer one to replace
// an existing notified node, however this might give
// false prefererence if remote nodes' clocks are not
// well-synchronized
if (newnode.node.time < (it->second).node.time)
store = false;
}
if (store)
{
// store the new notified node with my own time
// this is to ensure time-based disposal can work properly
newnode.node.time = _net->get_curr_timestamp (); //_time;
_notified_nodes[newnode.node.id] = newnode;
}
/*
if (_notified_nodes.find (newnode.node.id) != _notified_nodes.end ())
{
_NM_known++;
if (_notified_nodes[newnode.node.id].node.time < newnode.node.time)
_notified_nodes[newnode.node.id] = newnode;
}
else
_notified_nodes[newnode.node.id] = newnode;
*/
// notify network knowledge source of IP address
// TODO: queue all nofities in the same step
vector<VAST::id_t> idlist;
idlist.push_back (newnode.node.id);
_net->notify_id_mapper (from_id, idlist);
}
}
break;
case DC_OVERCAP:
if (size == 0)
{
if(is_neighbor (from_id) == false)
break;
// remote node has exceeded its capacity,
// we need to shrink our own AOI
adjust_aoi (&_id2node[from_id].pos);
}
break;
/*
case DC_PAYLOAD:
{
// create a buffer
netmsg *newnode = new netmsg (from_id, msg, size, msgtype, recvtime, NULL, NULL);
// put the content into a per-neighbor queue
if (_id2msg.find (from_id) == _id2msg.end ())
_id2msg[from_id] = newnode;
else
_id2msg[from_id]->append (newnode, recvtime);
}
break;
*/
case DISCONNECT:
if ((size-1) % sizeof (id_t) == 0)
{
int n = msg[0];
char *p = msg+1;
id_t id;
for (int i=0; i<n; ++i, p+=sizeof (id_t))
{
memcpy (&id, p, sizeof (id_t));
// see if it's a remote node disconnecting me
if (id == _self.id)
delete_node (from_id, false);
/*
// delete this node from list of known neighbors
else
{
map<id_t, int> *list;
map<id_t, int>::iterator it;
if (_neighbor_states.find (from_id) != _neighbor_states.end ())
{
list = _neighbor_states[from_id];
if ((it = list->find (id)) != list->end ())
list->erase (it);
}
}
*/
}
}
// allow other handlers to handle the DISCONNECT message
return false;
default:
#ifdef DEBUG_DETAIL
// throw to additional message handler, if it exists
ERROR_MSG_ID (("unknown message"));
#endif
return false;
}
return true;
}
