void DCMessenger::startReceiveMsg( classy_counted_ptr<DCMsg> msg, Sock *sock )
{
// Currently, only one pending message per messenger.
ASSERT( !m_callback_msg.get() );
ASSERT( !m_callback_sock );
ASSERT( m_pending_operation == NOTHING_PENDING );
msg->setMessenger( this );
std::string name;
formatstr(name, "DCMessenger::receiveMsgCallback %s", msg->name());
incRefCount();
int reg_rc = daemonCore->
Register_Socket( sock, peerDescription(),
(SocketHandlercpp)&DCMessenger::receiveMsgCallback,
name.c_str(), this, ALLOW );
if(reg_rc < 0) {
msg->addError(
CEDAR_ERR_REGISTER_SOCK_FAILED,
"failed to register socket (Register_Socket returned %d)",
reg_rc );
msg->callMessageReceiveFailed( this );
doneWithSock(sock);
decRefCount();
return;
}
m_callback_msg = msg; // prevent msg from going out of reference
m_callback_sock = sock;
m_pending_operation = RECEIVE_MSG_PENDING;
}
ProtocolEvent ProtocolUDPGeneric::receiveDataObjectNoControl()
{
ProtocolEvent pEvent;
size_t len;
pEvent = receiveData(buffer, bufferSize, MSG_DONTWAIT, &len);
if (pEvent != PROT_EVENT_SUCCESS) {
return pEvent;
}
buffer[bufferSize-1] = '\0';
if (len == 0) {
HAGGLE_DBG("%s Received zero-length message\n", getName());
return PROT_EVENT_ERROR;
}
if(lastReceivedSessionNo == lastValidReceivedSessionNo && lastReceivedSeqNo == lastValidReceivedSeqNo) {
HAGGLE_DBG("%s Ignoring duplicate message - session no %s sequence no %d\n", getName(), DataObject::idString(lastValidReceivedSessionNo).c_str(), lastReceivedSeqNo);
return PROT_EVENT_SUCCESS;
}
memcpy(lastValidReceivedSessionNo, lastReceivedSessionNo, sizeof(DataObjectId_t));
lastValidReceivedSeqNo = lastReceivedSeqNo;
// MOS - fastpath based on session id = data object id
if (getKernel()->getThisNode()->getBloomfilter()->has(lastValidReceivedSessionNo)) {
HAGGLE_DBG("%s Data object (session no %s) already in bloom filter - no event generated\n", getName(), DataObject::idString(lastValidReceivedSessionNo).c_str());
dataObjectsNotReceived += 1; // MOS
return PROT_EVENT_SUCCESS;
}
// MOS - quickly add to Bloom filter to reduce redundant processing in other procotols
getKernel()->getThisNode()->getBloomfilter()->add(lastValidReceivedSessionNo);
DataObjectRef dObj = DataObject::create_for_putting(localIface,
peerIface,
getKernel()->getStoragePath());
if (!dObj) {
HAGGLE_DBG("%s Could not create data object\n", getName());
return PROT_EVENT_ERROR;
}
size_t bytesRemaining = DATAOBJECT_METADATA_PENDING;
ssize_t bytesPut = dObj->putData(buffer, len, &bytesRemaining, true);
if (bytesPut < 0) {
HAGGLE_ERR("%s Could not put data\n", getName());
return PROT_EVENT_ERROR;
}
if(bytesRemaining != len - bytesPut) {
HAGGLE_ERR("%s Received data object not complete - discarding\n", getName());
return PROT_EVENT_ERROR;
}
HAGGLE_DBG("%s Metadata header received [%s].\n", getName(), dObj->getIdStr());
dObj->setReceiveTime(Timeval::now());
// MOS - the following was happening after posting INCOMING but that distorts the statistics
HAGGLE_DBG("%s %ld bytes data received (including header), %ld bytes put\n", getName(), len, bytesPut);
dataObjectsIncoming += 1; // MOS
if(!dObj->isControlMessage()) dataObjectsIncomingNonControl += 1; // MOS
dataObjectBytesIncoming += len; // MOS
HAGGLE_DBG("%s Received data object [%s] from node %s\n", getName(),
DataObject::idString(dObj).c_str(), peerDescription().c_str());
// MOS - removed interface due to locking issue
if (getKernel()->getThisNode()->getBloomfilter()->hasParentDataObject(dObj)) {
HAGGLE_DBG("%s Data object [%s] already in bloom filter - no event generated\n", getName(), DataObject::idString(dObj).c_str());
dataObjectsNotReceived += 1; // MOS
return PROT_EVENT_SUCCESS;
}
NodeRef node = Node::create(dObj);
if (node && (node == getKernel()->getThisNode())) {
HAGGLE_DBG("%s Received own node description, discarding early.\n", getName());
dataObjectsNotReceived += 1; // MOS
return PROT_EVENT_SUCCESS;
}
// MOS - this now happens even before xml parsing
// getKernel()->getThisNode()->getBloomfilter()->add(dObj);
if(bytesRemaining > 0) {
ssize_t bytesPut2 = dObj->putData(buffer + bytesPut, len - bytesPut, &bytesRemaining, false);
HAGGLE_DBG("%s processing payload - %ld bytes put\n", getName(), bytesPut2);
if (bytesPut2 < 0) {
HAGGLE_ERR("%s Could not put data\n", getName());
return PROT_EVENT_ERROR;
}
if(bytesRemaining != 0) {
HAGGLE_ERR("%s Received data object not complete - discarding\n", getName());
return PROT_EVENT_ERROR;
}
}
NodeRef currentPeer;
{ Mutex::AutoLocker l(mutex); currentPeer = peerNode; } // MOS
// Generate first an incoming event to conform with the base Protocol class
getKernel()->addEvent(new Event(EVENT_TYPE_DATAOBJECT_INCOMING, dObj, currentPeer));
receiveDataObjectSuccessHook(dObj);
// Since there is no data following, we generate the received event immediately
// following the incoming one
getKernel()->addEvent(new Event(EVENT_TYPE_DATAOBJECT_RECEIVED, dObj, currentPeer));
dataObjectsReceived += 1; // MOS
dataObjectBytesReceived += len; // MOS
if(dObj->isNodeDescription()) { nodeDescReceived += 1; nodeDescBytesReceived += len; } // MOS
return PROT_EVENT_SUCCESS;
}
void DCMessenger::startCommand( classy_counted_ptr<DCMsg> msg )
{
MyString error;
msg->setMessenger( this );
if( msg->deliveryStatus() == DCMsg::DELIVERY_CANCELED ) {
msg->callMessageSendFailed( this );
return;
}
time_t deadline = msg->getDeadline();
if( deadline && deadline < time(NULL) ) {
msg->addError(CEDAR_ERR_DEADLINE_EXPIRED,
"deadline for delivery of this message expired");
msg->callMessageSendFailed( this );
return;
}
// For a UDP message, we may need to register two sockets, one for
// the SafeSock and another for a ReliSock to establish the
// security session.
Stream::stream_type st = msg->getStreamType();
if( daemonCore->TooManyRegisteredSockets(-1,&error,st==Stream::safe_sock?2:1) ) {
// Try again in a sec
// Eventually, it would be better to queue this centrally
// (i.e. in DaemonCore) rather than having an independent
// timer for each case. Then it would be possible to control
// priority of different messages etc.
dprintf(D_FULLDEBUG, "Delaying delivery of %s to %s, because %s\n",
msg->name(),peerDescription(),error.Value());
startCommandAfterDelay( 1, msg );
return;
}
// Currently, there may be only one pending operation per messenger.
ASSERT(!m_callback_msg.get());
ASSERT(!m_callback_sock);
ASSERT(m_pending_operation == NOTHING_PENDING);
m_pending_operation = START_COMMAND_PENDING;
m_callback_msg = msg;
m_callback_sock = m_sock.get();
if( !m_callback_sock ) {
if (IsDebugLevel(D_COMMAND)) {
const char * addr = m_daemon->addr();
const int cmd = msg->m_cmd;
dprintf (D_COMMAND, "DCMessenger::startCommand(%s,...) making non-blocking connection to %s\n", getCommandStringSafe(cmd), addr ? addr : "NULL");
}
const bool nonblocking = true;
m_callback_sock = m_daemon->makeConnectedSocket(st,msg->getTimeout(),msg->getDeadline(),&msg->m_errstack,nonblocking);
if( !m_callback_sock ) {
msg->callMessageSendFailed( this );
return;
}
}
incRefCount();
m_daemon->startCommand_nonblocking (
msg->m_cmd,
m_callback_sock,
msg->getTimeout(),
&msg->m_errstack,
&DCMessenger::connectCallback,
this,
msg->name(),
msg->getRawProtocol(),
msg->getSecSessionId());
}
ProtocolEvent ProtocolUDPGeneric::sendDataObjectNowNoControl(
const DataObjectRef& dObj)
{
NodeRef currentPeer;
{ Mutex::AutoLocker l(mutex); currentPeer = peerNode; } // MOS
NodeRef actualPeer = getManager()->getKernel()->getNodeStore()->retrieve(currentPeer, true);
if (!actualPeer) {
HAGGLE_ERR("%s Peer not in node store\n", getName());
return PROT_EVENT_ERROR;
}
// check if already in peers bloomfilter
if (actualPeer->hasThisOrParentDataObject(dObj)) { // MOS
HAGGLE_DBG("%s Peer already had data object.\n", getName());
return PROT_EVENT_SUCCESS;
}
// done
// SW: we move the hook here to minimize race condition where we send
// too many redundant node descriptions
// TODO: we may want to synchronize on the dObj or have some serial
// queue so this is not probabilistic
sendDataObjectNowSuccessHook(dObj);
HAGGLE_DBG("%s Sending data object [%s] to peer \'%s\'\n",
getName(), dObj->getIdStr(), peerDescription().c_str());
DataObjectDataRetrieverRef retriever = dObj->getDataObjectDataRetriever();
if (!retriever || !retriever->isValid()) {
HAGGLE_ERR("%s unable to start reading data\n", getName());
return PROT_EVENT_ERROR;
}
ProtocolEvent pEvent = PROT_EVENT_SUCCESS;
ssize_t len = retriever->retrieve(buffer, bufferSize, false);
if (0 == len) {
HAGGLE_ERR("%s DataObject is empty\n", getName());
return PROT_EVENT_ERROR;
}
if (len < 0) {
HAGGLE_ERR("%s Could not retrieve data from data object\n", getName());
return PROT_EVENT_ERROR;
}
if ((size_t) len == bufferSize) {
HAGGLE_ERR("%s Buffer is too small for message\n", getName());
return PROT_EVENT_ERROR;
}
Timeval t_start;
t_start.setNow();
setSessionNo(dObj->getId()); // MOS
setSeqNo(0); // MOS
size_t bytesSent = 0;
// MOS - simple way to increase udp redundancy
for(int i = 0; i <= getConfiguration()->getRedundancy(); i++) {
pEvent = sendData(buffer, (size_t) len, 0, &bytesSent);
if (bytesSent != (size_t) len) {
pEvent = PROT_EVENT_ERROR;
}
if (pEvent != PROT_EVENT_SUCCESS) {
HAGGLE_ERR("%s Broadcast - Error\n", getName());
return pEvent;
}
}
#ifdef DEBUG
Timeval tx_time = Timeval::now() - t_start;
HAGGLE_DBG("%s Sent %lu bytes data in %.3lf seconds, average speed = %.2lf kB/s\n",
getName(), len, tx_time.getTimeAsSecondsDouble(),
(double)len / (1000*tx_time.getTimeAsSecondsDouble()));
#endif
dataObjectsOutgoing += 1; // MOS
dataObjectsSent += 1; // MOS
if(!dObj->isControlMessage()) dataObjectsOutgoingNonControl += 1; // MOS
dataObjectBytesOutgoing += bytesSent; // MOS
dataObjectBytesSent += len; // MOS
if(dObj->isNodeDescription()) { nodeDescSent += 1; nodeDescBytesSent += len; } // MOS
return PROT_EVENT_SUCCESS;
}