bool FragmentationEncoderService::addAttributes(DataObjectRef originalDataObject, DataObjectRef fragmentDataObject,
string sequenceNumberListCsv) {
//copy attributes. though eventually will use rich metadata?
const Attributes* originalAttributes = originalDataObject->getAttributes();
for (Attributes::const_iterator it = originalAttributes->begin(); it != originalAttributes->end(); it++) {
const Attribute attr = (*it).second;
bool addAttribute = fragmentDataObject->addAttribute(attr);
if (!addAttribute) {
HAGGLE_ERR("unable to add attribute\n");
return false;
}
}
//add sequence number attribute
// char sequenceBuffer[33];
// memset(sequenceBuffer, 0, sizeof(sequenceBuffer));
// sprintf(sequenceBuffer, "%d", sequenceNumber);
// HAGGLE_DBG("stringSequenceNumber=%s\n", sequenceBuffer);
// bool addedSequenceNUmber = fragmentDataObject->addAttribute(
// HAGGLE_ATTR_FRAGMENTATION_SEQUENCE_NUMBER, sequenceBuffer, 0);
// if (!addedSequenceNUmber) {
// HAGGLE_ERR("unable to add addedSequenceNUmber attribute\n");
// return false;
// }
HAGGLE_DBG2("stringSequenceNumber=%s\n", sequenceNumberListCsv.c_str());
bool addedSequenceNumber = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_SEQUENCE_NUMBER,
sequenceNumberListCsv, 0);
if (!addedSequenceNumber) {
HAGGLE_ERR("Unable to add sequence number attribute\n");
return false;
}
//add attribute to indicate data object is fragmentation block
bool addedIsFragmentationCodedAttribute = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_NAME, "TRUE",
0);
if (!addedIsFragmentationCodedAttribute) {
HAGGLE_ERR("Unable to add fragmentation attribute\n");
return false;
}
//add original data len attribute
char lenBuffer[33];
memset(lenBuffer, 0, sizeof(lenBuffer));
int len = fragmentationDataObjectUtility->getFileLength(originalDataObject);
if(len == 0) {
HAGGLE_ERR("Orignal data len is zero - file already deleted\n");
return false;
}
sprintf(lenBuffer, "%d", len);
bool addedDataLenAttribute = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_ORIG_LEN, lenBuffer,
0);
if (!addedDataLenAttribute) {
HAGGLE_ERR("Unable to add original data len attribute\n");
return false;
}
//add dataobject id
const char* originalId = originalDataObject->getIdStr();
string originalStringId = originalId;
bool addedIdAttribute = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_DATAOBJECT_ID,
originalStringId, 0);
if (!addedIdAttribute) {
HAGGLE_ERR("Unable to add original data object id attribute\n");
return false;
}
//add dataobject name
string originalName = fragmentationDataObjectUtility->getFileName(originalDataObject);
HAGGLE_DBG2("Add original name %s as attribute\n", originalName.c_str());
bool addedNameAttribute = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_ORIG_NAME, originalName,
0);
if (!addedNameAttribute) {
HAGGLE_ERR("Unable to add original name attribute\n");
return false;
}
//add create time
string originalCreateTime = originalDataObject->getCreateTime().getAsString();
HAGGLE_DBG2("Add original create time %s as attribute\n", originalCreateTime.c_str());
bool addedCreatedTimeAttribute = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_CREATION_TIME,
originalCreateTime, 0);
if (!addedCreatedTimeAttribute) {
HAGGLE_ERR("Unable to add original create time attribute\n");
return false;
}
//set create time of fragment to same create time as parent so fragment data object ids can match up
Timeval createTime(originalCreateTime);
fragmentDataObject->setCreateTime(createTime);
if(originalDataObject->getSignature()) { // MOS
//add signee
string parentSignee = originalDataObject->getSignee();
HAGGLE_DBG2("Add original signee %s as attribute\n",parentSignee.c_str());
bool addedSigneeAttribute = fragmentDataObject->
addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_ORIG_SIGNEE,parentSignee,0);
if(!addedSigneeAttribute) {
HAGGLE_ERR("Unable to add original signee attribute\n");
return false;
}
//add signature
char *base64_signature = NULL;
if (base64_encode_alloc((char *)originalDataObject->getSignature(), originalDataObject->getSignatureLength(), &base64_signature) <= 0) {
HAGGLE_ERR("Unable to generate base64 encoded signature\n");
return false;
}
string parentSignature = base64_signature;
HAGGLE_DBG2("Add original signature %s as attribute\n",parentSignature.c_str());
bool addedSignatureAttribute = fragmentDataObject->
addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_ORIG_SIGNATURE,parentSignature,0);
if(!addedSignatureAttribute) {
HAGGLE_ERR("Unable to add original signature attribute\n");
return false;
}
if(base64_signature) {
free(base64_signature);
base64_signature = NULL;
}
}
return true;
}
ProtocolEvent ProtocolUDP::receiveDataObject()
{
size_t len = 0;
string haggleTag = "</Haggle>";
DataObjectRef dObj;
char buf[SOCKADDR_SIZE];
struct sockaddr *peer_addr = (struct sockaddr *)buf;
ProtocolEvent pEvent;
unsigned short port;
Address *addr = NULL;
struct sockaddr_in *sa = NULL;
#ifdef OS_WINDOWS
pEvent = receiveData(buffer, bufferSize, peer_addr, 0, &len);
#else
pEvent = receiveData(buffer, bufferSize, peer_addr, MSG_DONTWAIT, &len);
#endif
if (pEvent != PROT_EVENT_SUCCESS)
return pEvent;
if (peer_addr->sa_family == AF_INET) {
sa = (struct sockaddr_in *)peer_addr;
port = ntohs(sa->sin_port);
addr = new IPv4Address(sa->sin_addr, TransportUDP(port));
}
#if defined(ENABLE_IPv6)
else if (peer_addr->sa_family == AF_INET6) {
struct sockaddr_in6 *sa6 = (struct sockaddr_in6 *)peer_addr;
port = ntohs(sa6->sin6_port);
addr = new IPv6Address(sa6->sin6_addr, TransportUDP(port));
}
#endif
if (addr == NULL)
return PROT_EVENT_ERROR;
if (peerIface) {
HAGGLE_ERR("%s UDP peer interface was not null\n", getName());
delete addr;
return PROT_EVENT_ERROR;
}
peerIface = new ApplicationPortInterface(port, "Application", addr, IFFLAG_UP);
peerNode = getKernel()->getNodeStore()->retrieve(peerIface);
delete addr;
if (!peerNode) {
peerNode = Node::create(Node::TYPE_APPLICATION, "Unknown application");
if (!peerNode) {
HAGGLE_ERR("%s Could not create application node\n", getName());
return PROT_EVENT_ERROR;
}
}
dObj = DataObject::create(buffer, len, localIface, peerIface);
if (!dObj) {
HAGGLE_DBG("%s Could not create data object\n", getName());
return PROT_EVENT_ERROR;
}
// Haggle doesn't own files that applications have put in: // MOS - not true - maybe some code missing here
Timeval ct = dObj->getCreateTime();
Timeval now = Timeval::now();
if(!ct.isValid()) dObj->setCreateTime(now); // MOS - in case it was not set by application
dObj->setReceiveTime(now);
dataObjectsIncoming += 1; // MOS
dataObjectBytesIncoming += len; // MOS
if(!dObj->isControlMessage()) dataObjectsIncomingNonControl += 1; // MOS
// We must release the peer interface reference after
// the data object is created as the next incoming
// data might be from another peer
peerIface = NULL;
// MOS - no need to ignore objects from local application (avoid risk of blocking control messages)
// if (getKernel()->getThisNode()->getBloomfilter()->has(dObj)) {
// HAGGLE_DBG("Data object [%s] from interface %s:%u has already been received, ignoring.\n",
// dObj->getIdStr(), sa ? ip_to_str(sa->sin_addr) : "undefined", port);
// return PROT_EVENT_SUCCESS;
// }
// Generate first an incoming event to conform with the base Protocol class
getKernel()->addEvent(new Event(EVENT_TYPE_DATAOBJECT_INCOMING, dObj, peerNode));
dataObjectsReceived += 1; // MOS
dataObjectBytesReceived += len; // MOS
HAGGLE_DBG("%s Received data object [%s] from interface %s:%u\n", getName(),
dObj->getIdStr(), sa ? ip_to_str(sa->sin_addr) : "undefined", port);
// Since there is no data following, we generate the received event immediately
// following the incoming one
getKernel()->addEvent(new Event(EVENT_TYPE_DATAOBJECT_RECEIVED, dObj, peerNode));
peerNode = NULL; // MOS - similar to peerIface
return PROT_EVENT_SUCCESS;
}
/*
* Callback when a new data object is inserted. Updates the utility strategy
* state and notifies the utility functions.
*/
void
CacheStrategyUtility::_handleNewDataObject(
DataObjectRef &dObj)
{
if (!isResponsibleForDataObject(dObj)) {
//HAGGLE_DBG("Ignoring data object, in-eligible for caching\n");
return;
}
if (stats_replacement_strat && stats_replacement_strat->isResponsibleForDataObject(dObj)) {
stats_replacement_strat->handleNewDataObject(dObj);
return;
}
string id = string(dObj->getIdStr());
if (utilMetadata.find(id) != utilMetadata.end()) {
// the DO is already in the cache!
HAGGLE_DBG("Received data object already in the cache: %s\n", id.c_str());
current_dupe_do_recv++;
return;
}
getUtilityFunction()->notifyInsertion(dObj);
int cost = dObj->getOrigDataLen();
// NOTE: we handle zero sized data objects by "faking"
// a size of 1.
if (cost == 0) {
cost = 1;
}
string strResults="";
if (Trace::trace.getTraceType() == TRACE_TYPE_DEBUG2) {
strResults.append(id);
strResults.append("[I]=");
}
double utiltiy = getUtilityFunction()->compute(id, strResults);
HAGGLE_DBG2("%s --> %f\n", strResults.c_str(), utiltiy);
Timeval now = Timeval::now();
bool purgeNow = false;
if (utiltiy < getGlobalOptimizer()->getMinimumThreshold()) {
HAGGLE_DBG("Minimum threshold for incoming data object %s is insufficient: %f < %f\n", id.c_str(), utiltiy, getGlobalOptimizer()->getMinimumThreshold());
current_drop_on_insert++;
purgeNow = true;
}
if (!purgeOnInsert && ((current_size + cost) > max_capacity_kb*1024)) {
HAGGLE_DBG("Cache is full and purge on insert is disabled, evicting new data object!\n");
purgeNow = true;
total_do_hard_evicted++;
total_do_hard_evicted_bytes += cost;
}
if (purgeNow) {
// we need to properly remove from bloomfilter even when capacity is violated
getUtilityFunction()->notifyDelete(dObj);
// CBMEN, HL - Begin
// Remove any pending send events for this data object
HAGGLE_STAT("purging send events for dObj %s\n", dObj->getIdStr());
getManager()->getKernel()->cancelEvents(EVENT_TYPE_DATAOBJECT_SEND, dObj);
// CBMEN, HL, End
// delayed delete (although never inserted)
if (!keep_in_bloomfilter) {
int delay = (bloomfilter_remove_delay_ms <= 0) ? 1000 : bloomfilter_remove_delay_ms;
DataObjectId_t *heapId = (DataObjectId_t *)malloc(sizeof(DataObjectId_t));
DataObject::idStrToId(id, *heapId);
getManager()->getKernel()->addEvent(new Event(bloomfilterRemoveDelayEventType, heapId, delay/(double)1000));
}
return;
}
DataObjectUtilityMetadata *do_metadata = new DataObjectUtilityMetadata(
dObj,
id,
cost,
utiltiy,
now,
dObj->getCreateTime());
if (purgeOnInsert) {
do_metadata->setEnableDeletion(false);
}
if (false == utilMetadata.insert(make_pair(id, do_metadata)).second) {
HAGGLE_ERR("Somehow data object already in cache\n");
delete do_metadata;
return;
}
current_size += cost;
current_num_do++;
total_do_inserted++;
total_do_inserted_bytes += cost;
if (!purgeOnInsert) {
getManager()->insertDataObjectIntoDataStore(dObj);
return;
}
bool was_deleted = false;
_purgeCache(id, &was_deleted);
if (was_deleted) {
HAGGLE_DBG("Purged incoming data object %s on insert.\n", id.c_str());
current_drop_on_insert++;
return;
}
// DO still in cache, mark to allow deletion in future
do_metadata->setEnableDeletion(true);
getManager()->insertDataObjectIntoDataStore(dObj);
}