bool FragmentationEncoderService::addAttributes(DataObjectRef originalDataObject, DataObjectRef fragmentDataObject,
        string sequenceNumberListCsv) {

    //copy attributes. though eventually will use rich metadata?
    const Attributes* originalAttributes = originalDataObject->getAttributes();
    for (Attributes::const_iterator it = originalAttributes->begin(); it != originalAttributes->end(); it++) {
        const Attribute attr = (*it).second;
        bool addAttribute = fragmentDataObject->addAttribute(attr);
        if (!addAttribute) {
            HAGGLE_ERR("unable to add attribute\n");
            return false;
        }
    }

    //add sequence number attribute
//	char sequenceBuffer[33];
//	memset(sequenceBuffer, 0, sizeof(sequenceBuffer));
//	sprintf(sequenceBuffer, "%d", sequenceNumber);
//	HAGGLE_DBG("stringSequenceNumber=%s\n", sequenceBuffer);
//	bool addedSequenceNUmber = fragmentDataObject->addAttribute(
//			HAGGLE_ATTR_FRAGMENTATION_SEQUENCE_NUMBER, sequenceBuffer, 0);
//	if (!addedSequenceNUmber) {
//		HAGGLE_ERR("unable to add addedSequenceNUmber attribute\n");
//		return false;
//	}

    HAGGLE_DBG2("stringSequenceNumber=%s\n", sequenceNumberListCsv.c_str());
    bool addedSequenceNumber = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_SEQUENCE_NUMBER,
            sequenceNumberListCsv, 0);
    if (!addedSequenceNumber) {
        HAGGLE_ERR("Unable to add sequence number attribute\n");
        return false;
    }

    //add attribute to indicate data object is fragmentation block
    bool addedIsFragmentationCodedAttribute = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_NAME, "TRUE",
            0);
    if (!addedIsFragmentationCodedAttribute) {
        HAGGLE_ERR("Unable to add fragmentation attribute\n");
        return false;
    }

    //add original data len attribute
    char lenBuffer[33];
    memset(lenBuffer, 0, sizeof(lenBuffer));
    int len = fragmentationDataObjectUtility->getFileLength(originalDataObject);
    if(len == 0) {
        HAGGLE_ERR("Orignal data len is zero - file already deleted\n");
        return false;
    }
    sprintf(lenBuffer, "%d", len);
    bool addedDataLenAttribute = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_ORIG_LEN, lenBuffer,
            0);
    if (!addedDataLenAttribute) {
        HAGGLE_ERR("Unable to add original data len attribute\n");
        return false;
    }

    //add dataobject id
    const char* originalId = originalDataObject->getIdStr();
    string originalStringId = originalId;
    bool addedIdAttribute = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_DATAOBJECT_ID,
            originalStringId, 0);
    if (!addedIdAttribute) {
        HAGGLE_ERR("Unable to add original data object id attribute\n");
        return false;
    }

    //add dataobject name
    string originalName = fragmentationDataObjectUtility->getFileName(originalDataObject);
    HAGGLE_DBG2("Add original name %s as attribute\n", originalName.c_str());
    bool addedNameAttribute = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_ORIG_NAME, originalName,
            0);
    if (!addedNameAttribute) {
        HAGGLE_ERR("Unable to add original name attribute\n");
        return false;
    }

    //add create time
    string originalCreateTime = originalDataObject->getCreateTime().getAsString();
    HAGGLE_DBG2("Add original create time %s as attribute\n", originalCreateTime.c_str());
    bool addedCreatedTimeAttribute = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_CREATION_TIME,
            originalCreateTime, 0);
    if (!addedCreatedTimeAttribute) {
        HAGGLE_ERR("Unable to add original create time attribute\n");
        return false;
    }

    //set create time of fragment to same create time as parent so fragment data object ids can match up
    Timeval createTime(originalCreateTime);
    fragmentDataObject->setCreateTime(createTime);

    if(originalDataObject->getSignature()) { // MOS
      //add signee
      string parentSignee = originalDataObject->getSignee();
      HAGGLE_DBG2("Add original signee %s as attribute\n",parentSignee.c_str());
      bool addedSigneeAttribute = fragmentDataObject->
	addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_ORIG_SIGNEE,parentSignee,0);
      if(!addedSigneeAttribute) {
        HAGGLE_ERR("Unable to add original signee attribute\n");
        return false;
      }
      
      //add signature
      char *base64_signature = NULL;
      if (base64_encode_alloc((char *)originalDataObject->getSignature(), originalDataObject->getSignatureLength(), &base64_signature) <= 0) {
        HAGGLE_ERR("Unable to generate base64 encoded signature\n");
        return false;
      }
      string parentSignature = base64_signature;
      HAGGLE_DBG2("Add original signature %s as attribute\n",parentSignature.c_str());
      bool addedSignatureAttribute = fragmentDataObject->
	addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_ORIG_SIGNATURE,parentSignature,0);
      if(!addedSignatureAttribute) {
        HAGGLE_ERR("Unable to add original signature attribute\n");
        return false;
      }
      if(base64_signature) {
        free(base64_signature);
        base64_signature = NULL;
      }
    }

    return true;
}
Exemple #2
0
ProtocolEvent ProtocolUDP::receiveDataObject()
{
	size_t len = 0;
	string haggleTag = "</Haggle>";
	DataObjectRef dObj;
        char buf[SOCKADDR_SIZE];
        struct sockaddr *peer_addr = (struct sockaddr *)buf;
	ProtocolEvent pEvent;
        unsigned short port;
        Address *addr = NULL;
	struct sockaddr_in *sa = NULL;

#ifdef OS_WINDOWS
	pEvent = receiveData(buffer, bufferSize, peer_addr, 0, &len);
#else
	pEvent = receiveData(buffer, bufferSize, peer_addr, MSG_DONTWAIT, &len);
#endif

	if (pEvent != PROT_EVENT_SUCCESS)
		return pEvent;

        if (peer_addr->sa_family == AF_INET) {
                sa = (struct sockaddr_in *)peer_addr;
                port = ntohs(sa->sin_port);
                addr = new IPv4Address(sa->sin_addr, TransportUDP(port));
	}
#if defined(ENABLE_IPv6) 
        else if (peer_addr->sa_family == AF_INET6) {
                struct sockaddr_in6 *sa6 = (struct sockaddr_in6 *)peer_addr;
                port = ntohs(sa6->sin6_port);
                addr = new IPv6Address(sa6->sin6_addr, TransportUDP(port));
        }
#endif

        if (addr == NULL)
                return PROT_EVENT_ERROR;

	if (peerIface) {
		HAGGLE_ERR("%s UDP peer interface was not null\n", getName());          
                delete addr;
		return PROT_EVENT_ERROR;
	}

        peerIface = new ApplicationPortInterface(port, "Application", addr, IFFLAG_UP);
        peerNode = getKernel()->getNodeStore()->retrieve(peerIface);

	delete addr;

	if (!peerNode) {
		peerNode = Node::create(Node::TYPE_APPLICATION, "Unknown application");

		if (!peerNode) {      
			HAGGLE_ERR("%s Could not create application node\n", getName());
			return PROT_EVENT_ERROR;
		}
	}

	dObj = DataObject::create(buffer, len, localIface, peerIface);

	if (!dObj) {
                HAGGLE_DBG("%s Could not create data object\n", getName());
		return PROT_EVENT_ERROR;
	}

        // Haggle doesn't own files that applications have put in: // MOS - not true - maybe some code missing here

	Timeval ct = dObj->getCreateTime();
	Timeval now = Timeval::now();
	if(!ct.isValid()) dObj->setCreateTime(now); // MOS - in case it was not set by application
	dObj->setReceiveTime(now);

	dataObjectsIncoming += 1; // MOS
	dataObjectBytesIncoming += len; // MOS
	if(!dObj->isControlMessage()) dataObjectsIncomingNonControl += 1; // MOS

        // We must release the peer interface reference after
        // the data object is created as the next incoming
        // data might be from another peer
        peerIface = NULL;

	// MOS - no need to ignore objects from local application (avoid risk of blocking control messages)
	// if (getKernel()->getThisNode()->getBloomfilter()->has(dObj)) {
	//	HAGGLE_DBG("Data object [%s] from interface %s:%u has already been received, ignoring.\n", 
	//		dObj->getIdStr(), sa ? ip_to_str(sa->sin_addr) : "undefined", port);
	//	return PROT_EVENT_SUCCESS;
	// }

	// Generate first an incoming event to conform with the base Protocol class
	getKernel()->addEvent(new Event(EVENT_TYPE_DATAOBJECT_INCOMING, dObj, peerNode));
	
	dataObjectsReceived += 1; // MOS
	dataObjectBytesReceived += len; // MOS

	HAGGLE_DBG("%s Received data object [%s] from interface %s:%u\n", getName(), 
		dObj->getIdStr(), sa ? ip_to_str(sa->sin_addr) : "undefined", port);

	// Since there is no data following, we generate the received event immediately 
	// following the incoming one
	getKernel()->addEvent(new Event(EVENT_TYPE_DATAOBJECT_RECEIVED, dObj, peerNode));

	peerNode = NULL; // MOS - similar to peerIface

	return PROT_EVENT_SUCCESS;
}
/*
 * Callback when a new data object is inserted. Updates the utility strategy
 * state and notifies the utility functions.
 */
void
CacheStrategyUtility::_handleNewDataObject(
    DataObjectRef &dObj)
{
    if (!isResponsibleForDataObject(dObj)) {
        //HAGGLE_DBG("Ignoring data object, in-eligible for caching\n");
        return;
    }

    if (stats_replacement_strat && stats_replacement_strat->isResponsibleForDataObject(dObj)) {
        stats_replacement_strat->handleNewDataObject(dObj);
        return;
    }

    string id = string(dObj->getIdStr());

    if (utilMetadata.find(id) != utilMetadata.end()) {
        // the DO is already in the cache!
        HAGGLE_DBG("Received data object already in the cache: %s\n", id.c_str());
        current_dupe_do_recv++;
        return;
    }

    getUtilityFunction()->notifyInsertion(dObj);

    int cost = dObj->getOrigDataLen();
    // NOTE: we handle zero sized data objects by "faking"
    // a size of 1.
    if (cost == 0) {
        cost = 1; 
    }

    string strResults="";
    if (Trace::trace.getTraceType() == TRACE_TYPE_DEBUG2) {
        strResults.append(id);
        strResults.append("[I]=");
    }
    double utiltiy = getUtilityFunction()->compute(id, strResults);
    HAGGLE_DBG2("%s --> %f\n", strResults.c_str(), utiltiy);
    Timeval now = Timeval::now();

    bool purgeNow = false;
    if (utiltiy < getGlobalOptimizer()->getMinimumThreshold()) {
        HAGGLE_DBG("Minimum threshold for incoming data object %s is insufficient: %f < %f\n", id.c_str(), utiltiy, getGlobalOptimizer()->getMinimumThreshold());
        current_drop_on_insert++;
        purgeNow = true;
    }

    if (!purgeOnInsert && ((current_size + cost) > max_capacity_kb*1024))  {
        HAGGLE_DBG("Cache is full and purge on insert is disabled, evicting new data object!\n");
        purgeNow = true;
        total_do_hard_evicted++;
        total_do_hard_evicted_bytes += cost;
    }

    if (purgeNow) {
        // we need to properly remove from bloomfilter even when capacity is violated
        getUtilityFunction()->notifyDelete(dObj);

        // CBMEN, HL - Begin
        // Remove any pending send events for this data object
        HAGGLE_STAT("purging send events for dObj %s\n", dObj->getIdStr());
        getManager()->getKernel()->cancelEvents(EVENT_TYPE_DATAOBJECT_SEND, dObj);
        // CBMEN, HL, End

        // delayed delete (although never inserted)
        if (!keep_in_bloomfilter) {
            int delay = (bloomfilter_remove_delay_ms <= 0) ? 1000 : bloomfilter_remove_delay_ms;
            DataObjectId_t *heapId = (DataObjectId_t *)malloc(sizeof(DataObjectId_t));
            DataObject::idStrToId(id, *heapId);
            getManager()->getKernel()->addEvent(new Event(bloomfilterRemoveDelayEventType, heapId, delay/(double)1000));
        }

        return;
    }

    DataObjectUtilityMetadata *do_metadata = new DataObjectUtilityMetadata(
        dObj,
        id,
        cost,
        utiltiy,
        now,
        dObj->getCreateTime());

    if (purgeOnInsert) {
        do_metadata->setEnableDeletion(false);
    }

    if (false == utilMetadata.insert(make_pair(id, do_metadata)).second) {
        HAGGLE_ERR("Somehow data object already in cache\n"); 
        delete do_metadata;
        return;
    }

    current_size += cost;
    current_num_do++;
    total_do_inserted++;
    total_do_inserted_bytes += cost;

    if (!purgeOnInsert) {
        getManager()->insertDataObjectIntoDataStore(dObj);
        return;
    }

    bool was_deleted = false;
    _purgeCache(id, &was_deleted);

    if (was_deleted) {
        HAGGLE_DBG("Purged incoming data object %s on insert.\n", id.c_str());
        current_drop_on_insert++;
        return;
    }

    // DO still in cache, mark to allow deletion in future
    do_metadata->setEnableDeletion(true);
    getManager()->insertDataObjectIntoDataStore(dObj);
}