/*
* Publish a data object with utility based caching statistics.
*/
void
CacheStrategyUtility::_publishStatsDataObject()
{
DataObjectRef dObj = DataObject::create();
dObj->addAttribute("CacheStrategyUtility", "stats");
dObj->addAttribute("Timestamp", Timeval::now().getAsString().c_str());
dObj->addAttribute("PublisherID", getManager()->getKernel()->getThisNode()->getIdStr());
dObj->addAttribute("Statistics", _generateStatsString().c_str());
#ifdef DEBUG
dObj->print(NULL); // MOS - NULL means print to debug trace
#endif
getManager()->getKernel()->addEvent(new Event(EVENT_TYPE_DATAOBJECT_RECEIVED, dObj));
}
DataObjectRef Forwarder::createRoutingInformationDataObject()
{
// No need to have a reference in this function because it won't be
// visible outside until this function is done with it.
DataObjectRef dObj = DataObject::create();
if (!dObj)
return NULL;
HAGGLE_DBG2("Creating routing info data object\n");
dObj->setPersistent(false);
dObj->addAttribute("Forwarding", getName());
// Add the metric data to the forwarding section:
Metadata *md = dObj->getMetadata()->addMetadata(getManager()->getName());
md = md->addMetadata(getName());
md->setParameter("node_id", getKernel()->getThisNode()->getIdStr());
if (!addRoutingInformation(dObj, md)) {
// SW: ERR->DBG, this is not fatal, we return false when we don't want
// to add routing data.
// HAGGLE_DBG("Could not add routing information\n");
return NULL;
}
return dObj;
}
void DebugManager::onFindRepositoryKey(Event *e)
{
if (!e || !e->hasData())
return;
DataStoreQueryResult *qr = static_cast < DataStoreQueryResult * >(e->getData());
RepositoryEntryRef re = qr->detachFirstRepositoryEntry();
if (!re) {
// No repository entry: no data object.
DataObjectRef dObj;
// Name the log so that the files are more easily readable on the
// machine that receives them:
char filename[128];
sprintf(filename, "log-%s.txt", kernel->getThisNode()->getIdStr());
// Create data object:
// Empty at first:
dObj = DataObject::create(LogTrace::ltrace.getFile(), filename);
if (!dObj) {
HAGGLE_ERR("Could not create data object\n");
return;
}
// Add log file attribute:
Attribute a("Log file","Trace");
dObj->addAttribute(a);
// Add node id of local node, to make sure that two logs from different
// nodes don't clash:
Attribute b("Node id", kernel->getThisNode()->getIdStr());
dObj->addAttribute(b);
// Insert data object:
kernel->getDataStore()->insertDataObject(dObj);
// Insert a repository entry to show the data object exists:
kernel->getDataStore()->insertRepository(new RepositoryEntry("DebugManager", "has saved log file data object", "yes"));
}
delete qr;
}
/**
* This routine was taken from the benchmark manager, to create Dataobjects in code.
* It is used for the self test. Depending on compile options, benchmark manager
* may not be included, so the relevant code is copied here to be always available.
*
* @see BenchmarkManager::createDataObject
*
*/
DataObjectRef CacheStrategyUtility::createDataObject(unsigned int numAttr)
{
char name[128];
char value[128];
unsigned int r;
unsigned char macaddr[6];
macaddr[0] = (unsigned char) RANDOM_INT(255);
macaddr[1] = (unsigned char) RANDOM_INT(255);
macaddr[2] = (unsigned char) RANDOM_INT(255);
macaddr[3] = (unsigned char) RANDOM_INT(255);
macaddr[4] = (unsigned char) RANDOM_INT(255);
macaddr[5] = (unsigned char) RANDOM_INT(255);
unsigned char macaddr2[6];
macaddr2[0] = (unsigned char) RANDOM_INT(255);
macaddr2[1] = (unsigned char) RANDOM_INT(255);
macaddr2[2] = (unsigned char) RANDOM_INT(255);
macaddr2[3] = (unsigned char) RANDOM_INT(255);
macaddr2[4] = (unsigned char) RANDOM_INT(255);
macaddr2[5] = (unsigned char) RANDOM_INT(255);
EthernetAddress addr(macaddr);
EthernetAddress addr2(macaddr2);
InterfaceRef localIface = Interface::create<EthernetInterface>(macaddr, "eth", addr, 0);
InterfaceRef remoteIface = Interface::create<EthernetInterface>(macaddr2, "eth2", addr2, 0);
DataObjectRef dObj = DataObject::create(NULL, 0, localIface, remoteIface);
for (unsigned int i = 0; i < numAttr; i++) {
int tries = 0;
do {
r = RANDOM_INT(32000);
sprintf(name, "name");
sprintf(value, "value %d", r);
if (tries++ > 10) {
HAGGLE_ERR("WARNING: Cannot generate unique attributes in data object... check attribute pool size!\n");
break;
}
} while (dObj->getAttribute(name, value));
dObj->addAttribute(name, value, 1); //r);
}
return dObj;
}
bool FragmentationEncoderService::addAttributes(DataObjectRef originalDataObject, DataObjectRef fragmentDataObject,
string sequenceNumberListCsv) {
//copy attributes. though eventually will use rich metadata?
const Attributes* originalAttributes = originalDataObject->getAttributes();
for (Attributes::const_iterator it = originalAttributes->begin(); it != originalAttributes->end(); it++) {
const Attribute attr = (*it).second;
bool addAttribute = fragmentDataObject->addAttribute(attr);
if (!addAttribute) {
HAGGLE_ERR("unable to add attribute\n");
return false;
}
}
//add sequence number attribute
// char sequenceBuffer[33];
// memset(sequenceBuffer, 0, sizeof(sequenceBuffer));
// sprintf(sequenceBuffer, "%d", sequenceNumber);
// HAGGLE_DBG("stringSequenceNumber=%s\n", sequenceBuffer);
// bool addedSequenceNUmber = fragmentDataObject->addAttribute(
// HAGGLE_ATTR_FRAGMENTATION_SEQUENCE_NUMBER, sequenceBuffer, 0);
// if (!addedSequenceNUmber) {
// HAGGLE_ERR("unable to add addedSequenceNUmber attribute\n");
// return false;
// }
HAGGLE_DBG2("stringSequenceNumber=%s\n", sequenceNumberListCsv.c_str());
bool addedSequenceNumber = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_SEQUENCE_NUMBER,
sequenceNumberListCsv, 0);
if (!addedSequenceNumber) {
HAGGLE_ERR("Unable to add sequence number attribute\n");
return false;
}
//add attribute to indicate data object is fragmentation block
bool addedIsFragmentationCodedAttribute = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_NAME, "TRUE",
0);
if (!addedIsFragmentationCodedAttribute) {
HAGGLE_ERR("Unable to add fragmentation attribute\n");
return false;
}
//add original data len attribute
char lenBuffer[33];
memset(lenBuffer, 0, sizeof(lenBuffer));
int len = fragmentationDataObjectUtility->getFileLength(originalDataObject);
if(len == 0) {
HAGGLE_ERR("Orignal data len is zero - file already deleted\n");
return false;
}
sprintf(lenBuffer, "%d", len);
bool addedDataLenAttribute = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_ORIG_LEN, lenBuffer,
0);
if (!addedDataLenAttribute) {
HAGGLE_ERR("Unable to add original data len attribute\n");
return false;
}
//add dataobject id
const char* originalId = originalDataObject->getIdStr();
string originalStringId = originalId;
bool addedIdAttribute = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_DATAOBJECT_ID,
originalStringId, 0);
if (!addedIdAttribute) {
HAGGLE_ERR("Unable to add original data object id attribute\n");
return false;
}
//add dataobject name
string originalName = fragmentationDataObjectUtility->getFileName(originalDataObject);
HAGGLE_DBG2("Add original name %s as attribute\n", originalName.c_str());
bool addedNameAttribute = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_ORIG_NAME, originalName,
0);
if (!addedNameAttribute) {
HAGGLE_ERR("Unable to add original name attribute\n");
return false;
}
//add create time
string originalCreateTime = originalDataObject->getCreateTime().getAsString();
HAGGLE_DBG2("Add original create time %s as attribute\n", originalCreateTime.c_str());
bool addedCreatedTimeAttribute = fragmentDataObject->addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_CREATION_TIME,
originalCreateTime, 0);
if (!addedCreatedTimeAttribute) {
HAGGLE_ERR("Unable to add original create time attribute\n");
return false;
}
//set create time of fragment to same create time as parent so fragment data object ids can match up
Timeval createTime(originalCreateTime);
fragmentDataObject->setCreateTime(createTime);
if(originalDataObject->getSignature()) { // MOS
//add signee
string parentSignee = originalDataObject->getSignee();
HAGGLE_DBG2("Add original signee %s as attribute\n",parentSignee.c_str());
bool addedSigneeAttribute = fragmentDataObject->
addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_ORIG_SIGNEE,parentSignee,0);
if(!addedSigneeAttribute) {
HAGGLE_ERR("Unable to add original signee attribute\n");
return false;
}
//add signature
char *base64_signature = NULL;
if (base64_encode_alloc((char *)originalDataObject->getSignature(), originalDataObject->getSignatureLength(), &base64_signature) <= 0) {
HAGGLE_ERR("Unable to generate base64 encoded signature\n");
return false;
}
string parentSignature = base64_signature;
HAGGLE_DBG2("Add original signature %s as attribute\n",parentSignature.c_str());
bool addedSignatureAttribute = fragmentDataObject->
addAttribute(HAGGLE_ATTR_FRAGMENTATION_PARENT_ORIG_SIGNATURE,parentSignature,0);
if(!addedSignatureAttribute) {
HAGGLE_ERR("Unable to add original signature attribute\n");
return false;
}
if(base64_signature) {
free(base64_signature);
base64_signature = NULL;
}
}
return true;
}
/**
* selfTest() depends upon periodic purging. Every call, it will first fill up the database, taking memory snapshots,
* then purge, taking more memory snapshots. It creates a number of random data objects, inserts them into the system,
* and records current memory usage into a file called 'mem.results', for every interation. It does in the following steps:
*
* 1. Add, approximately, 20 DO's per second, done every poll period (e.g. 10 seconds, means add 200 for that interval).
*
* 2. When the number of DO's in the system match the threshold setting, we drop the threshold by the same number
* we increased it in step 1. In this case, we decrease it by 20, allowing the memory threshold purger to do its job.
*
* 3. Repeat step 1.
*
* 4. Repeat step 2.
*
* 5. Reset system functions to original defaults.
*
* This allows the system to approach maximum usage, drop to zero, back to maximum usage, then back to zero.
* The file can be parse into a plot showing how much memory is freed. This test works regardless of using
* in memory database, the improved all memory database, or disk based database.
*
* WARNING: We use mallinfo() to determine the amount of memory used and released, as the system does NOT
* see any memory freed at all, due to the fact dlmallopt(-1) is set in main.cpp. This tells free to not
* return freed memory to the system. Thus, using mallinfo is the only means available to show how much
* memory is actually freed to the application (but not to the system).
*
*
*/
void
CacheStrategyUtility::selfTest()
{
static int init=0;
static float amount_do=0;
static int count=0;
char countStr[50];
static float threshold_backup;
static char *direction;
if (!init) {
init=1;
threshold_backup=db_size_threshold;
amount_do=20.0*pollPeriodMs/1000; //20 per second seems reasonable
if (amount_do > db_size_threshold/10) {
amount_do = db_size_threshold/10;
}
direction="Start";
} // JM: Start DB purging
// JM: Testing code only, to prove it works. Lets do linear testing.
struct mallinfo mi=mallinfo();
//Due to file permission difficulties in android, we'll write it as a log
HAGGLE_DBG("\nThreshold(%s): %lld/%d -- Used bytes: %d, Free bytes: %d, SQL: %lld\n\n", direction, db_size_threshold,current_num_do, mi.uordblks, mi.fordblks, sqlite3_memory_used());
//send db_size_threshold DO's
//init = 1 means send DO's
//init = 2 means we are in purging mode
if ((init == 1) || (init == 3)) {
float upperlimit=current_num_do+amount_do;
if (upperlimit > db_size_threshold) {
upperlimit = db_size_threshold+1;
init++;
}
if (init ==1) {
direction="Up1";
} else if (init == 3) {
direction="Up2";
} else {
direction="StateChangeFromUp";
}
for(int i=current_num_do; i<upperlimit; i++) {
DataObjectRef dObj = createDataObject(2);
dObj->addAttribute("ContentOriginator", "self");
dObj->addAttribute("ContentType", "DelByRelTTL");
dObj->addAttribute("ContentType2", "DelByAbsTTL");
dObj->addAttribute("purge_by_timestamp", "2000000000");
dObj->addAttribute("purge_after_seconds", "2000000000");
char buffer[1025];
snprintf(buffer, 1024, "%llu", (unsigned long long)time(NULL));
sprintf(countStr, "%d", count++);
dObj->addAttribute("ContentCreationTime", buffer);
dObj->addAttribute("count", countStr);
dObj->calcId();
_handleNewDataObject(dObj);
}
} else if ((init == 2) || (init == 4)) { //init==2, reduction
db_size_threshold -= amount_do;
if (db_size_threshold < 0.0) {
init++;
db_size_threshold=threshold_backup;
}
if (init == 2) {
direction="Down1";
} else if (init == 4) {
direction="Down2";
} else {
direction="StateChangeFromDown";
}
} else { //if (init == 5)
//clear seltTest?
self_test = false;
db_size_threshold=threshold_backup;
HAGGLE_DBG("Self Test completed!\n");
//remove any last DO's
//write any STAT information
getKernel()->shutdown();
//return;
}
// JM: End testing
}