void ThreadedOptFossilCollManager::fossilCollect(SimulationObject *object,
const VTime ¤tTime, const unsigned int &threadId) {
unsigned int objId = object->getObjectID()->getSimulationObjectID();
fossilPeriod[objId]++;
if (fossilPeriod[objId] >= 20) {
int intCurTime = currentTime.getApproximateIntTime();
if (intCurTime > activeHistoryLength[objId]) {
int collectTime = intCurTime - activeHistoryLength[objId];
if (collectTime > lastCollectTimes[objId]) {
lastCollectTimes[objId] = collectTime;
cout << "Fossil Collecting " << collectTime << endl;
mySimManager->getStateManagerNew()->fossilCollect(object,
collectTime, threadId);
mySimManager->getOutputManagerNew()->fossilCollect(object,
collectTime, threadId);
mySimManager->getEventSetManagerNew()->fossilCollect(object,
collectTime, threadId);
/*
utils::debug << "Fossil Collecting Obj " << objId
<< " at time " << collectTime << " now at " << intCurTime << endl;
*/
}
}
fossilPeriod[objId] = 0;
}
mySimManager->setCheckpointing(false);
}
void
ThreadedChebyFossilCollManager::sampleRollback(SimulationObject* object, const VTime& rollVTime) {
int rollbackTime = rollVTime.getApproximateIntTime();
int rollbackDistance = object->getSimulationTime().getApproximateIntTime() - rollbackTime;
unsigned int objId = object->getObjectID()->getSimulationObjectID();
int threadId = *((int*) pthread_getspecific(threadKey));
getOfcChebyLock(threadId,mySimManager->getSyncMechanism());
if (numSamples[objId] < maxSamples) {
// Sample the rollback.
total[objId] = total[objId] - samples[objId][sampleIndex[objId]];
samples[objId][sampleIndex[objId]] = rollbackDistance;
total[objId] = total[objId] + rollbackDistance;
sampleIndex[objId]++;
numSamples[objId]++;
// If there are enough samples, then calculate the mean, variance
// and new active history length.
if (numSamples[objId] > minSamples) {
double sampleVariance = 0;
double mean1, mean2;
double variance1, variance2;
mean1 = mean2 = variance1 = variance2 = 0;
double sampleMean = total[objId] / double(numSamples[objId]);
for (int i = 0; i < numSamples[objId]; i++) {
sampleVariance += ((double(samples[objId][i]) - sampleMean) *
(double(samples[objId][i]) - sampleMean));
}
sampleVariance = sampleVariance / numSamples[objId];
mean1 = sampleMean - 1.96*sqrt(sampleVariance/(double)numSamples[objId]);
mean2 = sampleMean + 1.96*sqrt(sampleVariance/(double)numSamples[objId]);
for (int i = 0; i < numSamples[objId]; i++) {
variance1 += ((double(samples[objId][i]) - mean1) *
(double(samples[objId][i]) - mean1));
variance2 += ((double(samples[objId][i]) - mean2) *
(double(samples[objId][i]) - mean2));
}
variance1 = variance1 / numSamples[objId];
variance2 = variance2 / numSamples[objId];
if (variance1 < variance2) {
sampleVariance = variance2;
} else {
sampleVariance = variance1;
}
activeHistoryLength[objId] = sampleMean + errorTerm * sqrt(sampleVariance/(1.0-riskFactor));
debug::debugout << objId << " - NEW LENGTH: " << activeHistoryLength[objId] << endl;
}
}
if (lastCollectTimes[objId] >= 0 && !recovering && rollbackTime <= lastCollectTimes[objId]) {
debug::debugout << mySimManager->getSimulationManagerID()
<< " - Catastrophic Rollback: Last collection time: " << lastCollectTimes[objId]
<< ", Rollback Time: " << rollbackTime << ", Starting Recovery." << endl;
setRecovery(objId, rollbackTime);
}
releaseOfcChebyLock(threadId,mySimManager->getSyncMechanism());
}
void ThreadedTimeWarpMultiSet::rollback(SimulationObject* simObj,
const VTime& rollbackTime, int threadId) {
// Go through the entire processed events queue and put any events with
// a receive time greater than or equal to the rollback time back in the
// unprocessed queue.
unsigned int objId = simObj->getObjectID()->getSimulationObjectID();
this->getProcessedLock(threadId, objId);
vectorIterator[threadId] = processedQueue[objId]->begin();
int tempCount = 0;
if (rollbackTime.getApproximateIntTime() == 0) {
tempCount = processedQueue[objId]->size();
} else {
vectorIterator[threadId] = processedQueue[objId]->begin();
while (vectorIterator[threadId] != processedQueue[objId]->end()
&& (*(vectorIterator[threadId]))->getReceiveTime()
< rollbackTime) {
(vectorIterator[threadId])++;
tempCount++;
}
const unsigned int
eventIdRollback =
mySimulationManager->getStateManagerNew()->getEventIdForRollback(
threadId, objId);
const unsigned int
senderObjectId =
mySimulationManager->getStateManagerNew()->getSenderObjectIdForRollback(
threadId, objId);
const unsigned int
senderObjectSimId =
mySimulationManager->getStateManagerNew()->getSenderObjectSimIdForRollback(
threadId, objId);
// cout << "The saved EventId is --------------------->>>>>>>>>>>> : "
// << eventIdRollback << endl;
// cout << "The First EventId is --------------------->>>>>>>>>>>> : "
// << (*(vectorIterator[threadId]))->getEventId() << endl;
// cout << "The saved SenderObjectId is --------------------->>>>>>>>>>>> : "
// << senderObjectId << endl;
while (vectorIterator[threadId] != processedQueue[objId]->end()) {
EventId tempEventId = (*(vectorIterator[threadId]))->getEventId();
unsigned int
tempSenderObjectId =
(*(vectorIterator[threadId]))->getSender().getSimulationObjectID();
if (tempEventId.getEventNum() != eventIdRollback
|| tempSenderObjectId != senderObjectId) {
/*cout << "Skipping Event.......::::::::::::: "
<< **(vectorIterator[threadId]) << endl;*/
(vectorIterator[threadId])++;
tempCount++;
} else {
/* cout << " Matched EventId ::::" << **(vectorIterator[threadId])
<< endl;*/
break;
}
}
tempCount = processedQueue[objId]->size() - tempCount;
}
debug::debugout << "( " << mySimulationManager->getSimulationManagerID()
<< " ) Object - " << objId << " Rollback returns : " << tempCount
<< " events back to Unprocessed Queue - " << threadId << endl;
unProcessedQueue[objId]->insert(vectorIterator[threadId],
processedQueue[objId]->end());
processedQueue[objId]->erase(vectorIterator[threadId],
processedQueue[objId]->end());
this->releaseProcessedLock(threadId, objId);
// Increment number of rolled back events
//cout << "rollback completed " << tempCount << " events rolled back, LTSF " << LTSFObjId[objId][LTSFOWNER] << endl;
__sync_fetch_and_add(&(rolledBackEventsByObj[objId]), tempCount);
__sync_fetch_and_add(&(rolledBackEventsByLTSF[ LTSFObjId[objId][LTSFOWNER] ]), tempCount);
// Perform calculation to see if a load balance 'action' is necessary
// Load balancing function is performed using the currently running thread
if (lbType) {
myLoadBalancer->balanceCheck();
}
}
void ThreadedOptFossilCollManager::checkpoint(const VTime &checkTime,
const ObjectID &objId, const unsigned int &threadId) {
int time = checkTime.getApproximateIntTime();
int id = objId.getSimulationObjectID();
// If the time is less than the last checkpoint time, then save at the last
// checkpoint time again.
updateCheckpointTime(id, time);
while (time >= nextCheckpointTime[id]) {
utils::debug << mySimManager->getSimulationManagerID()
<< " - Checkpointing object " << id << " at " << time << endl;
int highestNextCheckpointTime = nextCheckpointTime[0];
for (int iter = 1; iter < mySimManager->getNumberOfSimulationObjects(); iter++) {
if (nextCheckpointTime[iter] > highestNextCheckpointTime)
highestNextCheckpointTime = nextCheckpointTime[iter];
}
vector<State *> *states;
if (nextCheckpointTime[id] == highestNextCheckpointTime) {
// No states have been check pointed for this time yet.
// Thus we create a state vector for all the objects
utils::debug << "Creating new states to be saved at time "
<< nextCheckpointTime[id] << endl;
states = new vector<State*> (
mySimManager->getNumberOfSimulationObjects(), NULL);
checkpointedStates.insert(
pair<int, vector<State*> *> (nextCheckpointTime[id], states));
} else {
// States have been saved for other objects but not this object at this time
// or a roll back in this object is causing this to happen.
utils::debug
<< "Adding the current state of the object to checkpointedStates "
<< nextCheckpointTime[id] << endl;
map<int, vector<State*>*>::iterator it = checkpointedStates.find(
nextCheckpointTime[id]);
states = it->second;
}
SimulationObject *object = mySimManager->getObjectHandle(objId);
// Save the state of the object at the checkpoint time.
State *newState = object->allocateState();
newState->copyState(object->getState());
(*states)[id] = newState;
lastCheckpointTime[id] = nextCheckpointTime[id];
nextCheckpointTime[id] += checkpointPeriod;
stringstream filename;
filename << ckptFilePath << "LP"
<< mySimManager->getSimulationManagerID() << "."
<< lastCheckpointTime[id] << "." << id;
ofstream ckFile(filename.str().c_str(), ofstream::binary);
if (!ckFile.is_open()) {
cerr << mySimManager->getSimulationManagerID()
<< " - Could not open file: " << filename.str()
<< ", aborting simulation." << endl;
abort();
}
mySimManager->getOutputManagerNew()->saveOutputCheckpoint(&ckFile,
objId, lastCheckpointTime[id], threadId);
ckFile.close();
}
}