void start(CmiUInt8 root, const CkCallback & cb) {
CkCallback startCb(CkIndex_BFSMultiVertex::foo(), g[root / (N / CmiNumPes())]);
CkCallback endCb(CkIndex_TestDriver::startVerificationPhase(), driverProxy);
aggregator.init(g.ckGetArrayID(), startCb, endCb, -1, true);
CkStartQD(CkCallbackResumeThread());
g[root / (N / CmiNumPes())].make_root(root);
CkStartQD(cb);
}
void ComputeMgr::splitComputes()
{
if ( ! CkMyRank() ) {
ComputeMap *computeMap = ComputeMap::Object();
const int nc = computeMap->numComputes();
for (int i=0; i<nc; i++) {
int nnp = computeMap->newNumPartitions(i);
if ( nnp > 0 ) {
if ( computeMap->numPartitions(i) != 1 ) {
CkPrintf("Warning: unable to partition compute %d\n", i);
computeMap->setNewNumPartitions(i,0);
continue;
}
//CkPrintf("splitting compute %d by %d\n",i,nnp);
computeMap->setNumPartitions(i,nnp);
if (computeMap->newNode(i) == -1) {
computeMap->setNewNode(i,computeMap->node(i));
}
for ( int j=1; j<nnp; ++j ) {
int newcid = computeMap->cloneCompute(i,j);
//CkPrintf("compute %d partition %d is %d\n",i,j,newcid);
}
}
}
computeMap->extendPtrs();
}
if (!CkMyPe())
{
CkStartQD(CkIndex_ComputeMgr::splitComputes2((CkQdMsg*)0), &thishandle);
}
}
void
ComputeMgr::updateLocalComputes3()
{
ComputeMap *computeMap = ComputeMap::Object();
CProxy_ProxyMgr pm(CkpvAccess(BOCclass_group).proxyMgr);
ProxyMgr *proxyMgr = pm.ckLocalBranch();
ProxyMgr::nodecount = 0;
const int nc = computeMap->numComputes();
if ( ! CkMyRank() ) {
for (int i=0; i<nc; i++) {
computeMap->setNewNumPartitions(i,0);
if (computeMap->newNode(i) != -1) {
computeMap->setNode(i,computeMap->newNode(i));
computeMap->setNewNode(i,-1);
}
}
}
for(int i=0; i<computeFlag.size(); i++) createCompute(computeFlag[i], computeMap);
computeFlag.clear();
proxyMgr->removeUnusedProxies();
if (!CkMyPe())
{
CkStartQD(CkIndex_ComputeMgr::updateLocalComputes4((CkQdMsg*)0), &thishandle);
}
}
void
ComputeMgr::updateLocalComputes5()
{
if ( ! CkMyRank() ) {
ComputeMap::Object()->checkMap();
PatchMap::Object()->checkMap();
}
// we always use the centralized building of spanning tree
// distributed building of ST called in Node.C only
if (proxySendSpanning || proxyRecvSpanning)
ProxyMgr::Object()->buildProxySpanningTree2();
// this code needs to be turned on if we want to
// shift the creation of ST to the load balancer
#if 0
if (proxySendSpanning || proxyRecvSpanning)
{
if (firstphase)
ProxyMgr::Object()->buildProxySpanningTree2();
else
if (CkMyPe() == 0)
ProxyMgr::Object()->sendSpanningTrees();
firstphase = 0;
}
#endif
if (!CkMyPe())
CkStartQD(CkIndex_ComputeMgr::doneUpdateLocalComputes(), &thishandle);
}
void doPageRank() {
PageRankGraph::Proxy & g = graph->getProxy();
double update_walltime = CkWallTimer() - starttime;
CkPrintf("Initialization completed:\n");
CkPrintf("CPU time used = %.6f seconds\n", update_walltime);
starttime = CkWallTimer();
for (int i = 0; i < iters; i++) {
CkPrintf("PageRank step %d:\n", i);
// do pagerank step initilization
g.doPageRankStep_init();
// wait for current step to be done
CkStartQD(CkCallbackResumeThread());
// do pagerank step
g.doPageRankStep_update();
// wait for current step to be done
CkStartQD(CkCallbackResumeThread());
}
startVerificationPhase();
}
void done() {
double update_walltime = CkWallTimer() - starttime;
//double gteps = 1e-9 * globalNubScannedVertices * 1.0/update_walltime;
CkPrintf("[Final] CPU time used = %.6f seconds\n", update_walltime);
//CkPrintf("Scanned vertices = %lld (%.0f%%)\n", globalNubScannedVertices, (double)globalNubScannedVertices*100/opts.N);
//CkPrintf("%.9f Billion(10^9) Traversed edges per second [GTEP/s]\n", gteps);
//CkPrintf("%.9f Billion(10^9) Traversed edges/PE per second [GTEP/s]\n",
// gteps / CkNumPes());
//g.print();
CkStartQD(CkIndex_TestDriver::exit(), &thishandle);
}
void waitqd_QDChare::waitQD(void) {
if (waitStarted == 1) {
CdsFifo_Enqueue((CdsFifo)threadList, (void *)CthSelf());
} else {
waitStarted = 1;
threadList = (void*) CdsFifo_Create();
CdsFifo_Enqueue((CdsFifo) threadList, (void *)CthSelf());
CkStartQD(CkIndex_waitqd_QDChare::onQD((CkQdMsg*)0), &thishandle);
}
CthSuspend();
}
void startGraphConstruction() {
while (opts.freeze_after_graph_creation) {}
CkPrintf("PageRank running...\n");
CkPrintf("\tnumber of mpi processes is %d\n", CkNumPes());
CkPrintf("\tgraph (s=%d, k=%d), scaling: %s\n", opts.scale, opts.K, (opts.strongscale) ? "strong" : "weak");
CkPrintf("Start graph construction:........\n");
starttime = CkWallTimer();
generator->generate();
CkStartQD(CkIndex_TestDriver::doPageRank(), &thishandle);
}
void start() {
starttime = CkWallTimer();
CkCallback startCb(CkIndex_Updater::generateUpdates(), updater_array);
CkCallback endCb(CkIndex_TestDriver::startVerificationPhase(), thisProxy);
updater_array.generateUpdates();
CkStartQD(CkIndex_TestDriver::startVerificationPhase(), &thishandle);
// Initialize the communication library, which, upon readiness,
// will initiate the test via startCb
//aggregator.init(updater_array.ckGetArrayID(), startCb, endCb, -1, false);
}
Main(CkArgMsg* msg) {
int n = atoi(msg->argv[1]);
mainProxy = thisProxy;
CkPrintf("n = %d\n",n);
BProxy = CProxy_B::ckNew(n);
AProxy = CProxy_A::ckNew(2);
AProxy.F();
CkCallback cb = CkCallback(CkReductionTarget(Main, done), thisProxy);
CkStartQD(cb);
}
Main(CkArgMsg *m) {
CkPrintf("running SDAG migration test\n");
CProxy_Test testProxy = CProxy_Test::ckNew(NUM_ELEMS);
testProxy.wrapper(100, 200);
for (int i = 0; i < NUM_ELEMS; i++) {
char str[100];
sprintf(str, "test %d", i);
Msg* m = new (strlen(str) + 1) Msg(i, str);
testProxy[i].method2(i * 2, i * 2 + 1);
testProxy[i].method3(m);
testProxy[i].methodA();
}
CkStartQD(CkCallback(CkIndex_Main::finished(), thisProxy));
}
TestDriver(CkArgMsg* args) {
parseCommandOptions(args->argc, args->argv, opts);
driverProxy = thishandle;
N = opts.N;
D = 0.85;
// Create graph
graph = new PageRankGraph(CProxy_PageRankVertex::ckNew(opts.N));
// create graph generator
generator = new Generator(*graph, opts);
starttime = CkWallTimer();
CkStartQD(CkIndex_TestDriver::startGraphConstruction(), &thishandle);
delete args;
}
void ComputeMgr::updateComputes(int ep, CkGroupID chareID)
{
updateComputesReturnEP = ep;
updateComputesReturnChareID = chareID;
updateComputesCount = CkNumPes();
if (CkMyPe())
{
iout << iPE << iERRORF << "updateComputes signaled on wrong Pe!\n" << endi;
CkExit();
return;
}
CkStartQD(CkIndex_ComputeMgr::updateComputes2((CkQdMsg*)0),&thishandle);
}
/// In sequential mode, resume after checkpointing or restarting
void sim::SeqResumeAfterCheckpoint() {
// Ensure this only happens on sim 0
CkAssert(thisIndex == 0);
// Ensure this function is only called once after a checkpoint
CkAssert(seqCheckpointInProgress);
seqCheckpointInProgress = 0;
POSE_GlobalClock = seqLastCheckpointGVT;
CkPrintf("POSE: checkpoint/restart complete on sim %d at GVT=%lld sim time=%.1f sec\n", thisIndex, POSE_GlobalClock, CmiWallTimer() + seqStartTime);
// restart simulation
while (POSE_Skipped_Events.length() > 0) {
// These Step iterations must all be enqueued now, before any messages
// are delivered, or else the event queues will break. Because
// messages spawned by these events may need to be inserted, these
// events should be enqueued in the same manner as that used by
// the translated verion of POSE_invoke
eventMsg *evtMsg = new eventMsg;
Skipped_Event se = POSE_Skipped_Events.deq();
int _POSE_handle = se.simIndex;
POSE_TimeType _POSE_timeOffset = se.timestamp - objID->ovt;
objID->registerTimestamp(_POSE_handle, evtMsg, _POSE_timeOffset);
if (pose_config.dop) {
ct = CmiWallTimer();
evtMsg->rst = ct - st + eq->currentPtr->srt;
}
#if !CMK_TRACE_DISABLED
evtMsg->sanitize();
#endif
#if !CMK_TRACE_DISABLED
if(pose_config.stats)
localStats->SwitchTimer(COMM_TIMER);
#endif
thisProxy[_POSE_handle].CheckpointStep(evtMsg);
#if !CMK_TRACE_DISABLED
if (pose_config.stats)
localStats->SwitchTimer(DO_TIMER);
#endif
}
// restart quiescence detection, which is used for termination of
// sequential POSE
CkStartQD(CkIndex_pose::stop(), &POSE_Coordinator_ID);
}
void ComputeMgr::updateLocalComputes()
{
ComputeMap *computeMap = ComputeMap::Object();
CProxy_ProxyMgr pm(CkpvAccess(BOCclass_group).proxyMgr);
ProxyMgr *proxyMgr = pm.ckLocalBranch();
LdbCoordinator *ldbCoordinator = LdbCoordinator::Object();
computeFlag.resize(0);
const int nc = computeMap->numComputes();
for (int i=0; i<nc; i++) {
if ( computeMap->node(i) == CkMyPe() &&
computeMap->newNumPartitions(i) > 1 ) {
Compute *c = computeMap->compute(i);
ldbCoordinator->Migrate(c->ldObjHandle,CkMyPe());
delete c;
computeMap->registerCompute(i,NULL);
if ( computeMap->newNode(i) == CkMyPe() ) computeFlag.add(i);
} else
if (computeMap->newNode(i) == CkMyPe() && computeMap->node(i) != CkMyPe())
{
computeFlag.add(i);
for (int n=0; n < computeMap->numPids(i); n++)
{
proxyMgr->createProxy(computeMap->pid(i,n));
}
}
else if (computeMap->node(i) == CkMyPe() &&
(computeMap->newNode(i) != -1 && computeMap->newNode(i) != CkMyPe() ))
{
// CkPrintf("delete compute %d on pe %d\n",i,CkMyPe());
delete computeMap->compute(i);
computeMap->registerCompute(i,NULL);
}
}
if (!CkMyPe())
{
CkStartQD(CkIndex_ComputeMgr::updateLocalComputes2((CkQdMsg*)0), &thishandle);
}
}
/// Stop the simulation
void pose::stop(void)
{
#ifdef SEQUENTIAL_POSE
// invoke any registered stop events and restart quiescence detection
if (poseIndexOfStopEvent >= 0) {
POSE_Objects[poseIndexOfStopEvent].invokeStopEvent();
CkStartQD(CkIndex_pose::stop(), &POSE_Coordinator_ID);
// don't stop if quiescence was reached for a checkpoint operation
} else if (seqCheckpointInProgress) {
POSE_Objects[0].SeqBeginCheckpoint();
} else {
#if USE_LONG_TIMESTAMPS
CkPrintf("Sequential Endtime Approximation: %lld\n", POSE_GlobalClock);
#else
CkPrintf("Sequential Endtime Approximation: %d\n", POSE_GlobalClock);
#endif
// Call sequential termination here, when done it calls prepExit
POSE_Objects.Terminate();
}
#endif
// prepExit();
}
void LdbCoordinator::updateComputesReady() {
DebugM(3,"updateComputesReady()\n");
CProxy_LdbCoordinator(thisgroup).resume();
CkStartQD(CkIndex_LdbCoordinator::resumeReady((CkQdMsg*)0),&thishandle);
}
void do_sssp(long root) {
vertex_proxy.init();
CkStartQD(CkCallbackResumeThread());
vertex_proxy[root].make_root();
CkStartQD(done_callback);
}
void POSE_init(int IDflag, int ET) // can specify both
{
CkPrintf("Initializing POSE... \n");
POSEreadCmdLine();
if (pose_config.checkpoint_gvt_interval) {
CkPrintf("POSE checkpointing interval set to %lld GVT ticks\n", pose_config.checkpoint_gvt_interval);
}
if (pose_config.checkpoint_time_interval) {
CkPrintf("POSE checkpointing interval set to %d seconds\n", pose_config.checkpoint_time_interval);
}
if (pose_config.dop) {
CkPrintf("POSE DOP analysis enabled...deleting dop log files...\n");
char fName[32];
for (int i = 0; i < CkNumPes(); i++) {
sprintf(fName, "dop%d.log", i);
unlink(fName);
}
sprintf(fName, "dop_mod.out");
unlink(fName);
sprintf(fName, "dop_sim.out");
unlink(fName);
}
POSE_inactDetect = IDflag;
totalNumPosers = 0;
POSE_endtime = ET;
#ifdef SEQUENTIAL_POSE
_POSE_SEQUENTIAL = 1;
#else
_POSE_SEQUENTIAL = 0;
#endif
#ifndef CMK_OPTIMIZE
traceRegisterUserEvent("Forward Execution", 10);
traceRegisterUserEvent("Cancellation", 20);
traceRegisterUserEvent("Cancel Spawn", 30);
traceRegisterUserEvent("Rollback", 40);
traceRegisterUserEvent("Commit", 50);
traceRegisterUserEvent("OptSync", 60);
#endif
#ifndef SEQUENTIAL_POSE
#ifdef POSE_COMM_ON
// Create the communication library for POSE
POSE_commlib_insthndl = CkGetComlibInstance();
// Create the communication strategy for POSE
StreamingStrategy *strategy = new StreamingStrategy(COMM_TIMEOUT,COMM_MAXMSG);
//MeshStreamingStrategy *strategy = new MeshStreamingStrategy(COMM_TIMEOUT,COMM_MAXMSG);
//PrioStreaming *strategy = new PrioStreaming(COMM_TIMEOUT,COMM_MAXMSG);
//Register the strategy
POSE_commlib_insthndl.setStrategy(strategy);
//com_debug=1;
//CkPrintf("Simulation run with PrioStreaming(%d,%d) for communication optimization...\n", COMM_TIMEOUT, COMM_MAXMSG);
CkPrintf("Simulation run with StreamingStrategy(%d,%d) for communication optimization...\n", COMM_TIMEOUT, COMM_MAXMSG);
//CkPrintf("Simulation run with MeshStreaming(%d,%d) for communication optimization...\n", COMM_TIMEOUT, COMM_MAXMSG);
#endif
// Create a MemoryPool with global handle for memory recycling
MemPoolID = CProxy_MemoryPool::ckNew();
// Create a Temporal Memory Manager
TempMemID = CProxy_TimePool::ckNew();
#endif
// Initialize statistics collection if desired
#ifndef CMK_OPTIMIZE
theLocalStats = CProxy_localStat::ckNew();
CProxy_globalStat::ckNew(&theGlobalStats);
#endif
#ifndef SEQUENTIAL_POSE
// Initialize global handles to GVT and PVT
ThePVT = CProxy_PVT::ckNew();
TheGVT = CProxy_GVT::ckNew();
// Start off using normal forward execution
if(pose_config.lb_on)
{
// Initialize the load balancer
TheLBG = CProxy_LBgroup::ckNew();
TheLBstrategy = CProxy_LBstrategy::ckNew();
CkPrintf("Load balancing is ON.\n");
}
#endif
CProxy_pose::ckNew(&POSE_Coordinator_ID, 0);
// Create array to hold all POSE objects
#ifdef POSE_COMM_ON
POSE_Objects_RO = CProxy_sim::ckNew();
POSE_Objects = POSE_Objects_RO;
#else
POSE_Objects = CProxy_sim::ckNew();
#endif
//#ifndef SEQUENTIAL_POSE
//#ifdef POSE_COMM_ON
// Make POSE_Objects use the comm lib
// ComlibDelegateProxy(&POSE_Objects);
//#endif
//#endif
#ifdef SEQUENTIAL_POSE
if (CkNumPes() > 1) CkAbort("ERROR: Cannot run a sequential simulation on more than one processor!\n");
CkPrintf("NOTE: POSE running in sequential simulation mode!\n");
int fnIdx = CkIndex_pose::stop();
CkStartQD(fnIdx, &POSE_Coordinator_ID);
POSE_GlobalClock = 0;
POSE_GlobalTS = 0;
seqCheckpointInProgress = 0;
seqLastCheckpointGVT = 0;
seqLastCheckpointTime = seqStartTime = 0.0;
poseIndexOfStopEvent = -1;
#else
/* CkPrintf("WARNING: Charm Quiescence termination enabled!\n");
int fnIdx = CkIndex_pose::stop();
CkStartQD(fnIdx, &POSE_Coordinator_ID);
*/
#endif
CkPrintf("POSE initialization complete.\n");
if (POSE_inactDetect) CkPrintf("Using Inactivity Detection for termination.\n");
else
#if USE_LONG_TIMESTAMPS
CkPrintf("Using endTime of %lld for termination.\n", POSE_endtime);
#else
CkPrintf("Using endTime of %d for termination.\n", POSE_endtime);
#endif
sim_timer = CmiWallTimer();
}
/// ENTRY: receive GVT estimate; wake up objects
void PVT::setGVT(GVTMsg *m)
{
#ifndef CMK_OPTIMIZE
if(pose_config.stats)
localStats->TimerStart(GVT_TIMER);
#endif
CProxy_PVT p(ThePVT);
CkAssert(m->estGVT >= estGVT);
estGVT = m->estGVT;
int i, end = objs.getNumSpaces();
#ifdef POSE_COMM_ON
//PrioStreaming *pstrat = (PrioStreaming *)(POSE_commlib_insthndl.getStrategy());
//pstrat->setBasePriority((estGVT+10) - POSE_TimeMax);
//pstrat->setBasePriority(estGVT+10);
#endif
simdone = m->done;
CkFreeMsg(m);
waitForFirst = 1;
objs.Commit();
objs.StratCalcs(); // sync strategy calculations
#ifdef MEM_TEMPORAL
localTimePool->set_min_time(estGVT);
#endif
// Parallel checkpointing: setGVT was broken into two functions, and
// beginCheckpoint was added. Only initiate the checkpointing
// procedure on PE 0, after commits have occurred. This should
// minimize the amount of data written to disk. In order to ensure
// a stable state, we wait for quiescence to be reached before
// beginning the checkpoint. Inconsistent results were obtained
// (possibly from messages still in transit) without this step.
// Once quiescence is reached, PE 0 begins the checkpoint, and then
// resumes the simulation in resumeAfterCheckpoint. This Callback
// function is also the first POSE function to be called when
// restarting from a checkpoint.
// Checkpoints are initiated approximately every
// pose_config.checkpoint_gvt_interval GVT ticks or
// pose_config.checkpoint_time_interval seconds (both defined in
// pose_config.h).
if ((CkMyPe() == 0) && (parCheckpointInProgress == 0) && (estGVT > 0) &&
(((pose_config.checkpoint_gvt_interval > 0) && (estGVT >= (parLastCheckpointGVT + pose_config.checkpoint_gvt_interval))) ||
((pose_config.checkpoint_time_interval > 0) &&
((CmiWallTimer() + parStartTime) >= (parLastCheckpointTime + (double)pose_config.checkpoint_time_interval))))) {
// ensure that everything that can be committed has been
objs.CheckpointCommit();
// wait for quiescence to occur before checkpointing
eventMsg *dummyMsg = new eventMsg();
CkCallback cb(CkIndex_PVT::beginCheckpoint(dummyMsg), CkMyPe(), ThePVT);
parCheckpointInProgress = 1;
parLastCheckpointTime = CmiWallTimer() + parStartTime;
CkStartQD(cb);
} else if ((CkMyPe() == 0) && (parLBInProgress == 0) &&
(((pose_config.lb_gvt_interval > 0) && (estGVT >= (parLastLBGVT + pose_config.lb_gvt_interval))))) {
// wait for quiescence to occur before checkpointing
eventMsg *dummyMsg = new eventMsg();
CkCallback cb(CkIndex_PVT::beginLoadbalancing(dummyMsg), CkMyPe(), ThePVT);
parLBInProgress = 1;
CkStartQD(cb);
} else {
// skip checkpointing
eventMsg *dummyMsg = new eventMsg();
p[CkMyPe()].resumeAfterCheckpoint(dummyMsg);
}
#ifndef CMK_OPTIMIZE
if(pose_config.stats)
localStats->TimerStop();
#endif
}
void start(CmiUInt8 root, const CkCallback & cb) {
g[root].make_root();
CkStartQD(cb);
}
void startVerificationPhase() {
PageRankGraph::Proxy & g = graph->getProxy();
if (opts.verify) g.verify();
CkStartQD(CkIndex_TestDriver::done(), &thishandle);
}
