static void namdInitPapiCounters(){
if(CkMyRank()==0){
//only initialize per OS process (i.e. a charm node)
int retval = PAPI_library_init(PAPI_VER_CURRENT);
if(retval != PAPI_VER_CURRENT) {
if(CkMyPe()==0){
CkPrintf("ERROR: PAPI library is not compatitible!");
CkExit();
}
}
#if CMK_SMP
//now only consider systems that are compatible with POSIX
if(PAPI_thread_init(pthread_self)!=PAPI_OK) {
if(CkMyPe()==0){
CkPrintf("ERROR: multi-thread mode in PAPI could not be initialized!");
CkExit();
}
}
#endif
}
CkpvInitialize(int *, papiEvents);
CkpvAccess(papiEvents) = new int[NUM_PAPI_EVENTS];
#if MEASURE_PAPI_CACHE
if(PAPI_query_event(PAPI_L1_DCM)==PAPI_OK) {
CkpvAccess(papiEvents)[0] = PAPI_L1_DCM;
}else{
if(CkMyPe()==0){
CkPrintf("WARNING: PAPI_L1_DCM doesn't exsit on this platform!\n");
}
//if not default to PAPI_TOT_INS
CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
}
if(PAPI_query_event(PAPI_L2_DCM)==PAPI_OK) {
CkpvAccess(papiEvents)[1] = PAPI_L2_DCM;
}else{
//if not default to PAPI_TOT_CYC
CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
}
#elif MEASURE_PAPI_FLOPS
if(PAPI_query_event(PAPI_FP_INS)==PAPI_OK) {
CkpvAccess(papiEvents)[0] = PAPI_FP_INS;
}else{
if(CkMyPe()==0){
CkPrintf("WARNING: PAPI_FP_INS doesn't exsit on this platform!\n");
}
//if not default to PAPI_TOT_INS
CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
}
if(PAPI_query_event(PAPI_FMA_INS)==PAPI_OK) {
CkpvAccess(papiEvents)[1] = PAPI_FMA_INS;
}else{
//if not default to PAPI_TOT_CYC
CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
}
#endif
}
void processGhosts(ghostMsg *msg) {
int kk=0;
int size = msg->size;
switch(msg->dir) {
case LEFT:
for(int j=jstart;j<=jend;j++) for (int i=iend+1; i<=iend+RADIUS; i++)
IN(i,j) = msg->edge[kk++];
break;
case RIGHT:
for(int j=jstart;j<=jend;j++) for (int i=istart-RADIUS; i<=istart-1; i++)
IN(i,j) = msg->edge[kk++];
break;
case BOTTOM:
for (int j=jend+1; j<=jend+RADIUS; j++) for(int i=istart;i<=iend;i++){
IN(i,j) = msg->edge[kk++];
}
break;
case TOP:
for (int j=jstart-RADIUS; j<=jstart-1; j++) for(int i=istart;i<=iend;i++)
IN(i,j) = msg->edge[kk++];
break;
default: CkPrintf("ERROR: invalid direction\n");
CkExit();
}
delete msg;
}
//============================================================================
//cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc
//============================================================================
CP_Rho_GSpacePlane::CP_Rho_GSpacePlane(UberCollection _instance) :
thisInstance(_instance)
{
#ifdef _CP_DEBUG_RHOG_VERBOSE_
CkPrintf("{%d} Rho GS [%d] constructor\n", thisInstance.proxyOffset, thisIndex);
#endif
myTime = 1;
divRhoX = NULL;
divRhoY = NULL;
divRhoZ = NULL;
//RAZ: Added spin flags;
cp_lsda = simReadOnly.cp_lsda;
mySpinIndex = thisInstance.idxU.s;
if(mySpinIndex==1 && cp_lsda!=1){
CkPrintf("@@@@@@@@@@@@@@@@@@@@_error_@@@@@@@@@@@@@@@@@@@@\n");
CkPrintf("Error in spin index and LSDA flag in Hartree call.\n");
CkPrintf("Good-bye.\n");
CkPrintf("@@@@@@@@@@@@@@@@@@@@_error_@@@@@@@@@@@@@@@@@@@@\n");
CkExit();
}//endif
doneWhiteByrd = 0;
usesAtSync = true;
if(config.lbdensity){
setMigratable(true);
}else{
setMigratable(false);
}
//---------------------------------------------------------------------------
}//end routine
main::main(CkArgMsg* m)
{
//Process command-line arguments
//Start the computation
mainProxy = thishandle;
if(m->argc<2)
{
CkPrintf("Needs number of array elements\n");
CkExit();
}
units=atoi(m->argv[1]);
// 4 MB size
allredSize= 4194304; //atoi(m->argv[2]);
baseSize = 262144;
currentSize = baseSize;
sizeInd = 0;
numItr = 10;
sizesNo = 5;
timeForEach = new double[sizesNo];
iterNo = 0;
for(int i=0; i<sizesNo; i++)
timeForEach[i] = 0.0;
arr = CProxy_AllReduce::ckNew(thisProxy, units);
CkPrintf("AllReduce for %d pes on %d units for %d size\n",
CkNumPes(),units,allredSize);
arr.init();
startTime = CkWallTimer();
arr.dowork(baseSize);
}
void Main::save_temp(int SIZE, double *number, int row_number){
int i,j;
double (*temp)[matrix_size] = (double (*)[matrix_size]) result;
for(i = 0; i < matrix_size/CkNumPes(); i++){
for(j = 0; j < matrix_size; j++) {
temp[i+row_number*matrix_size/CkNumPes()][j] = number[i*matrix_size+j];
}
}
doneCount ++;
//CkPrintf("donecount: %d ",doneCount);
if (doneCount == CkNumPes()){
/*
for (i = 0; i < matrix_size; i++) {
CkPrintf("tulemus: ");
for (j = 0; j < matrix_size; j++) {
CkPrintf(" %.1f ",temp[i][j]);
}
CkPrintf("\n");
}
*/
CkExit();
}
}
void Main::checkIn() {
checkInCount ++;
if( checkInCount >= m*n)
CkExit();
}
/** Count the number of elements on edge (n1, n2) */
int FEM_Mesh::countElementsOnEdge(int n1, int n2) {
if (n1==n2) {
CkPrintf("ERROR: You called countElementsOnEdge() with two identical nodes %d, and %d \n", n1, n2);
CkExit();
}
int *n1AdjElems=0, *n2AdjElems=0;
int n1NumElems, n2NumElems;
CkAssert(node.is_valid_any_idx(n1));
CkAssert(node.is_valid_any_idx(n2));
n2e_getAll(n1, n1AdjElems, n1NumElems);
n2e_getAll(n2, n2AdjElems, n2NumElems);
CkAssert(n1AdjElems!=0);
CkAssert(n2AdjElems!=0);
int count=0;
for (int i=0; i<n1NumElems; i++) {
for (int j=0; j<n2NumElems; j++) {
if (n1AdjElems[i] == n2AdjElems[j]) {
count++;
}
}
}
delete[] n1AdjElems;
delete[] n2AdjElems;
return count;
}
//! \brief Constructor
//! \details The main chare constructor is the main entry point of the
//! program, called by the Charm++ runtime system. The constructor does
//! basic initialization steps, e.g., parser the command-line, prints out
//! some useful information to screen (in verbose mode), and instantiates
//! a driver. Since Charm++ is fully asynchronous, the constructor
//! usually spawns asynchronous objects and immediately exits. Thus in the
//! body of the main chare constructor we fire up an 'execute' chare,
//! which then calls back to Main::execute(). Finishing the main chare
//! constructor the Charm++ runtime system then starts the
//! network-migration of all global-scope data (if any). The execute chare
//! calling back to Main::execute() signals the end of the migration of
//! the global-scope data. Then we are ready to execute the driver. Since
//! the unit test suite is parallel and asynchronous, its driver fires up
//! additional Charm++ chare objects which then call back to
//! Main::finalize() at some point in the future when all work has been
//! finished. finalize() then exits by calling Charm++'s CkExit(),
//! shutting down the runtime system.
//! \see http://charm.cs.illinois.edu/manuals/html/charm++/manual.html
Main( CkArgMsg* msg )
try :
// Parse command line into m_cmdline using default simple pretty printer
m_cmdParser( msg->argc, msg->argv, tk::Print(), m_cmdline, m_helped ),
// Create pretty printer initializing output streams based on command line
m_print( m_cmdline.get< tag::verbose >() ? std::cout : std::clog ),
// Create UnitTest driver
m_driver( tk::Main< unittest::UnitTestDriver >
( msg->argc, msg->argv,
m_cmdline,
tk::HeaderType::UNITTEST,
UNITTEST_EXECUTABLE,
m_print ) ),
m_timer(1) // Start new timer measuring the serial+Charm++ runtime
{
// Immediately exit if any help was requested; help is printed in main()
if (m_helped) CkExit();
// Save executable name to global-scope string so FileParser can access it
unittest::g_executable = msg->argv[0];
delete msg;
mainProxy = thisProxy;
// Fire up an asynchronous execute object, which when created at some
// future point in time will call back to this->execute(). This is
// necessary so that this->execute() can access already migrated
// global-scope data.
CProxy_execute::ckNew();
// Start new timer measuring the migration of global-scope data
m_timer.emplace_back();
} catch (...) { tk::processExceptionCharm(); }
void findarea(int x, bool done)
{
ckout<<"Radius of circle is: "<<x<<"Area of the circle is: "<<y*x*x<<endl;
if(done)
CkExit();
};
void processGhosts(ghostMsg *msg) {
int k; k=0;
int size = msg->size;
switch(msg->dir) {
case LEFT:
for(int j=0;j<size;++j) for (int i=0; i<RADIUS; i++)
IN(width+i,j) = msg->edge[k++];
break;
case RIGHT:
for(int j=0;j<size;++j) for (int i=0; i<RADIUS; i++)
IN(-RADIUS+i,j) = msg->edge[k++];
break;
case BOTTOM:
for (int j=0; j<RADIUS; j++) for(int i=0;i<size;++i){
IN(i,height+j) = msg->edge[k++];
}
break;
case TOP:
for (int j=0; j<RADIUS; j++) for(int i=0;i<size;++i)
IN(i,-RADIUS+j) = msg->edge[k++];
break;
default: CkPrintf("ERROR: invalid direction\n");
CkExit();
}
delete msg;
}
void TheMain::exit() {
//CkPrintf("called = %d\n",called);
if (++called >= CmiNumPes()) {
CkPrintf("All done\n");
CkExit();
}
}
/// Exit the simulation
void pose::exit(void)
{
if (callBackSet)
cb.send(); // need to make callback here
else
CkExit();
}
void done() {
endt = CkWallTimer();
CkPrintf("Main: done\n");
CkPrintf("\tnumber of uchares %ld, number of sent messages %f*10^6\n", N_uChares, ((double)(N_uChares*(N_uChares-1)))/(1000000));
CkPrintf("\t%f secs, %f.4 MMsg/s \n", endt-startt, ((double)(N_uChares*(N_uChares-1)))/(1000000 *(endt-startt)));
CkExit();
}
void maindone(void)
{
niter++;
if(niter == START_TRACE_ITER)
_traceControl.startTrace();
if(niter == END_TRACE_ITER)
_traceControl.endTrace();
if(niter == iterations)
{
double pingTimer = CkWallTimer() - start_time;
CkPrintf("Pingping %d\t\t %d \t\t%d \t\t%d \t\t%d \t\t%.1f\n",
nodeIndex, CharesPerPE, PEsPerNode, PEsPerNode * CharesPerPE, payload, 1000*1000*pingTimer/(iterations-WARM_UP));
if(nodeIndex == CkNumNodes() -1)
CkExit();
else
{
niter = 0;
for(int i=0; i<PEsPerNode * CharesPerPE; i++)
arr1[i].start(nodeIndex);
}
nodeIndex++;
}else
{
if(niter == WARM_UP)
start_time = CkWallTimer();
for(int i=0; i<PEsPerNode * CharesPerPE; i++)
arr1[i].start(nodeIndex);
}
};
void maindone(void)
{
static int count = 0;
count ++;
CkPrintf("All done\n");
CkExit();
};
void Main::done() {
numIterations++;
if(numIterations == 1) {
firstTime = CkWallTimer();
#if USE_CKDIRECT
CkPrintf("FIRST ITER TIME %f secs\n", firstTime - setupTime);
#else
CkPrintf("FIRST ITER TIME %f secs\n", firstTime - startTime);
#endif
compute.resetArrays();
} else {
if(numIterations == NUM_ITER) {
endTime[numIterations-2] = CkWallTimer() - firstTime;
double sum = 0;
for(int i=0; i<NUM_ITER-1; i++)
sum += endTime[i];
#if USE_CKDIRECT
CkPrintf("AVG TIME %f secs\n", sum/(NUM_ITER-1));
#else
CkPrintf("AVG TIME %f secs\n", sum/(NUM_ITER-1));
#endif
CkExit();
} else {
endTime[numIterations-2] = CkWallTimer() - firstTime;
compute.resetArrays();
}
}
}
void done(void)
{
count--;
if (count == 0) {
CkPrintf("All done\n");
CkExit();
}
};
void done(void)
{
counter++;
if(counter == 4) {
CkPrintf("All done\n");
CkExit();
}
};
void done(int totalIter)
{
if(totalIter >= maxiterations)
CkPrintf("Finish due to max iterations %d, total time %.3f seconds. \n", totalIter, CkWallTimer()-startTime);
else
CkPrintf("Finish due to convergence, iterations %d, total time %.3f seconds. \n", totalIter, CkWallTimer()-startTime);
CkExit();
}
// Entry point of Charm++ application
Main::Main(CkArgMsg* msg)
{
// Print a message for the user
CkPrintf("Hello World!\n");
// Exit the application --> This is needed. It does not work like normal C/C++ that just returning will do the job.
CkExit();
}
void maindone(void)
{
static int count = 0;
count ++;
// if (count < sectionSize*3) return;
CkPrintf("All done\n");
CkExit();
};
void Main::finished()
{
if (--numFinished == 0) {
double elapt = CmiWallTimer()-startTime;
CkPrintf("Finished in %fs %fs/step and iters is %d\n", elapt, elapt/ITER,ITER);
CkExit();
}
}
void exit() {
nDone++;
// Becase each array element sends a message to all array elements, there are a total of nElements^2 messages received
if (nDone == nElements*nElements){
CkPrintf("Successful Completion\n");
CkExit();
}
}
Main(CkArgMsg* m) {
#if CMK_BLUEGENEL
BGLPersonality bgl_p;
int i = rts_get_personality(&bgl_p, sizeof(BGLPersonality));
#elif CMK_BLUEGENEP
DCMF_Hardware_t bgp_hwt;
DCMF_Hardware(&bgp_hwt);
#elif XT3_TOPOLOGY
XT3TorusManager xt3tm;
#elif XT4_TOPOLOGY || XT5_TOPOLOGY
XTTorusManager xttm;
#endif
mainProxy = thishandle;
CkPrintf("Testing TopoManager .... \n");
TopoManager tmgr;
CkPrintf("Torus Size [%d] [%d] [%d] [%d]\n", tmgr.getDimNX(), tmgr.getDimNY(), tmgr.getDimNZ(), tmgr.getDimNT());
#if CMK_BLUEGENEP
CkPrintf("Torus Size [%d] [%d] [%d] [%d]\n", bgp_hwt.xSize, bgp_hwt.ySize, bgp_hwt.zSize, bgp_hwt.tSize);
#endif
int x, y, z, t;
for(int i=0; i<CkNumPes(); i++) {
tmgr.rankToCoordinates(i, x, y, z, t);
CkPrintf("---- Processor %d ---> x %d y %d z %d t %d\n", i, x, y, z, t);
#if CMK_BLUEGENEL
unsigned int tmp_t, tmp_x, tmp_y, tmp_z;
rts_coordinatesForRank(i, &tmp_x, &tmp_y, &tmp_z, &tmp_t);
CkPrintf("Real Processor %d ---> x %d y %d z %d t %d\n", i, tmp_x, tmp_y, tmp_z, tmp_t);
#elif CMK_BLUEGENEP
unsigned int tmp_t, tmp_x, tmp_y, tmp_z;
#if (DCMF_VERSION_MAJOR >= 3)
DCMF_NetworkCoord_t nc;
DCMF_Messager_rank2network(i, DCMF_DEFAULT_NETWORK, &nc);
tmp_x = nc.torus.x;
tmp_y = nc.torus.y;
tmp_z = nc.torus.z;
tmp_t = nc.torus.t;
#else
DCMF_Messager_rank2torus(c, &tmp_x, &tmp_y, &tmp_z, &tmp_t);
#endif
CkPrintf("Real Processor %d ---> x %d y %d z %d t %d\n", i, tmp_x, tmp_y, tmp_z, tmp_t);
#elif XT3_TOPOLOGY
int tmp_t, tmp_x, tmp_y, tmp_z;
xt3tm.realRankToCoordinates(i, tmp_x, tmp_y, tmp_z, tmp_t);
CkPrintf("Real Processor %d ---> x %d y %d z %d t %d\n", i, tmp_x, tmp_y, tmp_z, tmp_t);
#elif XT4_TOPOLOGY || XT5_TOPOLOGY
int tmp_t, tmp_x, tmp_y, tmp_z;
xttm.realRankToCoordinates(i, tmp_x, tmp_y, tmp_z, tmp_t);
CkPrintf("Real Processor %d ---> x %d y %d z %d t %d\n", i, tmp_x, tmp_y, tmp_z, tmp_t);
#endif
} // end of for loop
int size = tmgr.getDimNX() * tmgr.getDimNY() * tmgr.getDimNZ();
CkPrintf("Torus Contiguity Metric %d : %d [%f] \n", size, CkNumPes()/tmgr.getDimNT(), (float)(CkNumPes())/(tmgr.getDimNT()*size) );
CkExit();
};
void verify_contribute(CmiUInt8 r) {
CkPrintf("N = %lld, r = %lld\n", N, r);
if (r != N)
CkAbort("test failed");
else {
CkPrintf("test verified");
CkExit();
}
}
void pong(pingPongMsg* msg) {
if(msg->left == 0) {
delete msg;
CkExit();
} else {
msg->left --;
thisProxy[1].ping(msg);
}
}
int getTaskIdForMigration(ObjGraph *ogr,int pe,int start)
{
for(int vert = start; vert < ogr->vertices.size(); vert++)
{
if(ogr->vertices[vert].getCurrentPe()==pe && ogr->vertices[vert].getNewPe()==-1) return vert;
}
CkPrintf("THERE IS A PROBLEM IN TEMPREFINELB 222 start=%d pe=%d objArraySize=%d!!!!!\n",start,pe,ogr->vertices.size());
CkExit();
}
void receiveMessage(TestMessage *msg) {
allMsgs.push_back(msg);
if (allMsgs.size() == nCheck) {
alignmentTest(allMsgs, "destination");
for (int i = 0; i < allMsgs.size(); i++) {
delete allMsgs[i];
}
CkExit();
}
}
/** Get 2 elements on edge (n1, n2) where n1, n2 are chunk-local
node numberings; return the edges in result_e1 and result_e2
No preference is given to ghosts over local elements
*/
void FEM_Mesh::get2ElementsOnEdge(int n1, int n2, int *result_e1, int *result_e2)
{
int *n1AdjElems=0, *n2AdjElems=0;
int n1NumElems, n2NumElems;
if(n1==n2){
CkPrintf("ERROR: You called get2ElementsOnEdge() with two identical nodes %d, and %d \n", n1, n2);
CkExit();
}
n2e_getAll(n1, n1AdjElems, n1NumElems);
n2e_getAll(n2, n2AdjElems, n2NumElems);
CkAssert(n1AdjElems!=0);
CkAssert(n2AdjElems!=0);
int found=0;
*result_e1=-1;
*result_e2=-1;
for (int i=0; i<n1NumElems; i++) {
for (int j=0; j<n2NumElems; j++) {
if (n1AdjElems[i] == n2AdjElems[j]) {
if(found==0){
// CkPrintf("found element1 %d\n", n1AdjElems[i]);
*result_e1 = n1AdjElems[i];
found++;
}
else if(found==1){
// CkPrintf("found element2 %d\n", n1AdjElems[i]);
*result_e2 = n1AdjElems[i];
found++;
}
else {
CkPrintf("ERROR: Found a third element(%d) on edge %d,%d \n", n1AdjElems[i], n1, n2);
CkExit();
}
}
}
}
delete[] n1AdjElems;
delete[] n2AdjElems;
}
void finalize( bool worked ) {
try {
if (worked && !m_timer.empty()) {
m_timestamp.emplace_back( "Total runtime", m_timer[0].hms() );
m_print.time( "Timers (h:m:s)", m_timestamp );
m_print.endpart();
}
} catch (...) { tk::processExceptionCharm(); }
// Tell the Charm++ runtime system to exit
CkExit();
}