// generate migrate message from stats->from_proc and to_proc
LBMigrateMsg * CentralLB::createMigrateMsg(LDStats* stats)
{
int i;
CkVec<MigrateInfo*> migrateInfo;
for (i=0; i<stats->n_objs; i++) {
LDObjData &objData = stats->objData[i];
int frompe = stats->from_proc[i];
int tope = stats->to_proc[i];
if (frompe != tope) {
// CkPrintf("[%d] Obj %d migrating from %d to %d\n",
// CkMyPe(),obj,pe,dest);
MigrateInfo *migrateMe = new MigrateInfo;
migrateMe->obj = objData.handle;
migrateMe->from_pe = frompe;
migrateMe->to_pe = tope;
migrateMe->async_arrival = objData.asyncArrival;
migrateInfo.insertAtEnd(migrateMe);
}
}
int migrate_count=migrateInfo.length();
LBMigrateMsg* msg = new(migrate_count,CkNumPes(),CkNumPes(),0) LBMigrateMsg;
msg->n_moves = migrate_count;
for(i=0; i < migrate_count; i++) {
MigrateInfo* item = (MigrateInfo*) migrateInfo[i];
msg->moves[i] = *item;
delete item;
migrateInfo[i] = 0;
}
return msg;
}
/*
* Map objects to PE for load balance. It takes in a min heap of objects which
* can be transferred and finds suitable receiver PEs. The mapping is stored in
* obj_no and the corresponding entry in obj_pe_no indicates the receiver PE.
*/
void DistributedLB::MapObjsToPe(minHeap &objs, CkVec<int> &obj_no,
CkVec<int> &obj_pe_no) {
int p_id;
double p_load;
int rand_pe;
// While my load is more than the threshold, try to transfer objs
while (my_load > (thr_avg)) {
// If there is only one object, then nothing can be done to balance it.
if (objs_count < 2) break;
// Flag to indicate whether successful in finding a transfer
bool success = false;
// Get the smallest object
InfoRecord* obj = objs.deleteMin();
// No more objects to retrieve
if (obj == 0) break;
// If transferring this object makes this PE underloaded, then don't
// transfer
if ((my_load - obj->load) < (thr_avg)) {
break;
}
// Pick random PE based on the probability and the find is successful only
// if on transferring the object, that PE does not become overloaded
do {
rand_pe = PickRandReceiverPeIdx();
if (rand_pe == -1) break;
p_id = pe_no[rand_pe];
p_load = loads[rand_pe];
if ((p_load + obj->load) < avg_load) {
success = true;
}
kMaxTrials--;
} while (!success && (kMaxTrials > 0));
// No successful in finding a suitable PE to transfer the object
if (!success) {
break;
}
// Found an object and a suitable PE to transfer it to. Decrement the obj
// count and update the loads.
obj_no.insertAtEnd(obj->Id);
obj_pe_no.insertAtEnd(p_id);
objs_count--;
loads[rand_pe] += obj->load;
my_load -= obj->load;
// Send information to the receiver PE about this obj. This is necessary for
// ack as well as finding out how many objs are migrating in
thisProxy[p_id].InformMigration(obj->Id, CkMyPe(),
my_stats->objData[obj->Id].wallTime, false);
// This object is assigned, so we delete it from the heap
delete obj;
}
}
void BulkAdapt::get_elemsToLock(adaptAdj startElem, adaptAdj **elemsToLock, int edgeID, int *count)
{
CkVec<adaptAdj>* nbrElems;
// find the elements adjacent to startElem along the edge edgeID
//BULK_DEBUG(CkPrintf("[%d] BulkAdapt::get_elemsToLock: calling getEdgeAdaptAdj on elem %d\n",partitionID,startElem.localID));
nbrElems = getEdgeAdaptAdj(meshID, startElem.localID, startElem.elemType,
edgeID);
// extract adjacencies from CkVec into array needed by the locking code
(*count) = nbrElems->size();
(*elemsToLock) = (adaptAdj *)malloc((*count + 1) * sizeof(adaptAdj));
for (int i=0; i<*count; i++) {
(*elemsToLock)[i] = (*nbrElems)[i];
}
// add the start element
(*elemsToLock)[*count] = startElem;
(*count)++;
/*
printf("Elems to lock: ");
for (int i=0; i<*count; i++) {
printf("(%d, %d, %d) ", (*elemsToLock)[i].partID, (*elemsToLock)[i].localID,
(*elemsToLock)[i].elemType);
}
printf("\n");
*/
}
//Find next un-occupied global number:
int nextUnoccupied(void) {
while (occupiedBefore<occupied.size()) {
if (occupied[occupiedBefore]==0)
return occupiedBefore;
else occupiedBefore++;
}
/* occupiedBefore==occupied.size(), so add to end of list */
occupied.push_back(1);
return occupiedBefore;
}
static void readClause(StreamBuffer& in, par_SolverState& S, CkVec<par_Lit>& lits) {
int parsed_lit, var;
lits.removeAll();
for (;;){
parsed_lit = parseInt(in);
if (parsed_lit == 0) break;
var = abs(parsed_lit)-1;
S.occurrence[var]++;
if(parsed_lit>0)
S.positive_occurrence[var]++;
lits.push_back( par_Lit(parsed_lit));
}
}
/// Mark this entity's global numbers as used:
void mark(FEM_Entity &src) {
int l,len=src.size();
for (l=0;l<len;l++) {
int g=src.getGlobalno(l);
if (g!=-1) {
while (occupied.size()<=g) { //Make room for this marker
occupied.push_back(0);
}
//FIXME: make sure element global numbers aren't repeated
// (tough because *node* global numbers may be repeated)
occupied[g]=1;
}
}
}
void ComlibSectionInfo::getNodeLocalIndices(int nindices,
CkArrayIndex *idxlist,
CkArrayID &destArrayID,
CkVec<CkArrayIndex> &idx_vec){
int acount = 0;
idx_vec.resize(0);
CkArray *a = (CkArray *)_localBranch(destArrayID);
for(acount = 0; acount < nindices; acount++){
//int p = ComlibGetLastKnown(destArrayID, idxlist[acount]);
int p = a->lastKnown(idxlist[acount]);
if(p == CkMyPe())
idx_vec.insertAtEnd(idxlist[acount]);
}
}
/** Extract an IDXL_Side_t into Roccom format. */
static void getRoccomPconn(IDXL_Side_t is,int bias,CkVec<int> &pconn,const int *paneFmChunk)
{
int p,np=IDXL_Get_partners(is);
pconn.push_back(np);
for (p=0;p<np;p++) {
int chunk=IDXL_Get_partner(is,p);
int pane=1+chunk;
if(paneFmChunk) pane=paneFmChunk[chunk];
pconn.push_back(pane);
int n,nn=IDXL_Get_count(is,p);
pconn.push_back(nn); /* number of shared nodes */
for (n=0;n<nn;n++)
pconn.push_back(IDXL_Get_index(is,p,n)+1+bias); /* nodes are 1-based */
}
}
void Patch::localCreateSection() {
#ifdef USE_SECTION_MULTICAST
CkVec<CkArrayIndex6D> elems;
for (int num=0; num<numNbrs; num++)
elems.push_back(CkArrayIndex6D(computesList[num][0], computesList[num][1], computesList[num][2], computesList[num][3], computesList[num][4], computesList[num][5]));
CkArrayID computeArrayID = computeArray.ckGetArrayID();
mCastSecProxy = CProxySection_Compute::ckNew(computeArrayID, elems.getVec(), elems.size());
CkMulticastMgr *mCastGrp = CProxy_CkMulticastMgr(mCastGrpID).ckLocalBranch();
mCastSecProxy.ckSectionDelegate(mCastGrp);
mCastGrp->setReductionClient(mCastSecProxy, new CkCallback(CkIndex_Patch::reduceForces(NULL), thisProxy(thisIndex.x, thisIndex.y, thisIndex.z)));
#endif
}
int CtgGlobalList::isUserSymbol(const char *name) {
// return 1;
if((strncmp("_", name, 1) == 0) || (strncmp("Cpv_", name, 4) == 0)
|| (strncmp("Csv_", name, 4) == 0) || (strncmp("Ctv_", name, 4) == 0)
|| (strncmp("Bnv_", name, 4) == 0) || (strncmp("Bpv_", name, 4) == 0)
|| (strncmp("ckout", name, 5) == 0) || (strncmp("stdout", name, 6) == 0)
|| (strncmp("ckerr", name, 5) == 0)
|| (strncmp("environ", name, 7) == 0)
|| (strncmp("stderr", name, 6) == 0) || (strncmp("stdin", name, 5) == 0)) {
#ifdef CMK_GFORTRAN
if (match(name, "__.*_MOD_.*")) return 1;
#endif
return 0;
}
/**
if the name is on the blacklist, it is not a user symbol
*/
for(int i=0;i<_blacklist.size();i++){
if(strlen(name) == strlen(_blacklist[i]) && strncmp(name,_blacklist[i],strlen(name)) == 0){
return 0;
}
}
return 1;
}
/**
Analyze the current set of global variables, determine
which are user globals and which are system globals,
and store the list of user globals.
*/
CtgGlobalList::CtgGlobalList() {
datalen=0;
nRec=0;
int count;
for (count = 0; count < _namelist.size(); count ++)
{
unsigned long addr;
int size;
if (0 > lookup_obj_sym(_namelist[count], &addr, &size)) {
fprintf(stderr, "%s: no such symbol\n", _namelist[count]);
continue;
}
void *ptr = (void *)addr;
int gSize = ALIGN8(size);
//#if DEBUG_GOT_MANAGER
printf(" -> %s is a user global, of size %d, at %p\n",
_namelist[count], size, ptr);
//#endif
rec.push_back(CtgRec(ptr,datalen,size));
datalen+=gSize;
}
nRec=rec.size();
#if DEBUG_GOT_MANAGER
printf("relt has %d entries, %d of which are user globals\n\n",
relt_size,nRec);
#endif
}
void System::ImportFluidData_request(const CkVec< pair<int,int> > &reqData, const int recvIndex)
{
assert(thisIndex != recvIndex);
const int nrecv = reqData.size();
assert(nrecv > 0);
MeshFluidGrad_msg *msg = new (nrecv, nrecv, nrecv, nrecv) MeshFluidGrad_msg;
msg->n = nrecv;
for (int i = 0; i < nrecv; i++)
{
const int local_id = reqData[i].first;
const int remote_id = reqData[i].second;
assert(local_id >= 0);
assert(local_id < local_n);
msg-> id_list[i] = remote_id;
msg->mesh_pnts[i] = mesh_pnts[local_id];
msg->Wrec_list[i] = Wrec_list[local_id];
#if 1
if (!ptcl_list[local_id].is_active())
{
Fluid Wi = Wrec_list[local_id].w;
const real dt = t_global - mesh_pnts[local_id].tbeg;
for (int k = 0; k < Fluid::NFLUID; k++)
Wi[k] += Wrec_list[local_id].t[k]*dt;
msg->Wrec_list[i].w = Wi;
}
#endif
msg->Wrec_minmax[i] = Wrec_minmax_list[local_id];
}
systemProxy[recvIndex].ImportFluidData_recv(msg);
}
void System::ImportFluidPrimitives_request(const CkVec< pair<int,int> > &reqData, const int recvIndex)
{
assert(thisIndex != recvIndex);
const int nrecv = reqData.size();
assert(nrecv > 0);
CkVec< pair<int, pair<Fluid, MeshPoint> > > data2send(nrecv);
for (int i = 0; i < nrecv; i++)
{
const int local_id = reqData[i].first;
const int remote_id = reqData[i].second;
assert(local_id >= 0);
assert(local_id < local_n);
data2send[i].first = remote_id;
Fluid Wi = Wrec_list[local_id].w;
#if 1
if (!ptcl_list[local_id].is_active())
{
Fluid Wi = Wrec_list[local_id].w;
const real dt = t_global - mesh_pnts[local_id].tbeg;
for (int k = 0; k < Fluid::NFLUID; k++)
Wi[k] += Wrec_list[local_id].t[k]*dt;
}
#endif
data2send[i].second = std::make_pair(Wi, mesh_pnts[local_id]);
}
systemProxy[recvIndex].ImportFluidPrimitives_recv(data2send);
}
LBAllocFn getLBAllocFn(const char *name) {
char *ptr = strpbrk((char *)name, ":,");
int slen = ptr-name;
for (int i=0; i<lbtables.length(); i++)
if (0==strncmp(name, lbtables[i].name, slen)) return lbtables[i].afn;
return NULL;
}
LBCreateFn search(const char *name) {
char *ptr = strpbrk((char *)name, ":,");
int slen = ptr!=NULL?ptr-name:strlen(name);
for (int i=0; i<lbtables.length(); i++)
if (0==strncmp(name, lbtables[i].name, slen)) return lbtables[i].cfn;
return NULL;
}
void par_SolverState::assignclause(CkVec<par_Clause>& cls )
{
clauses.removeAll();
for(int i=0; i<cls.size(); i++)
{
clauses.push_back( cls[i]);
}
}
//function to add a node to the list for a particular node
void addToLists(int *listIndex,CkVec<CkVec<int> > &lists,int chunk,int node){
if(listIndex[chunk] == -1){
CkVec<int> tmpVec;
int index = lists.push_back_v(tmpVec);
listIndex[chunk] = index;
}
CkVec<int> &vec = lists[listIndex[chunk]];
vec.push_back(node);
}
void displayLBs()
{
CmiPrintf("\nAvailable load balancers:\n");
for (int i=0; i<lbtables.length(); i++) {
LBDBEntry &entry = lbtables[i];
if (entry.shown) CmiPrintf("* %s: %s\n", entry.name, entry.help);
}
CmiPrintf("\n");
}
void TraceBluegene::userBracketEvent(const char* name, double bt, double et, void** parentLogPtr, CkVec<void*> bgLogList){
if (!genTimeLog) return;
BgTimeLog* newLog = new BgTimeLog(_threadEP,name,bt,et);
newLog->addBackwardDeps(bgLogList);
CmiAssert(bgLogList.size()>0);
newLog->objId = ((BgTimeLog*)bgLogList[0])->objId; // for sdag
*parentLogPtr = newLog;
tTIMELINEREC.logEntryInsert(newLog);
}
/** Extract all FEM communication information into Roccom format. */
static CkVec<int> getRoccomPconn(int fem_mesh,int *ghost_len,const int *paneFmChunk)
{
CkVec<int> pconn;
// Shared nodes come first:
getRoccomPconn(IDXL_Get_send(FEM_Comm_shared(fem_mesh,FEM_NODE)),0,pconn,paneFmChunk);
int realLen=pconn.size();
// Sent ghost nodes:
getRoccomPconn(IDXL_Get_send(FEM_Comm_ghost(fem_mesh,FEM_NODE)),0,pconn,paneFmChunk);
// Received ghost nodes (use bias to switch to Roccom ghost node numbering)
getRoccomPconn(IDXL_Get_recv(FEM_Comm_ghost(fem_mesh,FEM_NODE)),
FEM_Mesh_get_length(fem_mesh,FEM_NODE),pconn,paneFmChunk);
// Handle elements (much tougher!)
// Find list of element types
int elems[1024];
int e, ne=FEM_Mesh_get_entities(fem_mesh,elems);
for (e=0;e<ne;e++)
if (elems[e]<FEM_ELEM || elems[e]>=FEM_SPARSE)
elems[e--]=elems[--ne]; // swap out bad entity with the end
// Make one output IDXL that combines all element types:
IDXL_t elghost=IDXL_Create();
int out_r=0, out_g=0; // output indices for real; ghost
for (e=0;e<ne;e++) {
IDXL_Combine(elghost,FEM_Comm_ghost(fem_mesh,elems[e]), out_r,out_g);
out_r+=FEM_Mesh_get_length(fem_mesh,elems[e]);
out_g+=FEM_Mesh_get_length(fem_mesh,elems[e]+FEM_GHOST);
}
// Sent ghost elements:
getRoccomPconn(IDXL_Get_send(elghost),0,pconn,paneFmChunk);
// Received ghost elements (shift all ghosts to start after real elements)
getRoccomPconn(IDXL_Get_recv(elghost),out_r,pconn,paneFmChunk);
IDXL_Destroy(elghost);
if (ghost_len) *ghost_len=pconn.size()-realLen;
return pconn;
}
// find sender comms
void HybridBaseLB::collectCommData(int objIdx, CkVec<LDCommData> &comms, int atlevel)
{
LevelData *lData = levelData[atlevel];
LDStats *statsData = lData->statsData;
LDObjData &objData = statsData->objData[objIdx];
for (int com=0; com<statsData->n_comm; com++) {
LDCommData &cdata = statsData->commData[com];
if (cdata.from_proc()) continue;
if (cdata.sender.objID() == objData.objID() && cdata.sender.omID() == objData.omID())
comms.push_back(cdata);
}
}
// return -1 when not supported
int numUniqNodes() {
#if 0
if (numNodes != 0) return numNodes;
int n = 0;
for (int i=0; i<CmiNumPes(); i++)
if (nodeIDs[i] > n)
n = nodeIDs[i];
numNodes = n+1;
return numNodes;
#else
if (numNodes > 0) return numNodes; // already calculated
CkVec<int> unodes;
int i;
for (i=0; i<numPes; i++) unodes.push_back(nodeIDs[i]);
//unodes.bubbleSort(0, numPes-1);
unodes.quickSort();
int last = -1;
std::map<int, int> nodemap; // nodeIDs can be out of range of [0,numNodes]
for (i=0; i<numPes; i++) {
if (unodes[i] != last) {
last=unodes[i];
nodemap[unodes[i]] = numNodes;
numNodes++;
}
}
if (numNodes == 0) {
numNodes = CmiNumNodes();
numPes = CmiNumPes();
}
else {
// re-number nodeIDs, which may be necessary e.g. on BlueGene/P
for (i=0; i<numPes; i++) nodeIDs[i] = nodemap[nodeIDs[i]];
CpuTopology::supported = 1;
}
return numNodes;
#endif
}
// find sender comms
void HbmLB::collectCommData(int objIdx, CkVec<LDCommData> &comms)
{
#if CMK_LBDB_ON
LevelData *lData = levelData[0];
LDObjData &objData = myStats.objData[objIdx];
for (int com=0; com<myStats.n_comm; com++) {
LDCommData &cdata = myStats.commData[com];
if (cdata.from_proc()) continue;
if (cdata.sender.objID() == objData.objID() && cdata.sender.omID() == objData.omID())
comms.push_back(cdata);
}
#endif
}
// Function that receives a set of particles and updates the
// forces of them into the local set
void Patch::receiveParticles(CkVec<Particle> &updates) {
updateCount++;
for( int i=0; i < updates.length(); i++) {
incomingParticles.push_back(updates[i]);
}
// if all the incoming particle updates have been received, we must check
// whether to proceed with next step
if(updateCount == numNbrs ) {
updateCount = 0;
incomingFlag = true;
checkNextStep();
}
}
void System::ImportNewDt_recv(const CkVec< pair<int, real> > &recvUpdates)
{
const int nrecv = recvUpdates.size();
for (int i = 0; i < nrecv; i++)
{
const int iId = recvUpdates[i].first;
assert(iId >= (int)(nactive_loc + nimport_loc));
assert(iId < (int) ptcl_act.size());
assert(ptcl_act[iId]->is_active());
mesh_act[iId]->dt_new = recvUpdates[i].second;
}
ImportNewDt_nRequested--;
assert(ImportNewDt_nRequested >= 0);
if (ImportNewDt_nRequested == 0)
ImportNewDtCb.send();
}
void HybridBaseLB::ReceiveVectorMigration(LBVectorMigrateMsg *msg)
{
#if CMK_LBDB_ON
FindNeighbors();
int atlevel = msg->level - 1;
DEBUGF(("[%d] ReceiveMigration\n", CkMyPe()));
LevelData *lData = levelData[atlevel];
LDStats *statsData = lData->statsData;
// pick objects for required load migration, first fit
lData->vector_expected = 0;
for (int i=0; i<msg->n_moves; i++) {
VectorMigrateInfo &move = msg->moves[i];
CkVec<LDObjData> objs;
CkVec<LDCommData> comms;
if (move.from_pe == CkMyPe()) {
int toPe = move.to_pe;
double load = move.load;
GetObjsToMigrate(toPe, load, statsData, atlevel, comms, objs);
int count = objs.size();
if (_lb_args.debug()>1)
CkPrintf("[%d] sending %d objects to %d at %f.\n", CkMyPe(), count, toPe, CkWallTimer());
if (objs.size() > 0)
thisProxy[toPe].ObjsMigrated(objs, objs.size(), comms.getVec(), comms.size(), atlevel);
thisProxy[toPe].TotalObjMigrated(count, atlevel);
}
else if (move.to_pe == CkMyPe()) {
// expecting objects
lData->vector_expected ++;
}
}
if (_lb_args.debug()>1)
CkPrintf("[%d] expecting %d vectors. \n", CkMyPe(), lData->vector_expected);
if (lData->vectorReceived()) {
VectorDone(atlevel);
if (lData->migrationDone())
StatsDone(atlevel);
}
delete msg;
#endif
}
// read from a file called "globals"
static void readGlobals()
{
if (loaded) return;
const char *fname = "globals";
printf("Loading globals from file \"%s\" ... \n", fname);
FILE *gf = fopen(fname, "r");
if (gf == NULL) {
CmiAbort("Failed to load globals, file may not exist!");
}
while (!feof(gf))
{
char name[1024];
fscanf(gf, "%s\n", name);
_namelist.push_back(strdup(name));
}
fclose(gf);
loaded = 1;
}
void Main::done(CkVec<int> assignment)
{
double endtimer = CkWallTimer();
CkPrintf("\nFile reading and processing time: %f\n", readfiletimer-starttimer);
CkPrintf("Solving time: %f\n", endtimer - readfiletimer);
FILE *file;
char outputfile[50];
sprintf(outputfile, "%s.sat", inputfile);
file = fopen(outputfile, "w");
for(int i=0; i<assignment.size(); i++)
{
fprintf(file, "%d\n", assignment[i]);
}
CkExit();
}
void System::ImportNewDt_request(const CkVec< pair<int,int> > &reqData, const int recvIndex)
{
assert(thisIndex != recvIndex);
const int nrecv = reqData.size();
assert(nrecv > 0);
CkVec< pair<int, real> > data2send;
data2send.reserve(nrecv);
for (int i = 0; i < nrecv; i++)
{
const int local_id = reqData[i].first;
const int remote_id = reqData[i].second;
assert(local_id >= 0);
assert(local_id < local_n);
assert(ptcl_list[local_id].is_active());
data2send.push_back(std::make_pair(remote_id, mesh_pnts[local_id].dt_new));
}
systemProxy[recvIndex].ImportNewDt_recv(data2send);
}
static void readBlacklist()
{
if (loaded) return;
const char *fname = "blacklist";
FILE *bl = fopen(fname, "r");
if (bl == NULL){
if (CmiMyPe() == 0) printf("WARNING: Running swapglobals without blacklist, globals from libraries might be getting un-necessarily swapped\n");
loaded = 1;
return;
}
printf("Loading blacklist from file \"%s\" ... \n", fname);
while (!feof(bl)){
char name[512];
fscanf(bl, "%s\n", name);
_blacklist.push_back(strdup(name));
}
fclose(bl);
loaded = 1;
}
