void Patch::localCreateSection() {
#ifdef USE_SECTION_MULTICAST
CkVec<CkArrayIndex6D> elems;
for (int num=0; num<numNbrs; num++)
elems.push_back(CkArrayIndex6D(computesList[num][0], computesList[num][1], computesList[num][2], computesList[num][3], computesList[num][4], computesList[num][5]));
CkArrayID computeArrayID = computeArray.ckGetArrayID();
mCastSecProxy = CProxySection_Compute::ckNew(computeArrayID, elems.getVec(), elems.size());
CkMulticastMgr *mCastGrp = CProxy_CkMulticastMgr(mCastGrpID).ckLocalBranch();
mCastSecProxy.ckSectionDelegate(mCastGrp);
mCastGrp->setReductionClient(mCastSecProxy, new CkCallback(CkIndex_Patch::reduceForces(NULL), thisProxy(thisIndex.x, thisIndex.y, thisIndex.z)));
#endif
}
void HybridBaseLB::ReceiveVectorMigration(LBVectorMigrateMsg *msg)
{
#if CMK_LBDB_ON
FindNeighbors();
int atlevel = msg->level - 1;
DEBUGF(("[%d] ReceiveMigration\n", CkMyPe()));
LevelData *lData = levelData[atlevel];
LDStats *statsData = lData->statsData;
// pick objects for required load migration, first fit
lData->vector_expected = 0;
for (int i=0; i<msg->n_moves; i++) {
VectorMigrateInfo &move = msg->moves[i];
CkVec<LDObjData> objs;
CkVec<LDCommData> comms;
if (move.from_pe == CkMyPe()) {
int toPe = move.to_pe;
double load = move.load;
GetObjsToMigrate(toPe, load, statsData, atlevel, comms, objs);
int count = objs.size();
if (_lb_args.debug()>1)
CkPrintf("[%d] sending %d objects to %d at %f.\n", CkMyPe(), count, toPe, CkWallTimer());
if (objs.size() > 0)
thisProxy[toPe].ObjsMigrated(objs, objs.size(), comms.getVec(), comms.size(), atlevel);
thisProxy[toPe].TotalObjMigrated(count, atlevel);
}
else if (move.to_pe == CkMyPe()) {
// expecting objects
lData->vector_expected ++;
}
}
if (_lb_args.debug()>1)
CkPrintf("[%d] expecting %d vectors. \n", CkMyPe(), lData->vector_expected);
if (lData->vectorReceived()) {
VectorDone(atlevel);
if (lData->migrationDone())
StatsDone(atlevel);
}
delete msg;
#endif
}
// pick objects to migrate "t" amount of work
void HbmLB::ReceiveMigrationDelta(double t, int lblevel, int fromlevel)
{
#if CMK_LBDB_ON
int i;
int atlevel = fromlevel-1;
LevelData *lData = levelData[atlevel];
if (atlevel != 0) {
thisProxy.ReceiveMigrationDelta(t, lblevel, atlevel, lData->nChildren, lData->children);
return;
}
// I am leave, find objects to migrate
CkVec<int> migs;
CkVec<LDObjData> &objData = myStats.objData;
for (i=0; i<myStats.n_objs; i++) {
LDObjData &oData = objData[i];
if (oData.wallTime < t) {
migs.push_back(i);
t -= oData.wallTime;
if (t == 0.0) break;
}
}
int nmigs = migs.size();
// send a message to
int matchPE = CkMyPe() ^ (1<<(lblevel-1));
DEBUGF(("[%d] migrating %d objs to %d at lblevel %d! \n", CkMyPe(),nmigs,matchPE,lblevel));
thisProxy[matchPE].ReceiveMigrationCount(nmigs, lblevel);
// migrate objects
for (i=0; i<nmigs; i++) {
int idx = migs[i]-i;
LDObjData &oData = objData[idx];
CkVec<LDCommData> comms;
collectCommData(idx, comms);
thisProxy[matchPE].ObjMigrated(oData, comms.getVec(), comms.size());
theLbdb->Migrate(oData.handle, matchPE);
// TODO modify LDStats
DEBUGF(("myStats.removeObject: %d, %d, %d\n", migs[i], i, objData.size()));
myStats.removeObject(idx);
}
#endif
}
void HybridBaseLB::CollectInfo(Location *loc, int n, int fromlevel)
{
int atlevel = fromlevel + 1;
LevelData *lData = levelData[atlevel];
lData->info_recved++;
CkVec<Location> &matchedObjs = lData->matchedObjs;
std::map<LDObjKey, int> &unmatchedObjs = lData->unmatchedObjs;
// sort into matched and unmatched list
#if 0
for (int i=0; i<n; i++) {
// search and see if we have answer, put to matched
// store in unknown
int found = 0;
for (int obj=0; obj<unmatchedObjs.size(); obj++) {
if (loc[i].key == unmatchedObjs[obj].key) {
// answer must exist
CmiAssert(unmatchedObjs[obj].loc != -1 || loc[i].loc != -1);
if (unmatchedObjs[obj].loc == -1) unmatchedObjs[obj].loc = loc[i].loc;
matchedObjs.push_back(unmatchedObjs[obj]);
unmatchedObjs.remove(obj);
found = 1;
break;
}
}
if (!found) unmatchedObjs.push_back(loc[i]);
}
#else
for (int i=0; i<n; i++) {
std::map<LDObjKey, int>::iterator iter = unmatchedObjs.find(loc[i].key);
if (iter != unmatchedObjs.end()) {
CmiAssert(iter->second != -1 || loc[i].loc != -1);
if (loc[i].loc == -1) loc[i].loc = iter->second;
matchedObjs.push_back(loc[i]);
unmatchedObjs.erase(iter);
}
else
unmatchedObjs[loc[i].key] = loc[i].loc;
}
#endif
DEBUGF(("[%d] level %d has %d unmatched and %d matched. \n", CkMyPe(), atlevel, unmatchedObjs.size(), matchedObjs.size()));
if (lData->info_recved == lData->nChildren) {
lData->info_recved = 0;
if (_lb_args.debug() > 1)
CkPrintf("[%d] CollectInfo at level %d started at %f\n",
CkMyPe(), atlevel, CkWallTimer());
if (lData->parent != -1) {
// send only unmatched ones up the tree
CkVec<Location> unmatchedbuf;
for(std::map<LDObjKey, int>::const_iterator it = unmatchedObjs.begin(); it != unmatchedObjs.end(); ++it)
{
unmatchedbuf.push_back(Location(it->first, it->second));
}
thisProxy[lData->parent].CollectInfo(unmatchedbuf.getVec(), unmatchedbuf.size(), atlevel);
}
else { // root
// we should have all answers now
CmiAssert(unmatchedObjs.size() == 0);
// start send match list down
thisProxy.PropagateInfo(matchedObjs.getVec(), matchedObjs.size(), atlevel, lData->nChildren, lData->children);
lData->statsData->clear();
}
}
}
// migrate only object LDStat in group
// leaf nodes actually migrate objects
void HybridBaseLB::ReceiveMigration(LBMigrateMsg *msg)
{
#if CMK_LBDB_ON
#if CMK_MEM_CHECKPOINT
CkResetInLdb();
#endif
FindNeighbors();
int atlevel = msg->level - 1;
DEBUGF(("[%d] ReceiveMigration\n", CkMyPe()));
LevelData *lData = levelData[atlevel];
// only non NULL when level > 0
LDStats *statsData = lData->statsData;
// do LDStats migration
const int me = CkMyPe();
lData->migrates_expected = 0;
for(int i=0; i < msg->n_moves; i++) {
MigrateInfo& move = msg->moves[i];
// incoming
if (move.from_pe != me && move.to_pe == me) {
// I can not be the parent node
DEBUGF(("[%d] expecting LDStats object from %d\n",me,move.from_pe));
// will receive a ObjData message
lData->migrates_expected ++;
}
else if (move.from_pe == me) { // outgoing
if (statsData) { // this is inner node
// send objdata
int obj;
int found = 0;
for (obj = 0; obj<statsData->n_objs; obj++) {
if (move.obj.omID() == statsData->objData[obj].handle.omID() &&
move.obj.objID() == statsData->objData[obj].handle.objID())
{
DEBUGF(("[%d] level: %d sending objData %d to %d. \n", CkMyPe(), atlevel, obj, move.to_pe));
found = 1;
// TODO send comm data
CkVec<LDCommData> comms;
collectCommData(obj, comms, atlevel);
if (move.to_pe != -1) {
// this object migrates to another PE of the parent domain
thisProxy[move.to_pe].ObjMigrated(statsData->objData[obj], comms.getVec(), comms.size(), atlevel);
}
lData->outObjs.push_back(MigrationRecord(move.obj, lData->children[statsData->from_proc[obj]], -1));
statsData->removeObject(obj);
break;
}
}
CmiAssert(found == 1);
}
else { // this is leave node
if (move.to_pe == -1) {
lData->outObjs.push_back(MigrationRecord(move.obj, CkMyPe(), -1));
}
else {
// migrate the object
theLbdb->Migrate(move.obj,move.to_pe);
}
}
} // end if
}
if (lData->migrationDone())
StatsDone(atlevel);
#endif
}
// LDStats data sent to parent contains real PE
// LDStats in parent should contain relative PE
void HybridBaseLB::Loadbalancing(int atlevel)
{
int i;
CmiAssert(atlevel >= 1);
CmiAssert(tree->isroot(CkMyPe(), atlevel));
LevelData *lData = levelData[atlevel];
LDStats *statsData = lData->statsData;
CmiAssert(statsData);
// at this time, all objects processor location is relative, and
// all incoming objects from outside group belongs to the fake root proc.
// clear background load if needed
if (_lb_args.ignoreBgLoad()) statsData->clearBgLoad();
currentLevel = atlevel;
int nclients = lData->nChildren;
DEBUGF(("[%d] Calling Strategy ... \n", CkMyPe()));
double start_lb_time, strat_end_time;
start_lb_time = CkWallTimer();
if ((statsStrategy == SHRINK || statsStrategy == SHRINK_NULL) && atlevel == tree->numLevels()-1) {
// no obj and comm data
LBVectorMigrateMsg* migrateMsg = VectorStrategy(statsData);
strat_end_time = CkWallTimer();
// send to children
thisProxy.ReceiveVectorMigration(migrateMsg, nclients, lData->children);
}
else {
LBMigrateMsg* migrateMsg = Strategy(statsData);
strat_end_time = CkWallTimer();
// send to children
//CmiPrintf("[%d] level: %d nclients:%d children: %d %d\n", CkMyPe(), atlevel, nclients, lData->children[0], lData->children[1]);
if (!group1_created)
thisProxy.ReceiveMigration(migrateMsg, nclients, lData->children);
else {
// send in multicast tree
thisProxy.ReceiveMigration(migrateMsg, group1);
//CkSendMsgBranchGroup(CkIndex_HybridBaseLB::ReceiveMigration(NULL), migrateMsg, thisgroup, group1);
}
// CkPrintf("[%d] ReceiveMigration takes %f \n", CkMyPe(), CkWallTimer()-strat_end_time);
}
if (_lb_args.debug()>0){
CkPrintf("[%d] Loadbalancing Level %d (%d children) started at %f, elapsed time %f\n", CkMyPe(), atlevel, lData->nChildren, start_lb_time, strat_end_time-start_lb_time);
if (atlevel == tree->numLevels()-1) {
CkPrintf("[%d] %s memUsage: %.2fKB\n", CkMyPe(), lbName(), (1.0*useMem())/1024);
}
}
// inform new objects that are from outside group
if (atlevel < tree->numLevels()-1) {
for (i=0; i<statsData->n_objs; i++) {
CmiAssert(statsData->from_proc[i] != -1); // ???
if (statsData->from_proc[i] == nclients) { // from outside
CmiAssert(statsData->to_proc[i] < nclients);
int tope = lData->children[statsData->to_proc[i]];
// comm data
CkVec<LDCommData> comms;
// collectCommData(i, comms, atlevel);
thisProxy[tope].ObjMigrated(statsData->objData[i], comms.getVec(), comms.size(), atlevel-1);
}
}
}
}
void FEM_REFINE2D_Split(int meshID,int nodeID,double *coord,int elemID,double *desiredAreas,int sparseID){
int nnodes = FEM_Mesh_get_length(meshID,nodeID);
int nelems = FEM_Mesh_get_length(meshID,elemID);
int actual_nodes = nnodes, actual_elems = nelems;
FEM_Refine_Operation_Data refine_data;
refine_data.meshID = meshID;
refine_data.nodeID = nodeID;
refine_data.sparseID = sparseID;
refine_data.elemID = elemID;
refine_data.cur_nodes = FEM_Mesh_get_length(meshID,nodeID);
/*Copy the cordinates of the nodes into a vector,
the cordinates of the new nodes will be inserted
into this vector and will be used to sort all the
nodes on the basis of the distance from origin
*/
CkVec<double> coordVec;
for(int i=0;i<nnodes*2;i++){
coordVec.push_back(coord[i]);
}
refine_data.coordVec = &coordVec;
refine_data.coord = coord;
/*find out the attributes of the node */
FEM_Entity *e=refine_data.node = FEM_Entity_lookup(meshID,nodeID,"REFINE2D_Mesh");
CkVec<FEM_Attribute *> *attrs = refine_data.attrs = e->getAttrVec();
/*
FEM_DataAttribute *boundaryAttr = (FEM_DataAttribute *)e->lookup(FEM_BOUNDARY,"split");
if(boundaryAttr != NULL){
AllocTable2d<int> &boundaryTable = boundaryAttr->getInt();
printf(" Node Boundary flags \n");
for(int i=0;i<nnodes;i++){
printf("Node %d flag %d \n",i,boundaryTable[i][0]);
}
}
*/
FEM_Entity *elem = refine_data.elem = FEM_Entity_lookup(meshID,elemID,"REFIN2D_Mesh_elem");
CkVec<FEM_Attribute *> *elemattrs = refine_data.elemattrs = elem->getAttrVec();
FEM_Attribute *connAttr = elem->lookup(FEM_CONN,"REFINE2D_Mesh");
if(connAttr == NULL){
CkAbort("Grrrr element without connectivity \n");
}
AllocTable2d<int> &connTable = ((FEM_IndexAttribute *)connAttr)->get();
refine_data.connTable = &connTable;
//hashtable to store the new node number as a function of the two old numbers
CkHashtableT<intdual,int> newnodes(nnodes);
refine_data.newnodes = &newnodes;
/* Get the FEM_BOUNDARY data of sparse elements and load it into a hashtable
indexed by the 2 node ids that make up the edge. The data in the hashtable
is the index number of the sparse element */
FEM_Entity *sparse;
CkVec<FEM_Attribute *> *sparseattrs;
FEM_Attribute *sparseConnAttr, *sparseBoundaryAttr;
AllocTable2d<int> *sparseConnTable, *sparseBoundaryTable;
CkHashtableT<intdual,int> nodes2sparse(nelems);
refine_data.nodes2sparse = &nodes2sparse;
if(sparseID != -1){
sparse = refine_data.sparse = FEM_Entity_lookup(meshID,sparseID,"REFINE2D_Mesh_sparse");
refine_data.sparseattrs = sparseattrs = sparse->getAttrVec();
refine_data.sparseConnAttr = sparseConnAttr = sparse->lookup(FEM_CONN,"REFINE2D_Mesh_sparse");
sparseConnTable = &(((FEM_IndexAttribute *)sparseConnAttr)->get());
refine_data.sparseBoundaryAttr = sparseBoundaryAttr = sparse->lookup(FEM_BOUNDARY,"REFINE2D_Mesh_sparse");
if(sparseBoundaryAttr == NULL){
CkAbort("Specified sparse elements without boundary conditions");
}
FEM_DataAttribute *validEdgeAttribute = (FEM_DataAttribute *)sparse->lookup(FEM_VALID,"REFINE2D_Mesh_sparse");
if(validEdgeAttribute){
refine_data.validEdge = &(validEdgeAttribute->getInt());
}else{
refine_data.validEdge = NULL;
}
/* since the default value in the hashtable is 0, to distinguish between
uninserted keys and the sparse element with index 0, the index of the
sparse elements is incremented by 1 while inserting. */
// printf("[%d] Sparse elements\n",FEM_My_partition());
for(int j=0;j<sparse->size();j++){
if(refine_data.validEdge == NULL || (*(refine_data.validEdge))[j][0]){
int *cdata = (*sparseConnTable)[j];
// printf("%d < %d,%d > \n",j,cdata[0],cdata[1]);
nodes2sparse.put(intdual(cdata[0],cdata[1])) = j+1;
}
}
}else{
printf("Edge boundary conditions not passed into FEM_REFINE2D_Split \n");
}
//count the actual number of nodes and elements
if(refine_data.node->lookup(FEM_VALID,"refine2D_splilt") != NULL){
AllocTable2d<int> &validNodeTable = ((FEM_DataAttribute *)(refine_data.node->lookup(FEM_VALID,"refine2D_splilt")))->getInt();
actual_nodes = countValidEntities(validNodeTable.getData(),nnodes);
}
if(refine_data.elem->lookup(FEM_VALID,"refine2D_splilt") != NULL){
AllocTable2d<int> &validElemTable = ((FEM_DataAttribute *)(refine_data.elem->lookup(FEM_VALID,"refine2D_splilt")))->getInt();
actual_elems = countValidEntities(validElemTable.getData(),nelems);
}
DEBUGINT(printf("%d %d \n",nnodes,nelems));
REFINE2D_Split(actual_nodes,coord,actual_elems,desiredAreas,&refine_data);
int nSplits= refine_data.nSplits = REFINE2D_Get_Split_Length();
DEBUGINT(printf("called REFINE2D_Split nSplits = %d \n",nSplits));
if(nSplits == 0){
return;
}
for(int split = 0;split < nSplits;split++){
refineData op;
REFINE2D_Get_Split(split,&op);
FEM_Refine_Operation(&refine_data,op);
}
DEBUGINT(printf("Cordinate list length %d according to FEM %d\n",coordVec.size()/2,FEM_Mesh_get_length(meshID,nodeID)));
IDXL_Sort_2d(FEM_Comm_shared(meshID,nodeID),coordVec.getVec());
int read = FEM_Mesh_is_get(meshID) ;
assert(read);
}