Пример #1
0
/** Extract an IDXL_Side_t into Roccom format. */
static void getRoccomPconn(IDXL_Side_t is,int bias,CkVec<int> &pconn,const int *paneFmChunk)
{
	int p,np=IDXL_Get_partners(is);
	pconn.push_back(np);
	for (p=0;p<np;p++) {
		int chunk=IDXL_Get_partner(is,p);
		int pane=1+chunk;
		if(paneFmChunk) pane=paneFmChunk[chunk];
		pconn.push_back(pane);
		int n,nn=IDXL_Get_count(is,p);
		pconn.push_back(nn); /* number of shared nodes */
		for (n=0;n<nn;n++)
			pconn.push_back(IDXL_Get_index(is,p,n)+1+bias); /* nodes are 1-based */
	}
}
Пример #2
0
  //Find next un-occupied global number:
  int nextUnoccupied(void) {
    while (occupiedBefore<occupied.size()) {
      if (occupied[occupiedBefore]==0) 
	return occupiedBefore;
      else occupiedBefore++;
    }
    /* occupiedBefore==occupied.size(), so add to end of list */
    occupied.push_back(1);
    return occupiedBefore;
  }
Пример #3
0
// find sender comms
void HybridBaseLB::collectCommData(int objIdx, CkVec<LDCommData> &comms, int atlevel)
{
  LevelData *lData = levelData[atlevel];
  LDStats *statsData = lData->statsData;

  LDObjData &objData = statsData->objData[objIdx];

  for (int com=0; com<statsData->n_comm; com++) {
    LDCommData &cdata = statsData->commData[com];
    if (cdata.from_proc()) continue;
    if (cdata.sender.objID() == objData.objID() && cdata.sender.omID() == objData.omID())
      comms.push_back(cdata);
  }
}
Пример #4
0
static void readClause(StreamBuffer& in, par_SolverState& S, CkVec<par_Lit>& lits) {
    int     parsed_lit, var;
    lits.removeAll();
    for (;;){
        parsed_lit = parseInt(in);
        if (parsed_lit == 0) break;
        var = abs(parsed_lit)-1;
        
        S.occurrence[var]++;
        if(parsed_lit>0)
            S.positive_occurrence[var]++;
        lits.push_back( par_Lit(parsed_lit));
    }
}
Пример #5
0
  /// Mark this entity's global numbers as used:
  void mark(FEM_Entity &src) {
    int l,len=src.size();
    for (l=0;l<len;l++) {
      int g=src.getGlobalno(l);
      if (g!=-1) {
	while (occupied.size()<=g) { //Make room for this marker
	  occupied.push_back(0);
	}
	//FIXME: make sure element global numbers aren't repeated
	// (tough because *node* global numbers may be repeated)
	occupied[g]=1;
      }
    }
  }
Пример #6
0
void Patch::localCreateSection() {
#ifdef USE_SECTION_MULTICAST
  CkVec<CkArrayIndex6D> elems;
  for (int num=0; num<numNbrs; num++)
    elems.push_back(CkArrayIndex6D(computesList[num][0], computesList[num][1], computesList[num][2], computesList[num][3], computesList[num][4], computesList[num][5]));

  CkArrayID computeArrayID = computeArray.ckGetArrayID();
  mCastSecProxy = CProxySection_Compute::ckNew(computeArrayID, elems.getVec(), elems.size()); 

  CkMulticastMgr *mCastGrp = CProxy_CkMulticastMgr(mCastGrpID).ckLocalBranch();
  mCastSecProxy.ckSectionDelegate(mCastGrp);
  mCastGrp->setReductionClient(mCastSecProxy, new CkCallback(CkIndex_Patch::reduceForces(NULL), thisProxy(thisIndex.x, thisIndex.y, thisIndex.z)));

#endif
}
Пример #7
0
// find sender comms
void HbmLB::collectCommData(int objIdx, CkVec<LDCommData> &comms)
{
#if CMK_LBDB_ON
  LevelData *lData = levelData[0];

  LDObjData &objData = myStats.objData[objIdx];

  for (int com=0; com<myStats.n_comm; com++) {
    LDCommData &cdata = myStats.commData[com];
    if (cdata.from_proc()) continue;
    if (cdata.sender.objID() == objData.objID() && cdata.sender.omID() == objData.omID())
      comms.push_back(cdata);
  }
#endif
}
Пример #8
0
// pick objects to migrate "t" amount of work
void HbmLB::ReceiveMigrationDelta(double t, int lblevel, int fromlevel)
{
#if CMK_LBDB_ON
  int i;
  int atlevel = fromlevel-1;
  LevelData *lData = levelData[atlevel];
  if (atlevel != 0) {
    thisProxy.ReceiveMigrationDelta(t, lblevel, atlevel, lData->nChildren, lData->children);
    return;
  }

  // I am leave, find objects to migrate

  CkVec<int> migs;
  CkVec<LDObjData> &objData = myStats.objData;
  for (i=0; i<myStats.n_objs; i++) {
    LDObjData &oData = objData[i];
    if (oData.wallTime < t) {
      migs.push_back(i);
      t -= oData.wallTime;
      if (t == 0.0) break;
    }
  }

  int nmigs = migs.size();
  // send a message to 
  int matchPE = CkMyPe() ^ (1<<(lblevel-1));
  
  DEBUGF(("[%d] migrating %d objs to %d at lblevel %d! \n", CkMyPe(),nmigs,matchPE,lblevel));
  thisProxy[matchPE].ReceiveMigrationCount(nmigs, lblevel);

  // migrate objects
  for (i=0; i<nmigs; i++) {
    int idx = migs[i]-i;
    LDObjData &oData = objData[idx];
    CkVec<LDCommData> comms;
    collectCommData(idx, comms);
    thisProxy[matchPE].ObjMigrated(oData, comms.getVec(), comms.size());
    theLbdb->Migrate(oData.handle, matchPE);
    // TODO modify LDStats
    DEBUGF(("myStats.removeObject: %d, %d, %d\n", migs[i], i, objData.size()));
    myStats.removeObject(idx);
  }
#endif
}
Пример #9
0
// read from a file called "globals"
static void readGlobals()
{
  if (loaded) return;
  const char *fname = "globals";
  printf("Loading globals from file \"%s\" ... \n", fname);
  FILE *gf = fopen(fname, "r");
  if (gf == NULL) {
    CmiAbort("Failed to load globals, file may not exist!");
  }
  while (!feof(gf)) 
  {
    char name[1024];
    fscanf(gf, "%s\n", name);
    _namelist.push_back(strdup(name));
  }
  fclose(gf);
  loaded = 1;
}
Пример #10
0
static void readBlacklist()
{
  if (loaded) return;
  const char *fname = "blacklist";
  FILE *bl = fopen(fname, "r");
  if (bl == NULL){
		if (CmiMyPe() == 0) printf("WARNING: Running swapglobals without blacklist, globals from libraries might be getting un-necessarily swapped\n");
		loaded = 1;
		return;
  }
  printf("Loading blacklist from file \"%s\" ... \n", fname);
  while (!feof(bl)){
    char name[512];
    fscanf(bl, "%s\n", name);
     _blacklist.push_back(strdup(name));
  }
  fclose(bl);
  loaded = 1;
}
Пример #11
0
void HybridBaseLB::GetObjsToMigrate(int toPe, double load, LDStats *stats, int atlevel,
    CkVec<LDCommData>& comms, CkVec<LDObjData>& objs) {
  // TODO: sort max => low
  for (int obj=stats->n_objs-1; obj>=0; obj--) {
    LDObjData &objData = stats->objData[obj];
    if (!objData.migratable) continue;
    if (objData.wallTime <= load) {
      if (_lb_args.debug()>2) {
        CkPrintf("[%d] send obj: %d to PE %d (load: %f).\n", CkMyPe(), obj, toPe,
          objData.wallTime);
      }
      objs.push_back(objData);
      // send comm data
      collectCommData(obj, comms, atlevel);
      load -= objData.wallTime;
      CreateMigrationOutObjs(atlevel, stats, obj);
      stats->removeObject(obj);
      if (load <= 0.0) break;
    }
  }
}
Пример #12
0
    // return -1 when not supported
  int numUniqNodes() {
#if 0
    if (numNodes != 0) return numNodes;
    int n = 0;
    for (int i=0; i<CmiNumPes(); i++) 
      if (nodeIDs[i] > n)
	n = nodeIDs[i];
    numNodes = n+1;
    return numNodes;
#else
    if (numNodes > 0) return numNodes;     // already calculated
    CkVec<int> unodes;
    int i;
    for (i=0; i<numPes; i++)  unodes.push_back(nodeIDs[i]);
    //unodes.bubbleSort(0, numPes-1);
    unodes.quickSort();
    int last = -1;
    std::map<int, int> nodemap;  // nodeIDs can be out of range of [0,numNodes]
    for (i=0; i<numPes; i++)  { 
        if (unodes[i] != last) {
          last=unodes[i];
	  nodemap[unodes[i]] = numNodes;
          numNodes++; 
        }
    }
    if (numNodes == 0)  {
      numNodes = CmiNumNodes();
      numPes = CmiNumPes();
    }
    else {
        // re-number nodeIDs, which may be necessary e.g. on BlueGene/P
      for (i=0; i<numPes; i++) nodeIDs[i] = nodemap[nodeIDs[i]];
      CpuTopology::supported = 1;
    }
    return numNodes;
#endif
  }
Пример #13
0
void BaseLB::LDStats::computeNonlocalComm(int &nmsgs, int &nbytes)
{
#if CMK_LBDB_ON
    	nmsgs = 0;
	nbytes = 0;

	makeCommHash();

	int mcast_count = 0;
        for (int cidx=0; cidx < n_comm; cidx++) {
	    LDCommData& cdata = commData[cidx];
	    int senderPE, receiverPE;
	    if (cdata.from_proc())
	      senderPE = cdata.src_proc;
  	    else {
	      int idx = getHash(cdata.sender);
	      if (idx == -1) continue;    // sender has just migrated?
	      senderPE = to_proc[idx];
	      CmiAssert(senderPE != -1);
	    }
	    CmiAssert(senderPE < nprocs() && senderPE >= 0);

            // find receiver: point-to-point and multicast two cases
	    int receiver_type = cdata.receiver.get_type();
	    if (receiver_type == LD_PROC_MSG || receiver_type == LD_OBJ_MSG) {
              if (receiver_type == LD_PROC_MSG)
	        receiverPE = cdata.receiver.proc();
              else  {  // LD_OBJ_MSG
	        int idx = getHash(cdata.receiver.get_destObj());
		if (idx == -1) {		// receiver outside this domain
		  if (complete_flag) continue;
		  else receiverPE = -1;
		}
		else {
	          receiverPE = to_proc[idx];
                  CmiAssert(receiverPE < nprocs() && receiverPE >= 0);
		}
              }
	      if(senderPE != receiverPE)
	      {
	  	nmsgs += cdata.messages;
		nbytes += cdata.bytes;
	      }
	    }
            else if (receiver_type == LD_OBJLIST_MSG) {
              int nobjs;
              LDObjKey *objs = cdata.receiver.get_destObjs(nobjs);
	      mcast_count ++;
	      CkVec<int> pes;
	      for (int i=0; i<nobjs; i++) {
	        int idx = getHash(objs[i]);
		CmiAssert(idx != -1);
	        if (idx == -1) continue;    // receiver has just been removed?
	        receiverPE = to_proc[idx];
		CmiAssert(receiverPE < nprocs() && receiverPE >= 0);
		int exist = 0;
	        for (int p=0; p<pes.size(); p++) 
		  if (receiverPE == pes[p]) { exist=1; break; }
		if (exist) continue;
		pes.push_back(receiverPE);
	        if(senderPE != receiverPE)
	        {
	  	  nmsgs += cdata.messages;
		  nbytes += cdata.bytes;
	        }
              }
	    }
	}   // end of for
#endif
}
Пример #14
0
/** Perform a Delaunay flip of the edge (n1, n2) returning 1 if successful, 0 if
   not (likely due to the edge being on a boundary). The convexity of the 
   quadrilateral formed by two faces incident to edge (n1, n2) is assumed. n1 
   and n2 are assumed to be local to this chunk.  An adjacency test is 
   performed on n1 and n2 by searching for an element with edge [n1,n2].

       n3                 n3
        o                  o
       / \                /|\
      /   \              / | \
     /     \            /  |  \
    /       \          /   |   \
n1 o---------o n2  n1 o    |    o n2
    \       /          \   |   / 
     \     /            \  |  /
      \   /              \ | /
       \ /                \|/
        o                  o
       n4                 n4
*/
int FEM_Adapt::edge_flip_help(int e1, int e2, int n1, int n2, int e1_n1, 
			      int e1_n2, int e1_n3, int n3, int n4, int *locknodes) 
{
  int numNodes = 4;
  int numElems = 2;
  int lockelems[2];
  int elemConn[3];
  locknodes[0] = n1;
  locknodes[1] = n2;
  locknodes[2] = n3;
  locknodes[3] = n4;
  lockelems[0] = e1;
  lockelems[1] = e2;
  if(n1 < 0 || n2 < 0) {
    return -1;
  }
  int index = theMod->getIdx();
  bool flag = theMod->fmAdaptAlgs->controlQualityF(n1,n2,n3,n4);
  if(flag) return -1;
  int e1Topurge = e1;
  int e2Topurge = e2;

#ifdef DEBUG_1
  CkPrintf("Flipping edge %d->%d on chunk %d\n", n1, n2, theMod->getfmUtil()->getIdx());
#endif
  //FEM_Modify_Lock(theMesh, locknodes, numNodes, lockelems, numElems);
  //if any of the two elements is remote, eat those
  if(n3 < 0) {
    e1Topurge = theMod->fmUtil->eatIntoElement(e1);
    theMesh->e2n_getAll(e1Topurge,elemConn);
    for(int i=0; i<3; i++) {
      if(elemConn[i]!=n1 && elemConn[i]!=n2) {
	n3 = elemConn[i];
      }
    }
    locknodes[2] = n3;
  }
  if(n4 < 0) {
    e2Topurge = theMod->fmUtil->eatIntoElement(e2);
    theMesh->e2n_getAll(e2Topurge,elemConn);
    for(int i=0; i<3; i++) {
      if(elemConn[i]!=n1 && elemConn[i]!=n2) {
	n4 = elemConn[i];
      }
    }
    locknodes[3] = n4;
  }

  FEM_remove_element(theMesh,e1Topurge,0,0);
  FEM_remove_element(theMesh,e2Topurge,0,0);
  // add n1, n3, n4
  elemConn[e1_n1] = n1;  elemConn[e1_n2] = n4;  elemConn[e1_n3] = n3;
  lockelems[0] = FEM_add_element(theMesh, elemConn, 3, 0, index);
  //the attributes should really be interpolated, i.e. on both new elems,
  //the values should be an average of the previous two elements
  theMod->fmUtil->copyElemData(0,e1Topurge,lockelems[0]);
  // add n2, n3, n4
  elemConn[e1_n1] = n4;  elemConn[e1_n2] = n2;  elemConn[e1_n3] = n3;
  lockelems[1] = FEM_add_element(theMesh, elemConn, 3, 0, index);
  theMod->fmUtil->copyElemData(0,e1Topurge,lockelems[1]); //both of the new elements copy from one element
  //purge the two elements
  FEM_purge_element(theMesh,e1Topurge,0);
  FEM_purge_element(theMesh,e2Topurge,0);

  //get rid of some unnecessary ghost node sends
  for(int i=0; i<4;i++) {
    int nodeToUpdate = -1;
    if(i==0) nodeToUpdate = n1;
    else if(i==1) nodeToUpdate = n2;
    else if(i==2) nodeToUpdate = n3;
    else if(i==3) nodeToUpdate = n4;
    //if any of the chunks sharing this node sends this as a ghost, then all of them have to
    //so find out the set of chunks I need to send this as a ghost to
    //collect info from each of the shared chunks, do a union of all these chunks
    //send this updated list to everyone.
    //if anyone needs to add or delete some ghosts, they will
    int *chkl, numchkl=0;
    CkVec<int> finalchkl;
    theMod->fmUtil->findGhostSend(nodeToUpdate, chkl, numchkl);
    for(int j=0; j<numchkl; j++) {
      finalchkl.push_back(chkl[j]);
    }
    if(numchkl>0) delete[] chkl;

    const IDXL_Rec *irec=theMesh->node.shared.getRec(nodeToUpdate);
    int numchunks=0;
    int *chunks1, *inds1;
    if(irec) {
      numchunks = irec->getShared();
      chunks1 = new int[numchunks];
      inds1 = new int[numchunks];
      for(int j=0; j<numchunks; j++) {
	chunks1[j] = irec->getChk(j);
	inds1[j] = irec->getIdx(j);
      }
    }
    for(int j=0; j<numchunks; j++) {
      findgsMsg *fmsg = meshMod[chunks1[j]].findghostsend(index,inds1[j]);
      if(fmsg->numchks>0) {
	for(int k=0; k<fmsg->numchks; k++) {
	  bool shouldbeadded = true;
	  for(int l=0; l<finalchkl.size(); l++) {
	    if(fmsg->chunks[k]==finalchkl[l]) {
	      shouldbeadded = false;
	      break;
	    }
	  }
	  if(shouldbeadded) finalchkl.push_back(fmsg->chunks[k]);
	}
      }
      delete fmsg;
    }

    int *finall, numfinall=finalchkl.size();
    if(numfinall>0) finall = new int[numfinall];
    for(int j=0; j<numfinall; j++) finall[j] = finalchkl[j];
    finalchkl.free();

    theMod->fmUtil->UpdateGhostSend(nodeToUpdate, finall, numfinall);
    for(int j=0; j<numchunks; j++) {
      verifyghostsendMsg *vmsg = new(numfinall)verifyghostsendMsg();
      vmsg->fromChk = index;
      vmsg->sharedIdx = inds1[j];
      vmsg->numchks = numfinall;
      for(int k=0; k<numfinall; k++) vmsg->chunks[k] = finall[k];
      meshMod[chunks1[j]].updateghostsend(vmsg);
    }
    if(numfinall>0) delete[] finall;
    if(numchunks>0) {
      delete[] chunks1;
      delete[] inds1;
    }
  }
  //make sure that it always comes here, don't return with unlocking
  return 1; //return newNode;
}
Пример #15
0
void LBInfo::getInfo(BaseLB::LDStats* stats, int count, int considerComm)
{
#if CMK_LBDB_ON
	int i, pe;

	CmiAssert(peLoads);

	clear();

	double alpha = _lb_args.alpha();
	double beeta = _lb_args.beeta();

        minObjLoad = 1.0e20;  // I suppose no object load is beyond this
	maxObjLoad = 0.0;

    	msgCount = 0;
	msgBytes = 0;

	if (considerComm) stats->makeCommHash();

        // get background load
	if (bgLoads)
    	  for(pe = 0; pe < count; pe++)
    	   bgLoads[pe] = stats->procs[pe].bg_walltime;

	for(pe = 0; pe < count; pe++)
    	  peLoads[pe] = stats->procs[pe].bg_walltime;

    	for(int obj = 0; obj < stats->n_objs; obj++)
    	{
		int pe = stats->to_proc[obj];
		if (pe == -1) continue;     // this object is out
		CmiAssert(pe >=0 && pe < count);
		double oload = stats->objData[obj].wallTime;
		if (oload < minObjLoad) minObjLoad = oload;
		if (oload > maxObjLoad) maxObjLoad = oload;
		peLoads[pe] += oload;
		if (objLoads) objLoads[pe] += oload;
	}

	// handling of the communication overheads. 
	if (considerComm) {
	  int* msgSentCount = new int[count]; // # of messages sent by each PE
	  int* msgRecvCount = new int[count]; // # of messages received by each PE
	  int* byteSentCount = new int[count];// # of bytes sent by each PE
	  int* byteRecvCount = new int[count];// # of bytes reeived by each PE
	  for(i = 0; i < count; i++)
	    msgSentCount[i] = msgRecvCount[i] = byteSentCount[i] = byteRecvCount[i] = 0;

	  int mcast_count = 0;
          for (int cidx=0; cidx < stats->n_comm; cidx++) {
	    LDCommData& cdata = stats->commData[cidx];
	    int senderPE, receiverPE;
	    if (cdata.from_proc())
	      senderPE = cdata.src_proc;
  	    else {
	      int idx = stats->getHash(cdata.sender);
	      if (idx == -1) continue;    // sender has just migrated?
	      senderPE = stats->to_proc[idx];
	      CmiAssert(senderPE != -1);
	    }
	    CmiAssert(senderPE < count && senderPE >= 0);

            // find receiver: point-to-point and multicast two cases
	    int receiver_type = cdata.receiver.get_type();
	    if (receiver_type == LD_PROC_MSG || receiver_type == LD_OBJ_MSG) {
              if (receiver_type == LD_PROC_MSG)
	        receiverPE = cdata.receiver.proc();
              else  {  // LD_OBJ_MSG
	        int idx = stats->getHash(cdata.receiver.get_destObj());
	        if (idx == -1) continue;    // receiver has just been removed?
	        receiverPE = stats->to_proc[idx];
	        CmiAssert(receiverPE != -1);
              }
              CmiAssert(receiverPE < count && receiverPE >= 0);
	      if(senderPE != receiverPE)
	      {
	  	msgSentCount[senderPE] += cdata.messages;
		byteSentCount[senderPE] += cdata.bytes;
		msgRecvCount[receiverPE] += cdata.messages;
		byteRecvCount[receiverPE] += cdata.bytes;
	      }
	    }
            else if (receiver_type == LD_OBJLIST_MSG) {
              int nobjs;
              LDObjKey *objs = cdata.receiver.get_destObjs(nobjs);
	      mcast_count ++;
	      CkVec<int> pes;
	      for (i=0; i<nobjs; i++) {
	        int idx = stats->getHash(objs[i]);
		CmiAssert(idx != -1);
	        if (idx == -1) continue;    // receiver has just been removed?
	        receiverPE = stats->to_proc[idx];
		CmiAssert(receiverPE < count && receiverPE >= 0);
		int exist = 0;
	        for (int p=0; p<pes.size(); p++) 
		  if (receiverPE == pes[p]) { exist=1; break; }
		if (exist) continue;
		pes.push_back(receiverPE);
	        if(senderPE != receiverPE)
	        {
	  	msgSentCount[senderPE] += cdata.messages;
		byteSentCount[senderPE] += cdata.bytes;
		msgRecvCount[receiverPE] += cdata.messages;
		byteRecvCount[receiverPE] += cdata.bytes;
	        }
              }
	    }
	  }   // end of for
          if (_lb_args.debug())
             CkPrintf("Number of MULTICAST: %d\n", mcast_count);

	  // now for each processor, add to its load the send and receive overheads
	  for(i = 0; i < count; i++)
	  {
		double comload = msgRecvCount[i]  * PER_MESSAGE_RECV_OVERHEAD +
			      msgSentCount[i]  * alpha +
			      byteRecvCount[i] * PER_BYTE_RECV_OVERHEAD +
			      byteSentCount[i] * beeta;
		peLoads[i] += comload;
		if (comLoads) comLoads[i] += comload;
		msgCount += msgRecvCount[i] + msgSentCount[i];
		msgBytes += byteRecvCount[i] + byteSentCount[i];
	  }
	  delete [] msgRecvCount;
	  delete [] msgSentCount;
	  delete [] byteRecvCount;
	  delete [] byteSentCount;
	}
#endif
}
Пример #16
0
/** Given edge e:(n1, n2), remove the two elements (n1,n2,n3) and 
   (n2,n1,n4) adjacent to e, and bisect e by adding node 
   n5. Add elements (n1,n5,n3), (n5,n2,n3), (n5,n1,n4) and (n2,n5,n4); 
   returns new node n5.

       n3                 n3
        o                  o
       / \                /|\
      /   \              / | \
     /     \            /  |  \
    /       \          /   |n5 \
n1 o---------o n2  n1 o----o----o n2
    \       /          \   |   / 
     \     /            \  |  /
      \   /              \ | /
       \ /                \|/
        o                  o
       n4                 n4
*/
int FEM_Adapt::edge_bisect_help(int e1, int e2, int n1, int n2, int e1_n1, 
				int e1_n2, int e1_n3, int e2_n1, int e2_n2, 
				int e2_n3, int n3, int n4)
{
  int n5;
  int numNodes = 4;
  int numElems = 2;
  int numNodesNew = 5;
  int numElemsNew = 4;
  int locknodes[5];
  int lockelems[4];
  int elemConn[3];

  locknodes[0] = n1;
  locknodes[1] = n2;
  locknodes[2] = n3;
  locknodes[3] = n4;
  locknodes[4] = -1;
  lockelems[0] = e1;
  lockelems[1] = e2;
  lockelems[2] = -1;
  lockelems[3] = -1;

  //FEM_Modify_Lock(theMesh, locknodes, numNodes, lockelems, numElems);

  int e1chunk=-1, e2chunk=-1, e3chunk=-1, e4chunk=-1, n5chunk=-1;
  int index = theMod->getIdx();

#ifdef DEBUG_1
  CkPrintf("Bisect edge %d->%d on chunk %d\n", n1, n2, theMod->getfmUtil()->getIdx());
#endif
  //verify quality
  bool flag = theMod->fmAdaptAlgs->controlQualityR(n1,n2,n3,n4);
  if(flag) return -1;

  //add node
  if(e1==-1) e1chunk=-1;
  else if(e1>=0) e1chunk=index;
  else {
    int ghostid = FEM_To_ghost_index(e1);
    const IDXL_Rec *irec = theMesh->elem[0].ghost->ghostRecv.getRec(ghostid);
    CkAssert(irec->getShared()==1);
    e1chunk = irec->getChk(0);
  }
  if(e2==-1) e2chunk=-1;
  else if(e2>=0) e2chunk=index;
  else {
    int ghostid = FEM_To_ghost_index(e2);
    const IDXL_Rec *irec = theMesh->elem[0].ghost->ghostRecv.getRec(ghostid);
    CkAssert(irec->getShared()==1);
    e2chunk = irec->getChk(0);
  }
  int adjnodes[2];
  adjnodes[0] = n1;
  adjnodes[1] = n2;
  int *chunks;
  int numChunks=0;
  int forceshared = 0;
  if(e1chunk==e2chunk || (e1chunk==-1 || e2chunk==-1)) {
    forceshared = -1;
    numChunks = 1;
    chunks = new int[1];
    if(e1chunk!=-1) chunks[0] = e1chunk;
    else chunks[0] = e2chunk;
  }
  else {
    numChunks = 2;
    chunks = new int[2];
    chunks[0] = e1chunk;
    chunks[1] = e2chunk;
  }
  n5 = FEM_add_node(theMesh,adjnodes,2,chunks,numChunks,forceshared);
  delete[] chunks;
  //lock this node immediately
  FEM_Modify_LockN(theMesh, n5, 0);

  //remove elements
  e1chunk = FEM_remove_element(theMesh, e1, 0); 
  e2chunk = FEM_remove_element(theMesh, e2, 0);  // assumes intelligent behavior when no e2 exists

  // hmm... if e2 is a ghost and we remove it and create all the new elements
  // locally, then we don't really need to add a *shared* node
  //but we are not moving chunk boundaries for bisect
  if(e1chunk==-1 || e2chunk==-1) {
    //it is fine, let it continue
    e4chunk = e2chunk;
    e3chunk = e1chunk;
  }
  else if(e1chunk==e2chunk && e1chunk!=index) {
    n5chunk = e1chunk;
    e4chunk = e2chunk;
    e3chunk = e1chunk;
  }
  else {
    //there can be a lot of conditions, but we do not have to do aything special now
    n5chunk = -1;
    e4chunk = e2chunk;
    e3chunk = e1chunk;
  }

  // add n1, n5, n3
  elemConn[e1_n1] = n1;  elemConn[e1_n2] = n5;  elemConn[e1_n3] = n3;
  lockelems[0] = FEM_add_element(theMesh, elemConn, 3, 0, e1chunk);
  theMod->fmUtil->copyElemData(0,e1,lockelems[0]);
  // add n2, n5, n3
  elemConn[e1_n1] = n5;  elemConn[e1_n2] = n2;  elemConn[e1_n3] = n3;
  lockelems[1] = FEM_add_element(theMesh, elemConn, 3, 0, e3chunk);
  theMod->fmUtil->copyElemData(0,e1,lockelems[1]);
  if (e2 != -1) { // e2 exists
    // add n1, n5, n4
    elemConn[e2_n1] = n1;  elemConn[e2_n2] = n5;  elemConn[e2_n3] = n4;
    lockelems[2] = FEM_add_element(theMesh, elemConn, 3, 0, e2chunk);
    theMod->fmUtil->copyElemData(0,e2,lockelems[2]);
    // add n2, n5, n4
    elemConn[e2_n1] = n5;  elemConn[e2_n2] = n2;  elemConn[e2_n3] = n4;
    lockelems[3] = FEM_add_element(theMesh, elemConn, 3, 0, e4chunk);
    theMod->fmUtil->copyElemData(0,e2,lockelems[3]);
  }

  FEM_purge_element(theMesh,e1,0);
  FEM_purge_element(theMesh,e2,0);

  //get rid of some unnecessary ghost node sends
  for(int i=0; i<4;i++) {
    int nodeToUpdate = -1;
    if(i==0) nodeToUpdate = n1;
    else if(i==1) nodeToUpdate = n2;
    else if(i==2) nodeToUpdate = n3;
    else if(i==3) nodeToUpdate = n4;
    //if any of the chunks sharing this node sends this as a ghost, then all of them have to
    //so find out the set of chunks I need to send this as a ghost to
    //collect info from each of the shared chunks, do a union of all these chunks
    //send this updated list to everyone.
    //if anyone needs to add or delete some ghosts, they will
    int *chkl, numchkl=0;
    CkVec<int> finalchkl;
    theMod->fmUtil->findGhostSend(nodeToUpdate, chkl, numchkl);
    for(int j=0; j<numchkl; j++) {
      finalchkl.push_back(chkl[j]);
    }
    if(numchkl>0) delete[] chkl;

    const IDXL_Rec *irec=theMesh->node.shared.getRec(nodeToUpdate);
    int numchunks=0;
    int *chunks1, *inds1;
    if(irec) {
      numchunks = irec->getShared();
      chunks1 = new int[numchunks];
      inds1 = new int[numchunks];
      for(int j=0; j<numchunks; j++) {
	chunks1[j] = irec->getChk(j);
	inds1[j] = irec->getIdx(j);
      }
    }
    for(int j=0; j<numchunks; j++) {
      findgsMsg *fmsg = meshMod[chunks1[j]].findghostsend(index,inds1[j]);
      if(fmsg->numchks>0) {
	for(int k=0; k<fmsg->numchks; k++) {
	  bool shouldbeadded = true;
	  for(int l=0; l<finalchkl.size(); l++) {
	    if(fmsg->chunks[k]==finalchkl[l]) {
	      shouldbeadded = false;
	      break;
	    }
	  }
	  if(shouldbeadded) finalchkl.push_back(fmsg->chunks[k]);
	}
      }
      delete fmsg;
    }

    int *finall, numfinall=finalchkl.size();
    if(numfinall>0) finall = new int[numfinall];
    for(int j=0; j<numfinall; j++) finall[j] = finalchkl[j];
    finalchkl.free();

    theMod->fmUtil->UpdateGhostSend(nodeToUpdate, finall, numfinall);
    for(int j=0; j<numchunks; j++) {
      verifyghostsendMsg *vmsg = new(numfinall)verifyghostsendMsg();
      vmsg->fromChk = index;
      vmsg->sharedIdx = inds1[j];
      vmsg->numchks = numfinall;
      for(int k=0; k<numfinall; k++) vmsg->chunks[k] = finall[k];
      meshMod[chunks1[j]].updateghostsend(vmsg);
    }
    if(numfinall>0) delete[] finall;
    if(numchunks>0) {
      delete[] chunks1;
      delete[] inds1;
    }
  }

  FEM_Modify_UnlockN(theMesh, n5, 0);
  return n5;
}
Пример #17
0
void GreedyCommLB::work(LDStats* stats)
{
    int pe,obj,com;
    ObjectRecord *x;
    int i;
    
    if (_lb_args.debug()) CkPrintf("In GreedyCommLB strategy\n",CkMyPe());
    npe = stats->nprocs();
    nobj = stats->n_objs;

    // nmigobj is calculated as the number of migratable objects
    // ObjectHeap maxh is of size nmigobj
    nmigobj = stats->n_migrateobjs;

    stats->makeCommHash();

    assigned_array = new int[nobj];

    object_graph = new graph[nobj];

    init_data(assigned_array,object_graph,npe,nobj);

#define MAXDOUBLE   1e10;

    // processor heap
    processors = new processorInfo[npe];
    for (int p=0; p<npe; p++) {
      processors[p].Id = p;
      processors[p].backgroundLoad = stats->procs[p].bg_walltime;
      processors[p].computeLoad = 0;
      processors[p].pe_speed = stats->procs[p].pe_speed;
      if (!stats->procs[p].available) {
        processors[p].load = MAXDOUBLE;
      }
      else {
        processors[p].load = 0;
        if (!_lb_args.ignoreBgLoad())
          processors[p].load = processors[p].backgroundLoad;
      }
    }


    // assign communication graph
    for(com =0; com< stats->n_comm;com++) {
         int xcoord=0,ycoord=0;
	 LDCommData &commData = stats->commData[com];
	 if((!commData.from_proc())&&(commData.recv_type()==LD_OBJ_MSG))
	 {
		xcoord = stats->getHash(commData.sender);
		ycoord = stats->getHash(commData.receiver.get_destObj());
		if((xcoord == -1)||(ycoord == -1))
		    if (_lb_args.ignoreBgLoad() || stats->complete_flag==0) continue;
		    else CkAbort("Error in search\n");
		add_graph(xcoord,ycoord,commData.bytes, commData.messages);
	 }
         else if (commData.recv_type()==LD_OBJLIST_MSG) {
		int nobjs;
		LDObjKey *objs = commData.receiver.get_destObjs(nobjs);
		xcoord = stats->getHash(commData.sender);
		for (int i=0; i<nobjs; i++) {
		  ycoord = stats->getHash(objs[i]);
		  if((xcoord == -1)||(ycoord == -1))
		    if (_lb_args.migObjOnly()) continue;
		    else CkAbort("Error in search\n");
//printf("Multicast: %d => %d %d %d\n", xcoord, ycoord, commData.bytes, commData.messages);
		  add_graph(xcoord,ycoord,commData.bytes, commData.messages);
		}
         }
    }

    // only build heap with migratable objects, 
    // mapping nonmigratable objects to the same processors
    ObjectHeap maxh(nmigobj+1);
    for(obj=0; obj < stats->n_objs; obj++) {
      LDObjData &objData = stats->objData[obj];
      int onpe = stats->from_proc[obj];
      if (!objData.migratable) {
        if (!stats->procs[onpe].available) {
	  CmiAbort("Load balancer is not be able to move a nonmigratable object out of an unavailable processor.\n");
        }
        alloc(onpe, obj, objData.wallTime);
	update(stats, obj, onpe);	     // update communication cost on other pes
      }
      else {
        x = new ObjectRecord;
        x->id = obj;
        x->pos = obj;
        x->val = objData.wallTime;
        x->pe = onpe;
        maxh.insert(x);
      }
    }

    minHeap *lightProcessors = new minHeap(npe);
    for (i=0; i<npe; i++)
      if (stats->procs[i].available)
        lightProcessors->insert((InfoRecord *) &(processors[i]));

    int id,maxid,minpe=0;
    double temp,total_time,min_temp;
    // for(pe=0;pe < count;pe++)
    //	CkPrintf("avail for %d = %d\n",pe,stats[pe].available);

    double *pe_comm = new double[npe];
    for (int i=0; i<npe; i++) pe_comm[i] = 0.0;

    for(id = 0;id<nmigobj;id++){
	x  = maxh.deleteMax();

	maxid = x->id;

        processorInfo *donor = (processorInfo *) lightProcessors->deleteMin();
	CmiAssert(donor);
	int first_avail_pe = donor->Id;
	temp = compute_com(stats, maxid, first_avail_pe);
	min_temp = temp;
	//total_time = temp + alloc_array[first_avail_pe][nobj];
	total_time = temp + donor->load; 
	minpe = first_avail_pe;
	
    	// search all procs for best
 	// optimization: only search processors that it communicates
	// and the minimum of all others
        CkVec<int> commPes;
        graph * ptr = object_graph[maxid].next;
    
 	// find out all processors that this obj communicates
	double commload = 0.0;			// total comm load
        for(int com=0;(com<2*nobj)&&(ptr != NULL);com++,ptr=ptr->next){
	  int destObj = ptr->id;
	  if(assigned_array[destObj] == 0)  // this obj has not been assigned
	    continue;
	  int destPe = stats->to_proc[destObj];
	  if(stats->procs[destPe].available == 0) continue;
	  
	  double cload = alpha*ptr->nmsg + beeta*ptr->data;
	  pe_comm[destPe] += cload;
	  commload += cload;

          int exist = 0;
	  for (int pp=0; pp<commPes.size(); pp++)
            if (destPe == commPes[pp]) { exist=1; break; }    // duplicated
	  if (!exist) commPes.push_back(destPe);
        }

	int k;
	for(k = 0; k < commPes.size(); k++){
	    pe = commPes[k];
            processorInfo *commpe = (processorInfo *) &processors[pe];
	    
	    temp = commload - pe_comm[pe];
	    
	    //CkPrintf("check id = %d, processor = %d,com = %lf, pro = %lf, comp=%lf\n", maxid,pe,temp,alloc_array[pe][nobj],total_time);
	    if(total_time > (temp + commpe->load)){
		minpe = pe;
		total_time = temp + commpe->load;
		min_temp = temp;
	    }
	}
	/* CkPrintf("check id = %d, processor = %d, obj = %lf com = %lf, pro = %lf, comp=%lf\n", maxid,minpe,x->load,min_temp,alloc_array[minpe][nobj],total_time); */
	
	//    CkPrintf("before 2nd alloc\n");
        stats->assign(maxid, minpe);
	
	alloc(minpe, maxid, x->val + min_temp);

	// now that maxid assigned to minpe, update other pes load
	update(stats, maxid, minpe);

        // update heap
 	lightProcessors->insert(donor);
	for(k = 0; k < commPes.size(); k++) {
	    pe = commPes[k];
            processorInfo *commpe = (processorInfo *) &processors[pe];
            lightProcessors->update(commpe);
	    pe_comm[pe] = 0.0;			// clear
        }

	delete x;
    }
    
    // free up memory
    delete [] pe_comm;

    delete [] processors;
    delete [] assigned_array;

    delete lightProcessors;

    for(int oindex= 0; oindex < nobj; oindex++){
      graph * ptr = &object_graph[oindex];
      ptr = ptr->next;
      
      while(ptr != NULL){
	graph *cur = ptr;
	ptr = ptr->next;
	delete cur;
      }
    }
    delete [] object_graph;

}
Пример #18
0
void readinput(char* filename)
{
    FILE *file;

    char line[128];
    char variable[64];
    char value[64];

    file = fopen(filename, "r");
    if(file == NULL)
    {
        printf("file read error %s\n", filename);
        CkExit();
    }
    /* parse the header lines to get the number of vertices*/
    while(fgets(line, 128, file) != NULL)
    {
        if(strncmp(line, "DIMENSION", 9) == 0)
        {
            sscanf(line, "%s : %s", variable, value);
            verticesNum = atoi(value);
        }else if(strncmp(line, "EDGE_DATA_SECTION", 17) == 0)
        {
            break;
        }
    }
   
    vector<int>  verticeNbs;
    verticeNbs.resize(verticesNum);

    /* get the edges, src dest */
    int src, dest;
    int previous=-1;
    int countptr = 0;
    int edgeNum = 0;
    while(fgets(line, sizeof line, file) != NULL && strncmp(line, "-1", 2) != 0)
    {
        edgeNum += 1;
        sscanf(line, "%d %d", &src, &dest);
        if(src != previous)
        {
        
            inputGraph.push_back(src-1);
            //CkPrintf("\nSource: %d %d:", src, inputGraph[countptr]);
            previous = src;
            countptr = inputGraph.size(); 
            inputGraph.push_back(1);
        }else
        {
            inputGraph[countptr]++;
        }
        //CkPrintf("  %d  ", dest);
        inputGraph.push_back(dest-1);
    }
#ifdef YHDEBUG
    CkPrintf("\n");
    for(int i=0; i<inputGraph.size(); i++)
    {
   
        CkPrintf(" %d ", inputGraph[i]);
    }

    CkPrintf("+++++++++++++===\n");
#endif
    fclose(file);

}
Пример #19
0
 void addEntry(const char *name, LBCreateFn fn, LBAllocFn afn, const char *help, int shown) {
   lbtables.push_back(LBDBEntry(name, fn, afn, help, shown));
 }
Пример #20
0
 void addCompiletimeBalancer(const char *name) {
   compile_lbs.push_back(name); 
 }
Пример #21
0
void HybridBaseLB::CollectInfo(Location *loc, int n, int fromlevel)
{
   int atlevel = fromlevel + 1;
   LevelData *lData = levelData[atlevel];
   lData->info_recved++;

   CkVec<Location> &matchedObjs = lData->matchedObjs;
   std::map<LDObjKey, int> &unmatchedObjs = lData->unmatchedObjs;

   // sort into matched and unmatched list
#if 0
   for (int i=0; i<n; i++) {
     // search and see if we have answer, put to matched
     // store in unknown
     int found = 0;
     for (int obj=0; obj<unmatchedObjs.size(); obj++) {
       if (loc[i].key == unmatchedObjs[obj].key) {
         // answer must exist
         CmiAssert(unmatchedObjs[obj].loc != -1 || loc[i].loc != -1);
         if (unmatchedObjs[obj].loc == -1) unmatchedObjs[obj].loc = loc[i].loc;
         matchedObjs.push_back(unmatchedObjs[obj]);
         unmatchedObjs.remove(obj);
         found = 1;
         break;
       }
     }
     if (!found) unmatchedObjs.push_back(loc[i]);
   }
#else
   for (int i=0; i<n; i++) {
     std::map<LDObjKey, int>::iterator iter = unmatchedObjs.find(loc[i].key);
     if (iter != unmatchedObjs.end()) {
       CmiAssert(iter->second != -1 || loc[i].loc != -1);
       if (loc[i].loc == -1) loc[i].loc = iter->second;
       matchedObjs.push_back(loc[i]);
       unmatchedObjs.erase(iter);
     }
     else
       unmatchedObjs[loc[i].key] = loc[i].loc;
   }
#endif

  DEBUGF(("[%d] level %d has %d unmatched and %d matched. \n", CkMyPe(), atlevel, unmatchedObjs.size(), matchedObjs.size()));

   if (lData->info_recved == lData->nChildren) {
     lData->info_recved = 0;
     if (_lb_args.debug() > 1)
         CkPrintf("[%d] CollectInfo at level %d started at %f\n",
	        CkMyPe(), atlevel, CkWallTimer());
     if (lData->parent != -1) {
       // send only unmatched ones up the tree
       CkVec<Location> unmatchedbuf;
       for(std::map<LDObjKey, int>::const_iterator it = unmatchedObjs.begin(); it != unmatchedObjs.end(); ++it)
       {
         unmatchedbuf.push_back(Location(it->first, it->second));
       }

       thisProxy[lData->parent].CollectInfo(unmatchedbuf.getVec(), unmatchedbuf.size(), atlevel);
     }
     else { // root
       // we should have all answers now
       CmiAssert(unmatchedObjs.size() == 0);
       // start send match list down
       thisProxy.PropagateInfo(matchedObjs.getVec(), matchedObjs.size(), atlevel, lData->nChildren, lData->children);
       lData->statsData->clear();
     }
   }
}
Пример #22
0
 void addRuntimeBalancer(const char *name) {
   runtime_lbs.push_back(name); 
 }
Пример #23
0
void FEM_REFINE2D_Split(int meshID,int nodeID,double *coord,int elemID,double *desiredAreas,int sparseID){
  int nnodes = FEM_Mesh_get_length(meshID,nodeID);
  int nelems = FEM_Mesh_get_length(meshID,elemID);
  int actual_nodes = nnodes, actual_elems = nelems;
  
  FEM_Refine_Operation_Data refine_data;
  refine_data.meshID = meshID;
  refine_data.nodeID = nodeID;
  refine_data.sparseID = sparseID;
  refine_data.elemID = elemID;
  refine_data.cur_nodes = FEM_Mesh_get_length(meshID,nodeID);
  
  /*Copy the cordinates of the nodes into a vector, 
    the cordinates of the new nodes will be inserted
    into this vector and will be used to sort all the
    nodes on the basis of the distance from origin
  */
  CkVec<double> coordVec;
  for(int i=0;i<nnodes*2;i++){
    coordVec.push_back(coord[i]);
  }
  refine_data.coordVec = &coordVec;
  refine_data.coord = coord;
  
  /*find out the attributes of the node */
  FEM_Entity *e=refine_data.node = FEM_Entity_lookup(meshID,nodeID,"REFINE2D_Mesh");
  CkVec<FEM_Attribute *> *attrs = refine_data.attrs = e->getAttrVec();
  /* 
  FEM_DataAttribute *boundaryAttr = (FEM_DataAttribute *)e->lookup(FEM_BOUNDARY,"split");
  if(boundaryAttr != NULL){
	AllocTable2d<int> &boundaryTable = boundaryAttr->getInt();
	printf(" Node Boundary flags \n");
	for(int i=0;i<nnodes;i++){
		printf("Node %d flag %d \n",i,boundaryTable[i][0]);
	}
  }
  */
  FEM_Entity *elem = refine_data.elem = FEM_Entity_lookup(meshID,elemID,"REFIN2D_Mesh_elem");
  CkVec<FEM_Attribute *> *elemattrs = refine_data.elemattrs = elem->getAttrVec();
  
  FEM_Attribute *connAttr = elem->lookup(FEM_CONN,"REFINE2D_Mesh");
  if(connAttr == NULL){
    CkAbort("Grrrr element without connectivity \n");
  }
  AllocTable2d<int> &connTable = ((FEM_IndexAttribute *)connAttr)->get();
  refine_data.connTable = &connTable;
  
  //hashtable to store the new node number as a function of the two old numbers
  CkHashtableT<intdual,int> newnodes(nnodes);
  refine_data.newnodes = &newnodes;
  
  /* Get the FEM_BOUNDARY data of sparse elements and load it into a hashtable
    indexed by the 2 node ids that make up the edge. The data in the hashtable
    is the index number of the sparse element */
  FEM_Entity *sparse;
  CkVec<FEM_Attribute *> *sparseattrs;
  FEM_Attribute *sparseConnAttr, *sparseBoundaryAttr;
  AllocTable2d<int> *sparseConnTable, *sparseBoundaryTable;
  CkHashtableT<intdual,int> nodes2sparse(nelems);
  refine_data.nodes2sparse = &nodes2sparse;
  
  if(sparseID != -1){
    sparse = refine_data.sparse = FEM_Entity_lookup(meshID,sparseID,"REFINE2D_Mesh_sparse");
    refine_data.sparseattrs = sparseattrs = sparse->getAttrVec();
    refine_data.sparseConnAttr = sparseConnAttr = sparse->lookup(FEM_CONN,"REFINE2D_Mesh_sparse");
    sparseConnTable = &(((FEM_IndexAttribute *)sparseConnAttr)->get());
    refine_data.sparseBoundaryAttr = sparseBoundaryAttr = sparse->lookup(FEM_BOUNDARY,"REFINE2D_Mesh_sparse");
    if(sparseBoundaryAttr == NULL){
      CkAbort("Specified sparse elements without boundary conditions");
    }
    FEM_DataAttribute *validEdgeAttribute = (FEM_DataAttribute *)sparse->lookup(FEM_VALID,"REFINE2D_Mesh_sparse");
    if(validEdgeAttribute){
      refine_data.validEdge = &(validEdgeAttribute->getInt());
    }else{
      refine_data.validEdge = NULL;
    }
    /* since the default value in the hashtable is 0, to distinguish between 
       uninserted keys and the sparse element with index 0, the index of the 
       sparse elements is incremented by 1 while inserting. */
    //		printf("[%d] Sparse elements\n",FEM_My_partition());
    for(int j=0;j<sparse->size();j++){
      if(refine_data.validEdge == NULL || (*(refine_data.validEdge))[j][0]){
	int *cdata = (*sparseConnTable)[j];
	//		printf("%d < %d,%d > \n",j,cdata[0],cdata[1]);
	nodes2sparse.put(intdual(cdata[0],cdata[1])) = j+1;
      }	
    }
  }else{
    printf("Edge boundary conditions not passed into FEM_REFINE2D_Split \n");
  }
  
  //count the actual number of nodes and elements
  if(refine_data.node->lookup(FEM_VALID,"refine2D_splilt") != NULL){
    AllocTable2d<int> &validNodeTable = ((FEM_DataAttribute *)(refine_data.node->lookup(FEM_VALID,"refine2D_splilt")))->getInt();
    actual_nodes = countValidEntities(validNodeTable.getData(),nnodes);
  }
  if(refine_data.elem->lookup(FEM_VALID,"refine2D_splilt") != NULL){
    AllocTable2d<int> &validElemTable = ((FEM_DataAttribute *)(refine_data.elem->lookup(FEM_VALID,"refine2D_splilt")))->getInt();
    actual_elems = countValidEntities(validElemTable.getData(),nelems);
  }
  
  DEBUGINT(printf("%d %d \n",nnodes,nelems));	
  REFINE2D_Split(actual_nodes,coord,actual_elems,desiredAreas,&refine_data);
  
  int nSplits= refine_data.nSplits = REFINE2D_Get_Split_Length();
  DEBUGINT(printf("called REFINE2D_Split nSplits = %d \n",nSplits));
  
  if(nSplits == 0){
    return;
  }
  
  for(int split = 0;split < nSplits;split++){
    refineData op;
    REFINE2D_Get_Split(split,&op);
    FEM_Refine_Operation(&refine_data,op);
  }
  
  DEBUGINT(printf("Cordinate list length %d according to FEM %d\n",coordVec.size()/2,FEM_Mesh_get_length(meshID,nodeID)));
  IDXL_Sort_2d(FEM_Comm_shared(meshID,nodeID),coordVec.getVec());
  int read = FEM_Mesh_is_get(meshID) ;
  assert(read);
}