Example #1
0
void Patch::localCreateSection() {
#ifdef USE_SECTION_MULTICAST
  CkVec<CkArrayIndex6D> elems;
  for (int num=0; num<numNbrs; num++)
    elems.push_back(CkArrayIndex6D(computesList[num][0], computesList[num][1], computesList[num][2], computesList[num][3], computesList[num][4], computesList[num][5]));

  CkArrayID computeArrayID = computeArray.ckGetArrayID();
  mCastSecProxy = CProxySection_Compute::ckNew(computeArrayID, elems.getVec(), elems.size()); 

  CkMulticastMgr *mCastGrp = CProxy_CkMulticastMgr(mCastGrpID).ckLocalBranch();
  mCastSecProxy.ckSectionDelegate(mCastGrp);
  mCastGrp->setReductionClient(mCastSecProxy, new CkCallback(CkIndex_Patch::reduceForces(NULL), thisProxy(thisIndex.x, thisIndex.y, thisIndex.z)));

#endif
}
Example #2
0
void HybridBaseLB::ReceiveVectorMigration(LBVectorMigrateMsg *msg)
{
#if CMK_LBDB_ON
  FindNeighbors();

  int atlevel = msg->level - 1;

  DEBUGF(("[%d] ReceiveMigration\n", CkMyPe()));

  LevelData *lData = levelData[atlevel];
  LDStats *statsData = lData->statsData;

  // pick objects for required load migration, first fit
  lData->vector_expected  = 0;
  for (int i=0; i<msg->n_moves; i++)  {
    VectorMigrateInfo &move = msg->moves[i];
    CkVec<LDObjData> objs;
    CkVec<LDCommData> comms;
    if (move.from_pe == CkMyPe()) {
      int toPe = move.to_pe;
      double load = move.load;

      GetObjsToMigrate(toPe, load, statsData, atlevel, comms, objs);
      int count = objs.size();

      if (_lb_args.debug()>1)
        CkPrintf("[%d] sending %d objects to %d at %f.\n", CkMyPe(), count, toPe, CkWallTimer());
      if (objs.size() > 0)
        thisProxy[toPe].ObjsMigrated(objs, objs.size(), comms.getVec(), comms.size(), atlevel);
      thisProxy[toPe].TotalObjMigrated(count, atlevel);
    }
    else if (move.to_pe == CkMyPe()) {
      // expecting objects
      lData->vector_expected ++;
    }
  }

  if (_lb_args.debug()>1)
    CkPrintf("[%d] expecting %d vectors. \n", CkMyPe(), lData->vector_expected);
  if (lData->vectorReceived()) {
    VectorDone(atlevel);
    if (lData->migrationDone())
      StatsDone(atlevel);
  }

  delete msg;
#endif
}
Example #3
0
// pick objects to migrate "t" amount of work
void HbmLB::ReceiveMigrationDelta(double t, int lblevel, int fromlevel)
{
#if CMK_LBDB_ON
  int i;
  int atlevel = fromlevel-1;
  LevelData *lData = levelData[atlevel];
  if (atlevel != 0) {
    thisProxy.ReceiveMigrationDelta(t, lblevel, atlevel, lData->nChildren, lData->children);
    return;
  }

  // I am leave, find objects to migrate

  CkVec<int> migs;
  CkVec<LDObjData> &objData = myStats.objData;
  for (i=0; i<myStats.n_objs; i++) {
    LDObjData &oData = objData[i];
    if (oData.wallTime < t) {
      migs.push_back(i);
      t -= oData.wallTime;
      if (t == 0.0) break;
    }
  }

  int nmigs = migs.size();
  // send a message to 
  int matchPE = CkMyPe() ^ (1<<(lblevel-1));
  
  DEBUGF(("[%d] migrating %d objs to %d at lblevel %d! \n", CkMyPe(),nmigs,matchPE,lblevel));
  thisProxy[matchPE].ReceiveMigrationCount(nmigs, lblevel);

  // migrate objects
  for (i=0; i<nmigs; i++) {
    int idx = migs[i]-i;
    LDObjData &oData = objData[idx];
    CkVec<LDCommData> comms;
    collectCommData(idx, comms);
    thisProxy[matchPE].ObjMigrated(oData, comms.getVec(), comms.size());
    theLbdb->Migrate(oData.handle, matchPE);
    // TODO modify LDStats
    DEBUGF(("myStats.removeObject: %d, %d, %d\n", migs[i], i, objData.size()));
    myStats.removeObject(idx);
  }
#endif
}
Example #4
0
void HybridBaseLB::CollectInfo(Location *loc, int n, int fromlevel)
{
   int atlevel = fromlevel + 1;
   LevelData *lData = levelData[atlevel];
   lData->info_recved++;

   CkVec<Location> &matchedObjs = lData->matchedObjs;
   std::map<LDObjKey, int> &unmatchedObjs = lData->unmatchedObjs;

   // sort into matched and unmatched list
#if 0
   for (int i=0; i<n; i++) {
     // search and see if we have answer, put to matched
     // store in unknown
     int found = 0;
     for (int obj=0; obj<unmatchedObjs.size(); obj++) {
       if (loc[i].key == unmatchedObjs[obj].key) {
         // answer must exist
         CmiAssert(unmatchedObjs[obj].loc != -1 || loc[i].loc != -1);
         if (unmatchedObjs[obj].loc == -1) unmatchedObjs[obj].loc = loc[i].loc;
         matchedObjs.push_back(unmatchedObjs[obj]);
         unmatchedObjs.remove(obj);
         found = 1;
         break;
       }
     }
     if (!found) unmatchedObjs.push_back(loc[i]);
   }
#else
   for (int i=0; i<n; i++) {
     std::map<LDObjKey, int>::iterator iter = unmatchedObjs.find(loc[i].key);
     if (iter != unmatchedObjs.end()) {
       CmiAssert(iter->second != -1 || loc[i].loc != -1);
       if (loc[i].loc == -1) loc[i].loc = iter->second;
       matchedObjs.push_back(loc[i]);
       unmatchedObjs.erase(iter);
     }
     else
       unmatchedObjs[loc[i].key] = loc[i].loc;
   }
#endif

  DEBUGF(("[%d] level %d has %d unmatched and %d matched. \n", CkMyPe(), atlevel, unmatchedObjs.size(), matchedObjs.size()));

   if (lData->info_recved == lData->nChildren) {
     lData->info_recved = 0;
     if (_lb_args.debug() > 1)
         CkPrintf("[%d] CollectInfo at level %d started at %f\n",
	        CkMyPe(), atlevel, CkWallTimer());
     if (lData->parent != -1) {
       // send only unmatched ones up the tree
       CkVec<Location> unmatchedbuf;
       for(std::map<LDObjKey, int>::const_iterator it = unmatchedObjs.begin(); it != unmatchedObjs.end(); ++it)
       {
         unmatchedbuf.push_back(Location(it->first, it->second));
       }

       thisProxy[lData->parent].CollectInfo(unmatchedbuf.getVec(), unmatchedbuf.size(), atlevel);
     }
     else { // root
       // we should have all answers now
       CmiAssert(unmatchedObjs.size() == 0);
       // start send match list down
       thisProxy.PropagateInfo(matchedObjs.getVec(), matchedObjs.size(), atlevel, lData->nChildren, lData->children);
       lData->statsData->clear();
     }
   }
}
Example #5
0
// migrate only object LDStat in group
// leaf nodes actually migrate objects
void HybridBaseLB::ReceiveMigration(LBMigrateMsg *msg)
{
#if CMK_LBDB_ON
#if CMK_MEM_CHECKPOINT
  CkResetInLdb();
#endif
  FindNeighbors();

  int atlevel = msg->level - 1;

  DEBUGF(("[%d] ReceiveMigration\n", CkMyPe()));

  LevelData *lData = levelData[atlevel];

  // only non NULL when level > 0
  LDStats *statsData = lData->statsData;

  // do LDStats migration
  const int me = CkMyPe();
  lData->migrates_expected = 0;
  for(int i=0; i < msg->n_moves; i++) {
    MigrateInfo& move = msg->moves[i];
    // incoming
    if (move.from_pe != me && move.to_pe == me) {
      // I can not be the parent node
      DEBUGF(("[%d] expecting LDStats object from %d\n",me,move.from_pe));
      // will receive a ObjData message
      lData->migrates_expected ++;
    }
    else if (move.from_pe == me) {   // outgoing
      if (statsData) {		// this is inner node
        // send objdata
        int obj;
        int found = 0;
        for (obj = 0; obj<statsData->n_objs; obj++) {
          if (move.obj.omID() == statsData->objData[obj].handle.omID() && 
            move.obj.objID() == statsData->objData[obj].handle.objID())
          {
            DEBUGF(("[%d] level: %d sending objData %d to %d. \n", CkMyPe(), atlevel, obj, move.to_pe));
	    found = 1;
            // TODO send comm data
            CkVec<LDCommData> comms;
            collectCommData(obj, comms, atlevel);
	    if (move.to_pe != -1) {
              // this object migrates to another PE of the parent domain
              thisProxy[move.to_pe].ObjMigrated(statsData->objData[obj], comms.getVec(), comms.size(), atlevel);
            }
            lData->outObjs.push_back(MigrationRecord(move.obj, lData->children[statsData->from_proc[obj]], -1));
            statsData->removeObject(obj);
            break;
          }
        }
        CmiAssert(found == 1);
      }
      else {		// this is leave node
        if (move.to_pe == -1) {
          lData->outObjs.push_back(MigrationRecord(move.obj, CkMyPe(), -1));
        }
        else {
          // migrate the object
          theLbdb->Migrate(move.obj,move.to_pe);
        }
      }
    }   // end if
  }

  if (lData->migrationDone())
    StatsDone(atlevel);
#endif
}
Example #6
0
//  LDStats data sent to parent contains real PE
//  LDStats in parent should contain relative PE
void HybridBaseLB::Loadbalancing(int atlevel)
{
  int i;

  CmiAssert(atlevel >= 1);
  CmiAssert(tree->isroot(CkMyPe(), atlevel));

  LevelData *lData = levelData[atlevel];
  LDStats *statsData = lData->statsData;
  CmiAssert(statsData);

  // at this time, all objects processor location is relative, and 
  // all incoming objects from outside group belongs to the fake root proc.

  // clear background load if needed
  if (_lb_args.ignoreBgLoad()) statsData->clearBgLoad();

  currentLevel = atlevel;
  int nclients = lData->nChildren;

  DEBUGF(("[%d] Calling Strategy ... \n", CkMyPe()));
  double start_lb_time, strat_end_time;
  start_lb_time = CkWallTimer();

  if ((statsStrategy == SHRINK || statsStrategy == SHRINK_NULL) && atlevel == tree->numLevels()-1) {
    // no obj and comm data
    LBVectorMigrateMsg* migrateMsg = VectorStrategy(statsData);
    strat_end_time = CkWallTimer();

    // send to children 
    thisProxy.ReceiveVectorMigration(migrateMsg, nclients, lData->children);
  }
  else {
    LBMigrateMsg* migrateMsg = Strategy(statsData);
    strat_end_time = CkWallTimer();

    // send to children 
    //CmiPrintf("[%d] level: %d nclients:%d children: %d %d\n", CkMyPe(), atlevel, nclients, lData->children[0], lData->children[1]);
    if (!group1_created)
      thisProxy.ReceiveMigration(migrateMsg, nclients, lData->children);
    else {
        // send in multicast tree
      thisProxy.ReceiveMigration(migrateMsg, group1);
      //CkSendMsgBranchGroup(CkIndex_HybridBaseLB::ReceiveMigration(NULL),  migrateMsg, thisgroup, group1);
    }
    // CkPrintf("[%d] ReceiveMigration takes %f \n", CkMyPe(), CkWallTimer()-strat_end_time);
  }

  if (_lb_args.debug()>0){
    CkPrintf("[%d] Loadbalancing Level %d (%d children) started at %f, elapsed time %f\n", CkMyPe(), atlevel, lData->nChildren, start_lb_time, strat_end_time-start_lb_time);
    if (atlevel == tree->numLevels()-1) {
    	CkPrintf("[%d] %s memUsage: %.2fKB\n", CkMyPe(), lbName(), (1.0*useMem())/1024);
    }
  }

  // inform new objects that are from outside group
  if (atlevel < tree->numLevels()-1) {
    for (i=0; i<statsData->n_objs; i++) {
      CmiAssert(statsData->from_proc[i] != -1);   // ???
      if (statsData->from_proc[i] == nclients)  {    // from outside
        CmiAssert(statsData->to_proc[i] < nclients);
        int tope = lData->children[statsData->to_proc[i]];
        // comm data
        CkVec<LDCommData> comms;
//        collectCommData(i, comms, atlevel);
        thisProxy[tope].ObjMigrated(statsData->objData[i], comms.getVec(), comms.size(), atlevel-1);
      }
    }
  }
}
Example #7
0
void FEM_REFINE2D_Split(int meshID,int nodeID,double *coord,int elemID,double *desiredAreas,int sparseID){
  int nnodes = FEM_Mesh_get_length(meshID,nodeID);
  int nelems = FEM_Mesh_get_length(meshID,elemID);
  int actual_nodes = nnodes, actual_elems = nelems;
  
  FEM_Refine_Operation_Data refine_data;
  refine_data.meshID = meshID;
  refine_data.nodeID = nodeID;
  refine_data.sparseID = sparseID;
  refine_data.elemID = elemID;
  refine_data.cur_nodes = FEM_Mesh_get_length(meshID,nodeID);
  
  /*Copy the cordinates of the nodes into a vector, 
    the cordinates of the new nodes will be inserted
    into this vector and will be used to sort all the
    nodes on the basis of the distance from origin
  */
  CkVec<double> coordVec;
  for(int i=0;i<nnodes*2;i++){
    coordVec.push_back(coord[i]);
  }
  refine_data.coordVec = &coordVec;
  refine_data.coord = coord;
  
  /*find out the attributes of the node */
  FEM_Entity *e=refine_data.node = FEM_Entity_lookup(meshID,nodeID,"REFINE2D_Mesh");
  CkVec<FEM_Attribute *> *attrs = refine_data.attrs = e->getAttrVec();
  /* 
  FEM_DataAttribute *boundaryAttr = (FEM_DataAttribute *)e->lookup(FEM_BOUNDARY,"split");
  if(boundaryAttr != NULL){
	AllocTable2d<int> &boundaryTable = boundaryAttr->getInt();
	printf(" Node Boundary flags \n");
	for(int i=0;i<nnodes;i++){
		printf("Node %d flag %d \n",i,boundaryTable[i][0]);
	}
  }
  */
  FEM_Entity *elem = refine_data.elem = FEM_Entity_lookup(meshID,elemID,"REFIN2D_Mesh_elem");
  CkVec<FEM_Attribute *> *elemattrs = refine_data.elemattrs = elem->getAttrVec();
  
  FEM_Attribute *connAttr = elem->lookup(FEM_CONN,"REFINE2D_Mesh");
  if(connAttr == NULL){
    CkAbort("Grrrr element without connectivity \n");
  }
  AllocTable2d<int> &connTable = ((FEM_IndexAttribute *)connAttr)->get();
  refine_data.connTable = &connTable;
  
  //hashtable to store the new node number as a function of the two old numbers
  CkHashtableT<intdual,int> newnodes(nnodes);
  refine_data.newnodes = &newnodes;
  
  /* Get the FEM_BOUNDARY data of sparse elements and load it into a hashtable
    indexed by the 2 node ids that make up the edge. The data in the hashtable
    is the index number of the sparse element */
  FEM_Entity *sparse;
  CkVec<FEM_Attribute *> *sparseattrs;
  FEM_Attribute *sparseConnAttr, *sparseBoundaryAttr;
  AllocTable2d<int> *sparseConnTable, *sparseBoundaryTable;
  CkHashtableT<intdual,int> nodes2sparse(nelems);
  refine_data.nodes2sparse = &nodes2sparse;
  
  if(sparseID != -1){
    sparse = refine_data.sparse = FEM_Entity_lookup(meshID,sparseID,"REFINE2D_Mesh_sparse");
    refine_data.sparseattrs = sparseattrs = sparse->getAttrVec();
    refine_data.sparseConnAttr = sparseConnAttr = sparse->lookup(FEM_CONN,"REFINE2D_Mesh_sparse");
    sparseConnTable = &(((FEM_IndexAttribute *)sparseConnAttr)->get());
    refine_data.sparseBoundaryAttr = sparseBoundaryAttr = sparse->lookup(FEM_BOUNDARY,"REFINE2D_Mesh_sparse");
    if(sparseBoundaryAttr == NULL){
      CkAbort("Specified sparse elements without boundary conditions");
    }
    FEM_DataAttribute *validEdgeAttribute = (FEM_DataAttribute *)sparse->lookup(FEM_VALID,"REFINE2D_Mesh_sparse");
    if(validEdgeAttribute){
      refine_data.validEdge = &(validEdgeAttribute->getInt());
    }else{
      refine_data.validEdge = NULL;
    }
    /* since the default value in the hashtable is 0, to distinguish between 
       uninserted keys and the sparse element with index 0, the index of the 
       sparse elements is incremented by 1 while inserting. */
    //		printf("[%d] Sparse elements\n",FEM_My_partition());
    for(int j=0;j<sparse->size();j++){
      if(refine_data.validEdge == NULL || (*(refine_data.validEdge))[j][0]){
	int *cdata = (*sparseConnTable)[j];
	//		printf("%d < %d,%d > \n",j,cdata[0],cdata[1]);
	nodes2sparse.put(intdual(cdata[0],cdata[1])) = j+1;
      }	
    }
  }else{
    printf("Edge boundary conditions not passed into FEM_REFINE2D_Split \n");
  }
  
  //count the actual number of nodes and elements
  if(refine_data.node->lookup(FEM_VALID,"refine2D_splilt") != NULL){
    AllocTable2d<int> &validNodeTable = ((FEM_DataAttribute *)(refine_data.node->lookup(FEM_VALID,"refine2D_splilt")))->getInt();
    actual_nodes = countValidEntities(validNodeTable.getData(),nnodes);
  }
  if(refine_data.elem->lookup(FEM_VALID,"refine2D_splilt") != NULL){
    AllocTable2d<int> &validElemTable = ((FEM_DataAttribute *)(refine_data.elem->lookup(FEM_VALID,"refine2D_splilt")))->getInt();
    actual_elems = countValidEntities(validElemTable.getData(),nelems);
  }
  
  DEBUGINT(printf("%d %d \n",nnodes,nelems));	
  REFINE2D_Split(actual_nodes,coord,actual_elems,desiredAreas,&refine_data);
  
  int nSplits= refine_data.nSplits = REFINE2D_Get_Split_Length();
  DEBUGINT(printf("called REFINE2D_Split nSplits = %d \n",nSplits));
  
  if(nSplits == 0){
    return;
  }
  
  for(int split = 0;split < nSplits;split++){
    refineData op;
    REFINE2D_Get_Split(split,&op);
    FEM_Refine_Operation(&refine_data,op);
  }
  
  DEBUGINT(printf("Cordinate list length %d according to FEM %d\n",coordVec.size()/2,FEM_Mesh_get_length(meshID,nodeID)));
  IDXL_Sort_2d(FEM_Comm_shared(meshID,nodeID),coordVec.getVec());
  int read = FEM_Mesh_is_get(meshID) ;
  assert(read);
}