void HybridBaseLB::ReceiveVectorMigration(LBVectorMigrateMsg *msg)
{
#if CMK_LBDB_ON
FindNeighbors();
int atlevel = msg->level - 1;
DEBUGF(("[%d] ReceiveMigration\n", CkMyPe()));
LevelData *lData = levelData[atlevel];
LDStats *statsData = lData->statsData;
// pick objects for required load migration, first fit
lData->vector_expected = 0;
for (int i=0; i<msg->n_moves; i++) {
VectorMigrateInfo &move = msg->moves[i];
CkVec<LDObjData> objs;
CkVec<LDCommData> comms;
if (move.from_pe == CkMyPe()) {
int toPe = move.to_pe;
double load = move.load;
GetObjsToMigrate(toPe, load, statsData, atlevel, comms, objs);
int count = objs.size();
if (_lb_args.debug()>1)
CkPrintf("[%d] sending %d objects to %d at %f.\n", CkMyPe(), count, toPe, CkWallTimer());
if (objs.size() > 0)
thisProxy[toPe].ObjsMigrated(objs, objs.size(), comms.getVec(), comms.size(), atlevel);
thisProxy[toPe].TotalObjMigrated(count, atlevel);
}
else if (move.to_pe == CkMyPe()) {
// expecting objects
lData->vector_expected ++;
}
}
if (_lb_args.debug()>1)
CkPrintf("[%d] expecting %d vectors. \n", CkMyPe(), lData->vector_expected);
if (lData->vectorReceived()) {
VectorDone(atlevel);
if (lData->migrationDone())
StatsDone(atlevel);
}
delete msg;
#endif
}
int CtgGlobalList::isUserSymbol(const char *name) {
// return 1;
if((strncmp("_", name, 1) == 0) || (strncmp("Cpv_", name, 4) == 0)
|| (strncmp("Csv_", name, 4) == 0) || (strncmp("Ctv_", name, 4) == 0)
|| (strncmp("Bnv_", name, 4) == 0) || (strncmp("Bpv_", name, 4) == 0)
|| (strncmp("ckout", name, 5) == 0) || (strncmp("stdout", name, 6) == 0)
|| (strncmp("ckerr", name, 5) == 0)
|| (strncmp("environ", name, 7) == 0)
|| (strncmp("stderr", name, 6) == 0) || (strncmp("stdin", name, 5) == 0)) {
#ifdef CMK_GFORTRAN
if (match(name, "__.*_MOD_.*")) return 1;
#endif
return 0;
}
/**
if the name is on the blacklist, it is not a user symbol
*/
for(int i=0;i<_blacklist.size();i++){
if(strlen(name) == strlen(_blacklist[i]) && strncmp(name,_blacklist[i],strlen(name)) == 0){
return 0;
}
}
return 1;
}
void BulkAdapt::get_elemsToLock(adaptAdj startElem, adaptAdj **elemsToLock, int edgeID, int *count)
{
CkVec<adaptAdj>* nbrElems;
// find the elements adjacent to startElem along the edge edgeID
//BULK_DEBUG(CkPrintf("[%d] BulkAdapt::get_elemsToLock: calling getEdgeAdaptAdj on elem %d\n",partitionID,startElem.localID));
nbrElems = getEdgeAdaptAdj(meshID, startElem.localID, startElem.elemType,
edgeID);
// extract adjacencies from CkVec into array needed by the locking code
(*count) = nbrElems->size();
(*elemsToLock) = (adaptAdj *)malloc((*count + 1) * sizeof(adaptAdj));
for (int i=0; i<*count; i++) {
(*elemsToLock)[i] = (*nbrElems)[i];
}
// add the start element
(*elemsToLock)[*count] = startElem;
(*count)++;
/*
printf("Elems to lock: ");
for (int i=0; i<*count; i++) {
printf("(%d, %d, %d) ", (*elemsToLock)[i].partID, (*elemsToLock)[i].localID,
(*elemsToLock)[i].elemType);
}
printf("\n");
*/
}
void System::ImportFluidData_request(const CkVec< pair<int,int> > &reqData, const int recvIndex)
{
assert(thisIndex != recvIndex);
const int nrecv = reqData.size();
assert(nrecv > 0);
MeshFluidGrad_msg *msg = new (nrecv, nrecv, nrecv, nrecv) MeshFluidGrad_msg;
msg->n = nrecv;
for (int i = 0; i < nrecv; i++)
{
const int local_id = reqData[i].first;
const int remote_id = reqData[i].second;
assert(local_id >= 0);
assert(local_id < local_n);
msg-> id_list[i] = remote_id;
msg->mesh_pnts[i] = mesh_pnts[local_id];
msg->Wrec_list[i] = Wrec_list[local_id];
#if 1
if (!ptcl_list[local_id].is_active())
{
Fluid Wi = Wrec_list[local_id].w;
const real dt = t_global - mesh_pnts[local_id].tbeg;
for (int k = 0; k < Fluid::NFLUID; k++)
Wi[k] += Wrec_list[local_id].t[k]*dt;
msg->Wrec_list[i].w = Wi;
}
#endif
msg->Wrec_minmax[i] = Wrec_minmax_list[local_id];
}
systemProxy[recvIndex].ImportFluidData_recv(msg);
}
void System::ImportFluidPrimitives_request(const CkVec< pair<int,int> > &reqData, const int recvIndex)
{
assert(thisIndex != recvIndex);
const int nrecv = reqData.size();
assert(nrecv > 0);
CkVec< pair<int, pair<Fluid, MeshPoint> > > data2send(nrecv);
for (int i = 0; i < nrecv; i++)
{
const int local_id = reqData[i].first;
const int remote_id = reqData[i].second;
assert(local_id >= 0);
assert(local_id < local_n);
data2send[i].first = remote_id;
Fluid Wi = Wrec_list[local_id].w;
#if 1
if (!ptcl_list[local_id].is_active())
{
Fluid Wi = Wrec_list[local_id].w;
const real dt = t_global - mesh_pnts[local_id].tbeg;
for (int k = 0; k < Fluid::NFLUID; k++)
Wi[k] += Wrec_list[local_id].t[k]*dt;
}
#endif
data2send[i].second = std::make_pair(Wi, mesh_pnts[local_id]);
}
systemProxy[recvIndex].ImportFluidPrimitives_recv(data2send);
}
/**
Analyze the current set of global variables, determine
which are user globals and which are system globals,
and store the list of user globals.
*/
CtgGlobalList::CtgGlobalList() {
datalen=0;
nRec=0;
int count;
for (count = 0; count < _namelist.size(); count ++)
{
unsigned long addr;
int size;
if (0 > lookup_obj_sym(_namelist[count], &addr, &size)) {
fprintf(stderr, "%s: no such symbol\n", _namelist[count]);
continue;
}
void *ptr = (void *)addr;
int gSize = ALIGN8(size);
//#if DEBUG_GOT_MANAGER
printf(" -> %s is a user global, of size %d, at %p\n",
_namelist[count], size, ptr);
//#endif
rec.push_back(CtgRec(ptr,datalen,size));
datalen+=gSize;
}
nRec=rec.size();
#if DEBUG_GOT_MANAGER
printf("relt has %d entries, %d of which are user globals\n\n",
relt_size,nRec);
#endif
}
void par_SolverState::assignclause(CkVec<par_Clause>& cls )
{
clauses.removeAll();
for(int i=0; i<cls.size(); i++)
{
clauses.push_back( cls[i]);
}
}
// pick objects to migrate "t" amount of work
void HbmLB::ReceiveMigrationDelta(double t, int lblevel, int fromlevel)
{
#if CMK_LBDB_ON
int i;
int atlevel = fromlevel-1;
LevelData *lData = levelData[atlevel];
if (atlevel != 0) {
thisProxy.ReceiveMigrationDelta(t, lblevel, atlevel, lData->nChildren, lData->children);
return;
}
// I am leave, find objects to migrate
CkVec<int> migs;
CkVec<LDObjData> &objData = myStats.objData;
for (i=0; i<myStats.n_objs; i++) {
LDObjData &oData = objData[i];
if (oData.wallTime < t) {
migs.push_back(i);
t -= oData.wallTime;
if (t == 0.0) break;
}
}
int nmigs = migs.size();
// send a message to
int matchPE = CkMyPe() ^ (1<<(lblevel-1));
DEBUGF(("[%d] migrating %d objs to %d at lblevel %d! \n", CkMyPe(),nmigs,matchPE,lblevel));
thisProxy[matchPE].ReceiveMigrationCount(nmigs, lblevel);
// migrate objects
for (i=0; i<nmigs; i++) {
int idx = migs[i]-i;
LDObjData &oData = objData[idx];
CkVec<LDCommData> comms;
collectCommData(idx, comms);
thisProxy[matchPE].ObjMigrated(oData, comms.getVec(), comms.size());
theLbdb->Migrate(oData.handle, matchPE);
// TODO modify LDStats
DEBUGF(("myStats.removeObject: %d, %d, %d\n", migs[i], i, objData.size()));
myStats.removeObject(idx);
}
#endif
}
//Find next un-occupied global number:
int nextUnoccupied(void) {
while (occupiedBefore<occupied.size()) {
if (occupied[occupiedBefore]==0)
return occupiedBefore;
else occupiedBefore++;
}
/* occupiedBefore==occupied.size(), so add to end of list */
occupied.push_back(1);
return occupiedBefore;
}
void TraceBluegene::userBracketEvent(const char* name, double bt, double et, void** parentLogPtr, CkVec<void*> bgLogList){
if (!genTimeLog) return;
BgTimeLog* newLog = new BgTimeLog(_threadEP,name,bt,et);
newLog->addBackwardDeps(bgLogList);
CmiAssert(bgLogList.size()>0);
newLog->objId = ((BgTimeLog*)bgLogList[0])->objId; // for sdag
*parentLogPtr = newLog;
tTIMELINEREC.logEntryInsert(newLog);
}
/** Extract all FEM communication information into Roccom format. */
static CkVec<int> getRoccomPconn(int fem_mesh,int *ghost_len,const int *paneFmChunk)
{
CkVec<int> pconn;
// Shared nodes come first:
getRoccomPconn(IDXL_Get_send(FEM_Comm_shared(fem_mesh,FEM_NODE)),0,pconn,paneFmChunk);
int realLen=pconn.size();
// Sent ghost nodes:
getRoccomPconn(IDXL_Get_send(FEM_Comm_ghost(fem_mesh,FEM_NODE)),0,pconn,paneFmChunk);
// Received ghost nodes (use bias to switch to Roccom ghost node numbering)
getRoccomPconn(IDXL_Get_recv(FEM_Comm_ghost(fem_mesh,FEM_NODE)),
FEM_Mesh_get_length(fem_mesh,FEM_NODE),pconn,paneFmChunk);
// Handle elements (much tougher!)
// Find list of element types
int elems[1024];
int e, ne=FEM_Mesh_get_entities(fem_mesh,elems);
for (e=0;e<ne;e++)
if (elems[e]<FEM_ELEM || elems[e]>=FEM_SPARSE)
elems[e--]=elems[--ne]; // swap out bad entity with the end
// Make one output IDXL that combines all element types:
IDXL_t elghost=IDXL_Create();
int out_r=0, out_g=0; // output indices for real; ghost
for (e=0;e<ne;e++) {
IDXL_Combine(elghost,FEM_Comm_ghost(fem_mesh,elems[e]), out_r,out_g);
out_r+=FEM_Mesh_get_length(fem_mesh,elems[e]);
out_g+=FEM_Mesh_get_length(fem_mesh,elems[e]+FEM_GHOST);
}
// Sent ghost elements:
getRoccomPconn(IDXL_Get_send(elghost),0,pconn,paneFmChunk);
// Received ghost elements (shift all ghosts to start after real elements)
getRoccomPconn(IDXL_Get_recv(elghost),out_r,pconn,paneFmChunk);
IDXL_Destroy(elghost);
if (ghost_len) *ghost_len=pconn.size()-realLen;
return pconn;
}
/// Mark this entity's global numbers as used:
void mark(FEM_Entity &src) {
int l,len=src.size();
for (l=0;l<len;l++) {
int g=src.getGlobalno(l);
if (g!=-1) {
while (occupied.size()<=g) { //Make room for this marker
occupied.push_back(0);
}
//FIXME: make sure element global numbers aren't repeated
// (tough because *node* global numbers may be repeated)
occupied[g]=1;
}
}
}
void Patch::localCreateSection() {
#ifdef USE_SECTION_MULTICAST
CkVec<CkArrayIndex6D> elems;
for (int num=0; num<numNbrs; num++)
elems.push_back(CkArrayIndex6D(computesList[num][0], computesList[num][1], computesList[num][2], computesList[num][3], computesList[num][4], computesList[num][5]));
CkArrayID computeArrayID = computeArray.ckGetArrayID();
mCastSecProxy = CProxySection_Compute::ckNew(computeArrayID, elems.getVec(), elems.size());
CkMulticastMgr *mCastGrp = CProxy_CkMulticastMgr(mCastGrpID).ckLocalBranch();
mCastSecProxy.ckSectionDelegate(mCastGrp);
mCastGrp->setReductionClient(mCastSecProxy, new CkCallback(CkIndex_Patch::reduceForces(NULL), thisProxy(thisIndex.x, thisIndex.y, thisIndex.z)));
#endif
}
void System::ImportNewDt_recv(const CkVec< pair<int, real> > &recvUpdates)
{
const int nrecv = recvUpdates.size();
for (int i = 0; i < nrecv; i++)
{
const int iId = recvUpdates[i].first;
assert(iId >= (int)(nactive_loc + nimport_loc));
assert(iId < (int) ptcl_act.size());
assert(ptcl_act[iId]->is_active());
mesh_act[iId]->dt_new = recvUpdates[i].second;
}
ImportNewDt_nRequested--;
assert(ImportNewDt_nRequested >= 0);
if (ImportNewDt_nRequested == 0)
ImportNewDtCb.send();
}
void System::ImportNewDt_request(const CkVec< pair<int,int> > &reqData, const int recvIndex)
{
assert(thisIndex != recvIndex);
const int nrecv = reqData.size();
assert(nrecv > 0);
CkVec< pair<int, real> > data2send;
data2send.reserve(nrecv);
for (int i = 0; i < nrecv; i++)
{
const int local_id = reqData[i].first;
const int remote_id = reqData[i].second;
assert(local_id >= 0);
assert(local_id < local_n);
assert(ptcl_list[local_id].is_active());
data2send.push_back(std::make_pair(remote_id, mesh_pnts[local_id].dt_new));
}
systemProxy[recvIndex].ImportNewDt_recv(data2send);
}
void Main::done(CkVec<int> assignment)
{
double endtimer = CkWallTimer();
CkPrintf("\nFile reading and processing time: %f\n", readfiletimer-starttimer);
CkPrintf("Solving time: %f\n", endtimer - readfiletimer);
FILE *file;
char outputfile[50];
sprintf(outputfile, "%s.sat", inputfile);
file = fopen(outputfile, "w");
for(int i=0; i<assignment.size(); i++)
{
fprintf(file, "%d\n", assignment[i]);
}
CkExit();
}
void System::ImportFluidPrimitives_recv(const CkVec< pair<int, pair<Fluid, MeshPoint> > > &recvUpdates)
{
const int nrecv = recvUpdates.size();
for (int i = 0; i < nrecv; i++)
{
const int iId = recvUpdates[i].first;
assert(iId >= (int)(nactive_loc + nimport_loc));
assert(iId < (int) ptcl_act.size());
Wrec_act [iId]->w = recvUpdates[i].second.first;
*mesh_act[iId] = recvUpdates[i].second.second;
Wrec_act[iId]->vel = mesh_act[iId]->vel;
assert(Wrec_act[iId]->w[Fluid::DENS] > 0.0);
assert(Wrec_act[iId]->w[Fluid::ETHM] > 0.0);
}
ImportFluidPrimitives_nRequested--;
assert(ImportFluidPrimitives_nRequested >= 0);
if (ImportFluidPrimitives_nRequested == 0)
ImportFluidPrimitivesCb.send();
}
void GreedyCommLB::work(LDStats* stats)
{
int pe,obj,com;
ObjectRecord *x;
int i;
if (_lb_args.debug()) CkPrintf("In GreedyCommLB strategy\n",CkMyPe());
npe = stats->nprocs();
nobj = stats->n_objs;
// nmigobj is calculated as the number of migratable objects
// ObjectHeap maxh is of size nmigobj
nmigobj = stats->n_migrateobjs;
stats->makeCommHash();
assigned_array = new int[nobj];
object_graph = new graph[nobj];
init_data(assigned_array,object_graph,npe,nobj);
#define MAXDOUBLE 1e10;
// processor heap
processors = new processorInfo[npe];
for (int p=0; p<npe; p++) {
processors[p].Id = p;
processors[p].backgroundLoad = stats->procs[p].bg_walltime;
processors[p].computeLoad = 0;
processors[p].pe_speed = stats->procs[p].pe_speed;
if (!stats->procs[p].available) {
processors[p].load = MAXDOUBLE;
}
else {
processors[p].load = 0;
if (!_lb_args.ignoreBgLoad())
processors[p].load = processors[p].backgroundLoad;
}
}
// assign communication graph
for(com =0; com< stats->n_comm;com++) {
int xcoord=0,ycoord=0;
LDCommData &commData = stats->commData[com];
if((!commData.from_proc())&&(commData.recv_type()==LD_OBJ_MSG))
{
xcoord = stats->getHash(commData.sender);
ycoord = stats->getHash(commData.receiver.get_destObj());
if((xcoord == -1)||(ycoord == -1))
if (_lb_args.ignoreBgLoad() || stats->complete_flag==0) continue;
else CkAbort("Error in search\n");
add_graph(xcoord,ycoord,commData.bytes, commData.messages);
}
else if (commData.recv_type()==LD_OBJLIST_MSG) {
int nobjs;
LDObjKey *objs = commData.receiver.get_destObjs(nobjs);
xcoord = stats->getHash(commData.sender);
for (int i=0; i<nobjs; i++) {
ycoord = stats->getHash(objs[i]);
if((xcoord == -1)||(ycoord == -1))
if (_lb_args.migObjOnly()) continue;
else CkAbort("Error in search\n");
//printf("Multicast: %d => %d %d %d\n", xcoord, ycoord, commData.bytes, commData.messages);
add_graph(xcoord,ycoord,commData.bytes, commData.messages);
}
}
}
// only build heap with migratable objects,
// mapping nonmigratable objects to the same processors
ObjectHeap maxh(nmigobj+1);
for(obj=0; obj < stats->n_objs; obj++) {
LDObjData &objData = stats->objData[obj];
int onpe = stats->from_proc[obj];
if (!objData.migratable) {
if (!stats->procs[onpe].available) {
CmiAbort("Load balancer is not be able to move a nonmigratable object out of an unavailable processor.\n");
}
alloc(onpe, obj, objData.wallTime);
update(stats, obj, onpe); // update communication cost on other pes
}
else {
x = new ObjectRecord;
x->id = obj;
x->pos = obj;
x->val = objData.wallTime;
x->pe = onpe;
maxh.insert(x);
}
}
minHeap *lightProcessors = new minHeap(npe);
for (i=0; i<npe; i++)
if (stats->procs[i].available)
lightProcessors->insert((InfoRecord *) &(processors[i]));
int id,maxid,minpe=0;
double temp,total_time,min_temp;
// for(pe=0;pe < count;pe++)
// CkPrintf("avail for %d = %d\n",pe,stats[pe].available);
double *pe_comm = new double[npe];
for (int i=0; i<npe; i++) pe_comm[i] = 0.0;
for(id = 0;id<nmigobj;id++){
x = maxh.deleteMax();
maxid = x->id;
processorInfo *donor = (processorInfo *) lightProcessors->deleteMin();
CmiAssert(donor);
int first_avail_pe = donor->Id;
temp = compute_com(stats, maxid, first_avail_pe);
min_temp = temp;
//total_time = temp + alloc_array[first_avail_pe][nobj];
total_time = temp + donor->load;
minpe = first_avail_pe;
// search all procs for best
// optimization: only search processors that it communicates
// and the minimum of all others
CkVec<int> commPes;
graph * ptr = object_graph[maxid].next;
// find out all processors that this obj communicates
double commload = 0.0; // total comm load
for(int com=0;(com<2*nobj)&&(ptr != NULL);com++,ptr=ptr->next){
int destObj = ptr->id;
if(assigned_array[destObj] == 0) // this obj has not been assigned
continue;
int destPe = stats->to_proc[destObj];
if(stats->procs[destPe].available == 0) continue;
double cload = alpha*ptr->nmsg + beeta*ptr->data;
pe_comm[destPe] += cload;
commload += cload;
int exist = 0;
for (int pp=0; pp<commPes.size(); pp++)
if (destPe == commPes[pp]) { exist=1; break; } // duplicated
if (!exist) commPes.push_back(destPe);
}
int k;
for(k = 0; k < commPes.size(); k++){
pe = commPes[k];
processorInfo *commpe = (processorInfo *) &processors[pe];
temp = commload - pe_comm[pe];
//CkPrintf("check id = %d, processor = %d,com = %lf, pro = %lf, comp=%lf\n", maxid,pe,temp,alloc_array[pe][nobj],total_time);
if(total_time > (temp + commpe->load)){
minpe = pe;
total_time = temp + commpe->load;
min_temp = temp;
}
}
/* CkPrintf("check id = %d, processor = %d, obj = %lf com = %lf, pro = %lf, comp=%lf\n", maxid,minpe,x->load,min_temp,alloc_array[minpe][nobj],total_time); */
// CkPrintf("before 2nd alloc\n");
stats->assign(maxid, minpe);
alloc(minpe, maxid, x->val + min_temp);
// now that maxid assigned to minpe, update other pes load
update(stats, maxid, minpe);
// update heap
lightProcessors->insert(donor);
for(k = 0; k < commPes.size(); k++) {
pe = commPes[k];
processorInfo *commpe = (processorInfo *) &processors[pe];
lightProcessors->update(commpe);
pe_comm[pe] = 0.0; // clear
}
delete x;
}
// free up memory
delete [] pe_comm;
delete [] processors;
delete [] assigned_array;
delete lightProcessors;
for(int oindex= 0; oindex < nobj; oindex++){
graph * ptr = &object_graph[oindex];
ptr = ptr->next;
while(ptr != NULL){
graph *cur = ptr;
ptr = ptr->next;
delete cur;
}
}
delete [] object_graph;
}
void FEM_REFINE2D_Split(int meshID,int nodeID,double *coord,int elemID,double *desiredAreas,int sparseID){
int nnodes = FEM_Mesh_get_length(meshID,nodeID);
int nelems = FEM_Mesh_get_length(meshID,elemID);
int actual_nodes = nnodes, actual_elems = nelems;
FEM_Refine_Operation_Data refine_data;
refine_data.meshID = meshID;
refine_data.nodeID = nodeID;
refine_data.sparseID = sparseID;
refine_data.elemID = elemID;
refine_data.cur_nodes = FEM_Mesh_get_length(meshID,nodeID);
/*Copy the cordinates of the nodes into a vector,
the cordinates of the new nodes will be inserted
into this vector and will be used to sort all the
nodes on the basis of the distance from origin
*/
CkVec<double> coordVec;
for(int i=0;i<nnodes*2;i++){
coordVec.push_back(coord[i]);
}
refine_data.coordVec = &coordVec;
refine_data.coord = coord;
/*find out the attributes of the node */
FEM_Entity *e=refine_data.node = FEM_Entity_lookup(meshID,nodeID,"REFINE2D_Mesh");
CkVec<FEM_Attribute *> *attrs = refine_data.attrs = e->getAttrVec();
/*
FEM_DataAttribute *boundaryAttr = (FEM_DataAttribute *)e->lookup(FEM_BOUNDARY,"split");
if(boundaryAttr != NULL){
AllocTable2d<int> &boundaryTable = boundaryAttr->getInt();
printf(" Node Boundary flags \n");
for(int i=0;i<nnodes;i++){
printf("Node %d flag %d \n",i,boundaryTable[i][0]);
}
}
*/
FEM_Entity *elem = refine_data.elem = FEM_Entity_lookup(meshID,elemID,"REFIN2D_Mesh_elem");
CkVec<FEM_Attribute *> *elemattrs = refine_data.elemattrs = elem->getAttrVec();
FEM_Attribute *connAttr = elem->lookup(FEM_CONN,"REFINE2D_Mesh");
if(connAttr == NULL){
CkAbort("Grrrr element without connectivity \n");
}
AllocTable2d<int> &connTable = ((FEM_IndexAttribute *)connAttr)->get();
refine_data.connTable = &connTable;
//hashtable to store the new node number as a function of the two old numbers
CkHashtableT<intdual,int> newnodes(nnodes);
refine_data.newnodes = &newnodes;
/* Get the FEM_BOUNDARY data of sparse elements and load it into a hashtable
indexed by the 2 node ids that make up the edge. The data in the hashtable
is the index number of the sparse element */
FEM_Entity *sparse;
CkVec<FEM_Attribute *> *sparseattrs;
FEM_Attribute *sparseConnAttr, *sparseBoundaryAttr;
AllocTable2d<int> *sparseConnTable, *sparseBoundaryTable;
CkHashtableT<intdual,int> nodes2sparse(nelems);
refine_data.nodes2sparse = &nodes2sparse;
if(sparseID != -1){
sparse = refine_data.sparse = FEM_Entity_lookup(meshID,sparseID,"REFINE2D_Mesh_sparse");
refine_data.sparseattrs = sparseattrs = sparse->getAttrVec();
refine_data.sparseConnAttr = sparseConnAttr = sparse->lookup(FEM_CONN,"REFINE2D_Mesh_sparse");
sparseConnTable = &(((FEM_IndexAttribute *)sparseConnAttr)->get());
refine_data.sparseBoundaryAttr = sparseBoundaryAttr = sparse->lookup(FEM_BOUNDARY,"REFINE2D_Mesh_sparse");
if(sparseBoundaryAttr == NULL){
CkAbort("Specified sparse elements without boundary conditions");
}
FEM_DataAttribute *validEdgeAttribute = (FEM_DataAttribute *)sparse->lookup(FEM_VALID,"REFINE2D_Mesh_sparse");
if(validEdgeAttribute){
refine_data.validEdge = &(validEdgeAttribute->getInt());
}else{
refine_data.validEdge = NULL;
}
/* since the default value in the hashtable is 0, to distinguish between
uninserted keys and the sparse element with index 0, the index of the
sparse elements is incremented by 1 while inserting. */
// printf("[%d] Sparse elements\n",FEM_My_partition());
for(int j=0;j<sparse->size();j++){
if(refine_data.validEdge == NULL || (*(refine_data.validEdge))[j][0]){
int *cdata = (*sparseConnTable)[j];
// printf("%d < %d,%d > \n",j,cdata[0],cdata[1]);
nodes2sparse.put(intdual(cdata[0],cdata[1])) = j+1;
}
}
}else{
printf("Edge boundary conditions not passed into FEM_REFINE2D_Split \n");
}
//count the actual number of nodes and elements
if(refine_data.node->lookup(FEM_VALID,"refine2D_splilt") != NULL){
AllocTable2d<int> &validNodeTable = ((FEM_DataAttribute *)(refine_data.node->lookup(FEM_VALID,"refine2D_splilt")))->getInt();
actual_nodes = countValidEntities(validNodeTable.getData(),nnodes);
}
if(refine_data.elem->lookup(FEM_VALID,"refine2D_splilt") != NULL){
AllocTable2d<int> &validElemTable = ((FEM_DataAttribute *)(refine_data.elem->lookup(FEM_VALID,"refine2D_splilt")))->getInt();
actual_elems = countValidEntities(validElemTable.getData(),nelems);
}
DEBUGINT(printf("%d %d \n",nnodes,nelems));
REFINE2D_Split(actual_nodes,coord,actual_elems,desiredAreas,&refine_data);
int nSplits= refine_data.nSplits = REFINE2D_Get_Split_Length();
DEBUGINT(printf("called REFINE2D_Split nSplits = %d \n",nSplits));
if(nSplits == 0){
return;
}
for(int split = 0;split < nSplits;split++){
refineData op;
REFINE2D_Get_Split(split,&op);
FEM_Refine_Operation(&refine_data,op);
}
DEBUGINT(printf("Cordinate list length %d according to FEM %d\n",coordVec.size()/2,FEM_Mesh_get_length(meshID,nodeID)));
IDXL_Sort_2d(FEM_Comm_shared(meshID,nodeID),coordVec.getVec());
int read = FEM_Mesh_is_get(meshID) ;
assert(read);
}
void FEM_Refine_Operation(FEM_Refine_Operation_Data *data,refineData &op){
int meshID = data->meshID;
int nodeID = data->nodeID;
int sparseID = data->sparseID;
CkVec<FEM_Attribute *> *attrs = data->attrs;
CkVec<FEM_Attribute *> *elemattrs = data->elemattrs;
CkHashtableT<intdual,int> *newnodes=data->newnodes;
CkHashtableT<intdual,int> *nodes2sparse = data->nodes2sparse;
FEM_Attribute *sparseConnAttr = data->sparseConnAttr, *sparseBoundaryAttr = data->sparseBoundaryAttr;
AllocTable2d<int> *connTable = data->connTable;
double *coord = data->coord;
AllocTable2d<int> *sparseConnTable, *sparseBoundaryTable;
CkVec<FEM_Attribute *> *sparseattrs = data->sparseattrs;
/*
FEM_DataAttribute *boundaryAttr = (FEM_DataAttribute *)data->node->lookup(FEM_BOUNDARY,"split");
if(boundaryAttr != NULL){
AllocTable2d<int> &boundaryTable = boundaryAttr->getInt();
printf(" Node Boundary flags \n");
for(int i=0;i<data->node->size();i++){
printf("Node %d flag %d \n",i,boundaryTable[i][0]);
}
}
*/
int tri=op.tri,A=op.A,B=op.B,C=op.C,D=op.D;
double frac=op.frac;
// current number of nodes in the mesh
int *connData = connTable->getData();
int flags=op.flag;
if((flags & 0x1) || (flags & 0x2)){
//new node
DEBUGINT(CkPrintf("---- Adding node %d\n",D));
/* lastA=A; lastB=B; lastD=D;*/
if (A>=data->cur_nodes) CkAbort("Calculated A is invalid!");
if (B>=data->cur_nodes) CkAbort("Calculated B is invalid!");
if(D >= data->cur_nodes){
data->node->setLength(D+1);
data->cur_nodes = D+1;
}
for(int i=0;i<attrs->size();i++){
FEM_Attribute *a = (FEM_Attribute *)(*attrs)[i];
if(a->getAttr()<FEM_ATTRIB_TAG_MAX){
FEM_DataAttribute *d = (FEM_DataAttribute *)a;
d->interpolate(A,B,D,frac);
}else{
/* Boundary value of new node D should be boundary value of the edge
(sparse element) that contains the two nodes A & B. */
if(a->getAttr() == FEM_BOUNDARY){
if(sparseID != -1){
int sidx = nodes2sparse->get(intdual(A,B))-1;
if(sidx == -1){
CkAbort("no sparse element between these 2 nodes, are they really connected ??");
}
sparseBoundaryTable = &(((FEM_DataAttribute *)sparseBoundaryAttr)->getInt());
int boundaryVal = ((*sparseBoundaryTable)[sidx])[0];
(((FEM_DataAttribute *)a)->getInt()[D])[0] = boundaryVal;
}else{
/* if sparse elements don't exist, just do simple interpolation */
FEM_DataAttribute *d = (FEM_DataAttribute *)a;
d->interpolate(A,B,D,frac);
}
}
}
}
int AandB[2];
AandB[0]=A;
AandB[1]=B;
/* Add a new node D between A and B */
IDXL_Add_entity(FEM_Comm_shared(meshID,nodeID),D,2,AandB);
double Dx = coord[2*A]*(1-frac)+frac*coord[2*B];
double Dy = coord[2*A+1]*(1-frac)+frac*coord[2*B+1];
data->coordVec->push_back(Dx);
data->coordVec->push_back(Dy);
newnodes->put(intdual(A,B))=D;
/* add the new sparse element <D,B> and modify the connectivity of the old
one from <A,B> to <A,D> and change the hashtable to reflect that change
*/
if(sparseID != -1){
int oldsidx = nodes2sparse->get(intdual(A,B))-1;
int newsidx = data->sparse->size();
data->sparse->setLength(newsidx+1);
for(int satt = 0;satt<sparseattrs->size();satt++){
if((*sparseattrs)[satt]->getAttr() == FEM_CONN){
/* change the conn of the old sparse to A,D and new one to B,D */
sparseConnTable = &(((FEM_IndexAttribute *)sparseConnAttr)->get());
int *oldconn = (*sparseConnTable)[oldsidx];
int *newconn = (*sparseConnTable)[newsidx];
oldconn[0] = A;
oldconn[1] = D;
newconn[0] = D;
newconn[1] = B;
//printf("<%d,%d> edge being split into <%d,%d> <%d,%d> \n",A,B,A,D,D,B);
}else{
/* apart from conn copy everything else */
FEM_Attribute *attr = (FEM_Attribute *)(*sparseattrs)[satt];
attr->copyEntity(newsidx,*attr,oldsidx);
}
}
/* modify the hashtable - delete the old edge and the new ones */
nodes2sparse->remove(intdual(A,B));
nodes2sparse->put(intdual(A,D)) = oldsidx+1;
nodes2sparse->put(intdual(D,B)) = newsidx+1;
}
}
//add a new triangle
/*TODO: replace FEM_ELEM with parameter*/
int newTri = op._new;
int cur_elements = FEM_Mesh_get_length(data->meshID,data->elemID);
DEBUGINT(CkPrintf("---- Adding triangle %d after splitting %d \n",newTri,tri));
if(newTri >= cur_elements){
data->elem->setLength(newTri+1);
}
D = newnodes->get(intdual(A,B));
for(int j=0;j<elemattrs->size();j++){
if((*elemattrs)[j]->getAttr() == FEM_CONN){
DEBUGINT(CkPrintf("elem attr conn code %d \n",(*elemattrs)[j]->getAttr()));
//it is a connectivity attribute.. get the connectivity right
FEM_IndexAttribute *connAttr = (FEM_IndexAttribute *)(*elemattrs)[j];
AllocTable2d<int> &table = connAttr->get();
DEBUGINT(CkPrintf("Table of connectivity attribute starts at %p width %d \n",table[0],connAttr->getWidth()));
int *oldRow = table[tri];
int *newRow = table[newTri];
for (int i=0;i<3;i++){
if (oldRow[i]==A){
oldRow[i]=D;
DEBUGINT(CkPrintf("In triangle %d %d replaced by %d \n",tri,A,D));
}
}
for (int i=0; i<3; i++) {
if (oldRow[i] == B){
newRow[i] = D;
}
else if (oldRow[i] == C){
newRow[i] = C;
}
else if (oldRow[i] == D){
newRow[i] = A;
}
}
DEBUGINT(CkPrintf("New Triangle %d (%d %d %d) conn %p\n",newTri,newRow[0],newRow[1],newRow[2],newRow));
}else{
FEM_Attribute *elattr = (FEM_Attribute *)(*elemattrs)[j];
if(elattr->getAttr() < FEM_ATTRIB_FIRST){
elattr->copyEntity(newTri,*elattr,tri);
}
}
}
if(sparseID != -1){ /* add sparse element (edge between C and D) */
int cdidx = data->sparse->size();
data->sparse->setLength(cdidx+1);
for(int satt = 0; satt < sparseattrs->size();satt++){
if((*sparseattrs)[satt]->getAttr() == FEM_CONN){
sparseConnTable = &(((FEM_IndexAttribute *)sparseConnAttr)->get());
int *cdconn = (*sparseConnTable)[cdidx];
cdconn[0]=C;
cdconn[1]=D;
}
if((*sparseattrs)[satt]->getAttr() == FEM_BOUNDARY){
/* An edge connecting C and D has to be an internal edge */
sparseBoundaryTable = &(((FEM_DataAttribute *)sparseBoundaryAttr)->getInt());
((*sparseBoundaryTable)[cdidx])[0] = 0;
}
}
if(data->validEdge){
(*(data->validEdge))[cdidx][0] = 1;
}
nodes2sparse->put(intdual(C,D)) = cdidx+1;
}
}
/* add clause, before adding, check unit conflict */
bool par_SolverState::addClause(CkVec<par_Lit>& ps)
{
/*TODO precheck is needed here */
if(ps.size() == 1)//unit clause
{
for(int _i=0; _i<unit_clause_index.size(); _i++)
{
int index = unit_clause_index[_i];
par_Clause unit = clauses[index];
par_Lit unit_lit = unit[0];
if(unit_lit == ~ps[0])
{
CkPrintf("clause conflict between %d and %d\n", index, clauses.size());
return false;
}
}
/* all unit clauses are checked, no repeat, no conflict */
unit_clause_index.push_back(clauses.size());
}else{
//check whether the clause is already satisfiable in any case
/* if x and ~x exist in the same clause, this clause is already satisfiable, we do not have to deal with it */
for(int i=0; i< ps.size(); i++)
{
par_Lit lit = ps[i];
for(int j=0; j<i; j++)
{
if(lit == ~ps[j])
{
CkPrintf("This clause is already satisfiable\n");
for(int _i=0; _i<ps.size(); _i++)
{
occurrence[abs(toInt(ps[_i])-1)]--;
if(toInt(ps[_i]) > 0)
positive_occurrence[abs(toInt(ps[_i])-1)]--;
}
return true;
}
}
}
}
clauses.push_back(par_Clause());
clauses[clauses.size()-1].attachdata(ps, false);
// build the linklist for each literal pointing to the clause, where the literal occurs
for(int i=0; i< ps.size(); i++)
{
par_Lit lit = ps[i];
if(toInt(lit) > 0)
whichClauses[2*toInt(lit)-2].insert(pair<int, int>(clauses.size()-1, 1));
else
whichClauses[-2*toInt(lit)-1].insert(pair<int, int> (clauses.size()-1, 1));
}
return true;
}
// LDStats data sent to parent contains real PE
// LDStats in parent should contain relative PE
void HybridBaseLB::Loadbalancing(int atlevel)
{
int i;
CmiAssert(atlevel >= 1);
CmiAssert(tree->isroot(CkMyPe(), atlevel));
LevelData *lData = levelData[atlevel];
LDStats *statsData = lData->statsData;
CmiAssert(statsData);
// at this time, all objects processor location is relative, and
// all incoming objects from outside group belongs to the fake root proc.
// clear background load if needed
if (_lb_args.ignoreBgLoad()) statsData->clearBgLoad();
currentLevel = atlevel;
int nclients = lData->nChildren;
DEBUGF(("[%d] Calling Strategy ... \n", CkMyPe()));
double start_lb_time, strat_end_time;
start_lb_time = CkWallTimer();
if ((statsStrategy == SHRINK || statsStrategy == SHRINK_NULL) && atlevel == tree->numLevels()-1) {
// no obj and comm data
LBVectorMigrateMsg* migrateMsg = VectorStrategy(statsData);
strat_end_time = CkWallTimer();
// send to children
thisProxy.ReceiveVectorMigration(migrateMsg, nclients, lData->children);
}
else {
LBMigrateMsg* migrateMsg = Strategy(statsData);
strat_end_time = CkWallTimer();
// send to children
//CmiPrintf("[%d] level: %d nclients:%d children: %d %d\n", CkMyPe(), atlevel, nclients, lData->children[0], lData->children[1]);
if (!group1_created)
thisProxy.ReceiveMigration(migrateMsg, nclients, lData->children);
else {
// send in multicast tree
thisProxy.ReceiveMigration(migrateMsg, group1);
//CkSendMsgBranchGroup(CkIndex_HybridBaseLB::ReceiveMigration(NULL), migrateMsg, thisgroup, group1);
}
// CkPrintf("[%d] ReceiveMigration takes %f \n", CkMyPe(), CkWallTimer()-strat_end_time);
}
if (_lb_args.debug()>0){
CkPrintf("[%d] Loadbalancing Level %d (%d children) started at %f, elapsed time %f\n", CkMyPe(), atlevel, lData->nChildren, start_lb_time, strat_end_time-start_lb_time);
if (atlevel == tree->numLevels()-1) {
CkPrintf("[%d] %s memUsage: %.2fKB\n", CkMyPe(), lbName(), (1.0*useMem())/1024);
}
}
// inform new objects that are from outside group
if (atlevel < tree->numLevels()-1) {
for (i=0; i<statsData->n_objs; i++) {
CmiAssert(statsData->from_proc[i] != -1); // ???
if (statsData->from_proc[i] == nclients) { // from outside
CmiAssert(statsData->to_proc[i] < nclients);
int tope = lData->children[statsData->to_proc[i]];
// comm data
CkVec<LDCommData> comms;
// collectCommData(i, comms, atlevel);
thisProxy[tope].ObjMigrated(statsData->objData[i], comms.getVec(), comms.size(), atlevel-1);
}
}
}
}
void LBInfo::getInfo(BaseLB::LDStats* stats, int count, int considerComm)
{
#if CMK_LBDB_ON
int i, pe;
CmiAssert(peLoads);
clear();
double alpha = _lb_args.alpha();
double beeta = _lb_args.beeta();
minObjLoad = 1.0e20; // I suppose no object load is beyond this
maxObjLoad = 0.0;
msgCount = 0;
msgBytes = 0;
if (considerComm) stats->makeCommHash();
// get background load
if (bgLoads)
for(pe = 0; pe < count; pe++)
bgLoads[pe] = stats->procs[pe].bg_walltime;
for(pe = 0; pe < count; pe++)
peLoads[pe] = stats->procs[pe].bg_walltime;
for(int obj = 0; obj < stats->n_objs; obj++)
{
int pe = stats->to_proc[obj];
if (pe == -1) continue; // this object is out
CmiAssert(pe >=0 && pe < count);
double oload = stats->objData[obj].wallTime;
if (oload < minObjLoad) minObjLoad = oload;
if (oload > maxObjLoad) maxObjLoad = oload;
peLoads[pe] += oload;
if (objLoads) objLoads[pe] += oload;
}
// handling of the communication overheads.
if (considerComm) {
int* msgSentCount = new int[count]; // # of messages sent by each PE
int* msgRecvCount = new int[count]; // # of messages received by each PE
int* byteSentCount = new int[count];// # of bytes sent by each PE
int* byteRecvCount = new int[count];// # of bytes reeived by each PE
for(i = 0; i < count; i++)
msgSentCount[i] = msgRecvCount[i] = byteSentCount[i] = byteRecvCount[i] = 0;
int mcast_count = 0;
for (int cidx=0; cidx < stats->n_comm; cidx++) {
LDCommData& cdata = stats->commData[cidx];
int senderPE, receiverPE;
if (cdata.from_proc())
senderPE = cdata.src_proc;
else {
int idx = stats->getHash(cdata.sender);
if (idx == -1) continue; // sender has just migrated?
senderPE = stats->to_proc[idx];
CmiAssert(senderPE != -1);
}
CmiAssert(senderPE < count && senderPE >= 0);
// find receiver: point-to-point and multicast two cases
int receiver_type = cdata.receiver.get_type();
if (receiver_type == LD_PROC_MSG || receiver_type == LD_OBJ_MSG) {
if (receiver_type == LD_PROC_MSG)
receiverPE = cdata.receiver.proc();
else { // LD_OBJ_MSG
int idx = stats->getHash(cdata.receiver.get_destObj());
if (idx == -1) continue; // receiver has just been removed?
receiverPE = stats->to_proc[idx];
CmiAssert(receiverPE != -1);
}
CmiAssert(receiverPE < count && receiverPE >= 0);
if(senderPE != receiverPE)
{
msgSentCount[senderPE] += cdata.messages;
byteSentCount[senderPE] += cdata.bytes;
msgRecvCount[receiverPE] += cdata.messages;
byteRecvCount[receiverPE] += cdata.bytes;
}
}
else if (receiver_type == LD_OBJLIST_MSG) {
int nobjs;
LDObjKey *objs = cdata.receiver.get_destObjs(nobjs);
mcast_count ++;
CkVec<int> pes;
for (i=0; i<nobjs; i++) {
int idx = stats->getHash(objs[i]);
CmiAssert(idx != -1);
if (idx == -1) continue; // receiver has just been removed?
receiverPE = stats->to_proc[idx];
CmiAssert(receiverPE < count && receiverPE >= 0);
int exist = 0;
for (int p=0; p<pes.size(); p++)
if (receiverPE == pes[p]) { exist=1; break; }
if (exist) continue;
pes.push_back(receiverPE);
if(senderPE != receiverPE)
{
msgSentCount[senderPE] += cdata.messages;
byteSentCount[senderPE] += cdata.bytes;
msgRecvCount[receiverPE] += cdata.messages;
byteRecvCount[receiverPE] += cdata.bytes;
}
}
}
} // end of for
if (_lb_args.debug())
CkPrintf("Number of MULTICAST: %d\n", mcast_count);
// now for each processor, add to its load the send and receive overheads
for(i = 0; i < count; i++)
{
double comload = msgRecvCount[i] * PER_MESSAGE_RECV_OVERHEAD +
msgSentCount[i] * alpha +
byteRecvCount[i] * PER_BYTE_RECV_OVERHEAD +
byteSentCount[i] * beeta;
peLoads[i] += comload;
if (comLoads) comLoads[i] += comload;
msgCount += msgRecvCount[i] + msgSentCount[i];
msgBytes += byteRecvCount[i] + byteSentCount[i];
}
delete [] msgRecvCount;
delete [] msgSentCount;
delete [] byteRecvCount;
delete [] byteSentCount;
}
#endif
}
// migrate only object LDStat in group
// leaf nodes actually migrate objects
void HybridBaseLB::ReceiveMigration(LBMigrateMsg *msg)
{
#if CMK_LBDB_ON
#if CMK_MEM_CHECKPOINT
CkResetInLdb();
#endif
FindNeighbors();
int atlevel = msg->level - 1;
DEBUGF(("[%d] ReceiveMigration\n", CkMyPe()));
LevelData *lData = levelData[atlevel];
// only non NULL when level > 0
LDStats *statsData = lData->statsData;
// do LDStats migration
const int me = CkMyPe();
lData->migrates_expected = 0;
for(int i=0; i < msg->n_moves; i++) {
MigrateInfo& move = msg->moves[i];
// incoming
if (move.from_pe != me && move.to_pe == me) {
// I can not be the parent node
DEBUGF(("[%d] expecting LDStats object from %d\n",me,move.from_pe));
// will receive a ObjData message
lData->migrates_expected ++;
}
else if (move.from_pe == me) { // outgoing
if (statsData) { // this is inner node
// send objdata
int obj;
int found = 0;
for (obj = 0; obj<statsData->n_objs; obj++) {
if (move.obj.omID() == statsData->objData[obj].handle.omID() &&
move.obj.objID() == statsData->objData[obj].handle.objID())
{
DEBUGF(("[%d] level: %d sending objData %d to %d. \n", CkMyPe(), atlevel, obj, move.to_pe));
found = 1;
// TODO send comm data
CkVec<LDCommData> comms;
collectCommData(obj, comms, atlevel);
if (move.to_pe != -1) {
// this object migrates to another PE of the parent domain
thisProxy[move.to_pe].ObjMigrated(statsData->objData[obj], comms.getVec(), comms.size(), atlevel);
}
lData->outObjs.push_back(MigrationRecord(move.obj, lData->children[statsData->from_proc[obj]], -1));
statsData->removeObject(obj);
break;
}
}
CmiAssert(found == 1);
}
else { // this is leave node
if (move.to_pe == -1) {
lData->outObjs.push_back(MigrationRecord(move.obj, CkMyPe(), -1));
}
else {
// migrate the object
theLbdb->Migrate(move.obj,move.to_pe);
}
}
} // end if
}
if (lData->migrationDone())
StatsDone(atlevel);
#endif
}
/** Perform a Delaunay flip of the edge (n1, n2) returning 1 if successful, 0 if
not (likely due to the edge being on a boundary). The convexity of the
quadrilateral formed by two faces incident to edge (n1, n2) is assumed. n1
and n2 are assumed to be local to this chunk. An adjacency test is
performed on n1 and n2 by searching for an element with edge [n1,n2].
n3 n3
o o
/ \ /|\
/ \ / | \
/ \ / | \
/ \ / | \
n1 o---------o n2 n1 o | o n2
\ / \ | /
\ / \ | /
\ / \ | /
\ / \|/
o o
n4 n4
*/
int FEM_Adapt::edge_flip_help(int e1, int e2, int n1, int n2, int e1_n1,
int e1_n2, int e1_n3, int n3, int n4, int *locknodes)
{
int numNodes = 4;
int numElems = 2;
int lockelems[2];
int elemConn[3];
locknodes[0] = n1;
locknodes[1] = n2;
locknodes[2] = n3;
locknodes[3] = n4;
lockelems[0] = e1;
lockelems[1] = e2;
if(n1 < 0 || n2 < 0) {
return -1;
}
int index = theMod->getIdx();
bool flag = theMod->fmAdaptAlgs->controlQualityF(n1,n2,n3,n4);
if(flag) return -1;
int e1Topurge = e1;
int e2Topurge = e2;
#ifdef DEBUG_1
CkPrintf("Flipping edge %d->%d on chunk %d\n", n1, n2, theMod->getfmUtil()->getIdx());
#endif
//FEM_Modify_Lock(theMesh, locknodes, numNodes, lockelems, numElems);
//if any of the two elements is remote, eat those
if(n3 < 0) {
e1Topurge = theMod->fmUtil->eatIntoElement(e1);
theMesh->e2n_getAll(e1Topurge,elemConn);
for(int i=0; i<3; i++) {
if(elemConn[i]!=n1 && elemConn[i]!=n2) {
n3 = elemConn[i];
}
}
locknodes[2] = n3;
}
if(n4 < 0) {
e2Topurge = theMod->fmUtil->eatIntoElement(e2);
theMesh->e2n_getAll(e2Topurge,elemConn);
for(int i=0; i<3; i++) {
if(elemConn[i]!=n1 && elemConn[i]!=n2) {
n4 = elemConn[i];
}
}
locknodes[3] = n4;
}
FEM_remove_element(theMesh,e1Topurge,0,0);
FEM_remove_element(theMesh,e2Topurge,0,0);
// add n1, n3, n4
elemConn[e1_n1] = n1; elemConn[e1_n2] = n4; elemConn[e1_n3] = n3;
lockelems[0] = FEM_add_element(theMesh, elemConn, 3, 0, index);
//the attributes should really be interpolated, i.e. on both new elems,
//the values should be an average of the previous two elements
theMod->fmUtil->copyElemData(0,e1Topurge,lockelems[0]);
// add n2, n3, n4
elemConn[e1_n1] = n4; elemConn[e1_n2] = n2; elemConn[e1_n3] = n3;
lockelems[1] = FEM_add_element(theMesh, elemConn, 3, 0, index);
theMod->fmUtil->copyElemData(0,e1Topurge,lockelems[1]); //both of the new elements copy from one element
//purge the two elements
FEM_purge_element(theMesh,e1Topurge,0);
FEM_purge_element(theMesh,e2Topurge,0);
//get rid of some unnecessary ghost node sends
for(int i=0; i<4;i++) {
int nodeToUpdate = -1;
if(i==0) nodeToUpdate = n1;
else if(i==1) nodeToUpdate = n2;
else if(i==2) nodeToUpdate = n3;
else if(i==3) nodeToUpdate = n4;
//if any of the chunks sharing this node sends this as a ghost, then all of them have to
//so find out the set of chunks I need to send this as a ghost to
//collect info from each of the shared chunks, do a union of all these chunks
//send this updated list to everyone.
//if anyone needs to add or delete some ghosts, they will
int *chkl, numchkl=0;
CkVec<int> finalchkl;
theMod->fmUtil->findGhostSend(nodeToUpdate, chkl, numchkl);
for(int j=0; j<numchkl; j++) {
finalchkl.push_back(chkl[j]);
}
if(numchkl>0) delete[] chkl;
const IDXL_Rec *irec=theMesh->node.shared.getRec(nodeToUpdate);
int numchunks=0;
int *chunks1, *inds1;
if(irec) {
numchunks = irec->getShared();
chunks1 = new int[numchunks];
inds1 = new int[numchunks];
for(int j=0; j<numchunks; j++) {
chunks1[j] = irec->getChk(j);
inds1[j] = irec->getIdx(j);
}
}
for(int j=0; j<numchunks; j++) {
findgsMsg *fmsg = meshMod[chunks1[j]].findghostsend(index,inds1[j]);
if(fmsg->numchks>0) {
for(int k=0; k<fmsg->numchks; k++) {
bool shouldbeadded = true;
for(int l=0; l<finalchkl.size(); l++) {
if(fmsg->chunks[k]==finalchkl[l]) {
shouldbeadded = false;
break;
}
}
if(shouldbeadded) finalchkl.push_back(fmsg->chunks[k]);
}
}
delete fmsg;
}
int *finall, numfinall=finalchkl.size();
if(numfinall>0) finall = new int[numfinall];
for(int j=0; j<numfinall; j++) finall[j] = finalchkl[j];
finalchkl.free();
theMod->fmUtil->UpdateGhostSend(nodeToUpdate, finall, numfinall);
for(int j=0; j<numchunks; j++) {
verifyghostsendMsg *vmsg = new(numfinall)verifyghostsendMsg();
vmsg->fromChk = index;
vmsg->sharedIdx = inds1[j];
vmsg->numchks = numfinall;
for(int k=0; k<numfinall; k++) vmsg->chunks[k] = finall[k];
meshMod[chunks1[j]].updateghostsend(vmsg);
}
if(numfinall>0) delete[] finall;
if(numchunks>0) {
delete[] chunks1;
delete[] inds1;
}
}
//make sure that it always comes here, don't return with unlocking
return 1; //return newNode;
}
/** Given edge e:(n1, n2), remove the two elements (n1,n2,n3) and
(n2,n1,n4) adjacent to e, and bisect e by adding node
n5. Add elements (n1,n5,n3), (n5,n2,n3), (n5,n1,n4) and (n2,n5,n4);
returns new node n5.
n3 n3
o o
/ \ /|\
/ \ / | \
/ \ / | \
/ \ / |n5 \
n1 o---------o n2 n1 o----o----o n2
\ / \ | /
\ / \ | /
\ / \ | /
\ / \|/
o o
n4 n4
*/
int FEM_Adapt::edge_bisect_help(int e1, int e2, int n1, int n2, int e1_n1,
int e1_n2, int e1_n3, int e2_n1, int e2_n2,
int e2_n3, int n3, int n4)
{
int n5;
int numNodes = 4;
int numElems = 2;
int numNodesNew = 5;
int numElemsNew = 4;
int locknodes[5];
int lockelems[4];
int elemConn[3];
locknodes[0] = n1;
locknodes[1] = n2;
locknodes[2] = n3;
locknodes[3] = n4;
locknodes[4] = -1;
lockelems[0] = e1;
lockelems[1] = e2;
lockelems[2] = -1;
lockelems[3] = -1;
//FEM_Modify_Lock(theMesh, locknodes, numNodes, lockelems, numElems);
int e1chunk=-1, e2chunk=-1, e3chunk=-1, e4chunk=-1, n5chunk=-1;
int index = theMod->getIdx();
#ifdef DEBUG_1
CkPrintf("Bisect edge %d->%d on chunk %d\n", n1, n2, theMod->getfmUtil()->getIdx());
#endif
//verify quality
bool flag = theMod->fmAdaptAlgs->controlQualityR(n1,n2,n3,n4);
if(flag) return -1;
//add node
if(e1==-1) e1chunk=-1;
else if(e1>=0) e1chunk=index;
else {
int ghostid = FEM_To_ghost_index(e1);
const IDXL_Rec *irec = theMesh->elem[0].ghost->ghostRecv.getRec(ghostid);
CkAssert(irec->getShared()==1);
e1chunk = irec->getChk(0);
}
if(e2==-1) e2chunk=-1;
else if(e2>=0) e2chunk=index;
else {
int ghostid = FEM_To_ghost_index(e2);
const IDXL_Rec *irec = theMesh->elem[0].ghost->ghostRecv.getRec(ghostid);
CkAssert(irec->getShared()==1);
e2chunk = irec->getChk(0);
}
int adjnodes[2];
adjnodes[0] = n1;
adjnodes[1] = n2;
int *chunks;
int numChunks=0;
int forceshared = 0;
if(e1chunk==e2chunk || (e1chunk==-1 || e2chunk==-1)) {
forceshared = -1;
numChunks = 1;
chunks = new int[1];
if(e1chunk!=-1) chunks[0] = e1chunk;
else chunks[0] = e2chunk;
}
else {
numChunks = 2;
chunks = new int[2];
chunks[0] = e1chunk;
chunks[1] = e2chunk;
}
n5 = FEM_add_node(theMesh,adjnodes,2,chunks,numChunks,forceshared);
delete[] chunks;
//lock this node immediately
FEM_Modify_LockN(theMesh, n5, 0);
//remove elements
e1chunk = FEM_remove_element(theMesh, e1, 0);
e2chunk = FEM_remove_element(theMesh, e2, 0); // assumes intelligent behavior when no e2 exists
// hmm... if e2 is a ghost and we remove it and create all the new elements
// locally, then we don't really need to add a *shared* node
//but we are not moving chunk boundaries for bisect
if(e1chunk==-1 || e2chunk==-1) {
//it is fine, let it continue
e4chunk = e2chunk;
e3chunk = e1chunk;
}
else if(e1chunk==e2chunk && e1chunk!=index) {
n5chunk = e1chunk;
e4chunk = e2chunk;
e3chunk = e1chunk;
}
else {
//there can be a lot of conditions, but we do not have to do aything special now
n5chunk = -1;
e4chunk = e2chunk;
e3chunk = e1chunk;
}
// add n1, n5, n3
elemConn[e1_n1] = n1; elemConn[e1_n2] = n5; elemConn[e1_n3] = n3;
lockelems[0] = FEM_add_element(theMesh, elemConn, 3, 0, e1chunk);
theMod->fmUtil->copyElemData(0,e1,lockelems[0]);
// add n2, n5, n3
elemConn[e1_n1] = n5; elemConn[e1_n2] = n2; elemConn[e1_n3] = n3;
lockelems[1] = FEM_add_element(theMesh, elemConn, 3, 0, e3chunk);
theMod->fmUtil->copyElemData(0,e1,lockelems[1]);
if (e2 != -1) { // e2 exists
// add n1, n5, n4
elemConn[e2_n1] = n1; elemConn[e2_n2] = n5; elemConn[e2_n3] = n4;
lockelems[2] = FEM_add_element(theMesh, elemConn, 3, 0, e2chunk);
theMod->fmUtil->copyElemData(0,e2,lockelems[2]);
// add n2, n5, n4
elemConn[e2_n1] = n5; elemConn[e2_n2] = n2; elemConn[e2_n3] = n4;
lockelems[3] = FEM_add_element(theMesh, elemConn, 3, 0, e4chunk);
theMod->fmUtil->copyElemData(0,e2,lockelems[3]);
}
FEM_purge_element(theMesh,e1,0);
FEM_purge_element(theMesh,e2,0);
//get rid of some unnecessary ghost node sends
for(int i=0; i<4;i++) {
int nodeToUpdate = -1;
if(i==0) nodeToUpdate = n1;
else if(i==1) nodeToUpdate = n2;
else if(i==2) nodeToUpdate = n3;
else if(i==3) nodeToUpdate = n4;
//if any of the chunks sharing this node sends this as a ghost, then all of them have to
//so find out the set of chunks I need to send this as a ghost to
//collect info from each of the shared chunks, do a union of all these chunks
//send this updated list to everyone.
//if anyone needs to add or delete some ghosts, they will
int *chkl, numchkl=0;
CkVec<int> finalchkl;
theMod->fmUtil->findGhostSend(nodeToUpdate, chkl, numchkl);
for(int j=0; j<numchkl; j++) {
finalchkl.push_back(chkl[j]);
}
if(numchkl>0) delete[] chkl;
const IDXL_Rec *irec=theMesh->node.shared.getRec(nodeToUpdate);
int numchunks=0;
int *chunks1, *inds1;
if(irec) {
numchunks = irec->getShared();
chunks1 = new int[numchunks];
inds1 = new int[numchunks];
for(int j=0; j<numchunks; j++) {
chunks1[j] = irec->getChk(j);
inds1[j] = irec->getIdx(j);
}
}
for(int j=0; j<numchunks; j++) {
findgsMsg *fmsg = meshMod[chunks1[j]].findghostsend(index,inds1[j]);
if(fmsg->numchks>0) {
for(int k=0; k<fmsg->numchks; k++) {
bool shouldbeadded = true;
for(int l=0; l<finalchkl.size(); l++) {
if(fmsg->chunks[k]==finalchkl[l]) {
shouldbeadded = false;
break;
}
}
if(shouldbeadded) finalchkl.push_back(fmsg->chunks[k]);
}
}
delete fmsg;
}
int *finall, numfinall=finalchkl.size();
if(numfinall>0) finall = new int[numfinall];
for(int j=0; j<numfinall; j++) finall[j] = finalchkl[j];
finalchkl.free();
theMod->fmUtil->UpdateGhostSend(nodeToUpdate, finall, numfinall);
for(int j=0; j<numchunks; j++) {
verifyghostsendMsg *vmsg = new(numfinall)verifyghostsendMsg();
vmsg->fromChk = index;
vmsg->sharedIdx = inds1[j];
vmsg->numchks = numfinall;
for(int k=0; k<numfinall; k++) vmsg->chunks[k] = finall[k];
meshMod[chunks1[j]].updateghostsend(vmsg);
}
if(numfinall>0) delete[] finall;
if(numchunks>0) {
delete[] chunks1;
delete[] inds1;
}
}
FEM_Modify_UnlockN(theMesh, n5, 0);
return n5;
}
void BaseLB::LDStats::computeNonlocalComm(int &nmsgs, int &nbytes)
{
#if CMK_LBDB_ON
nmsgs = 0;
nbytes = 0;
makeCommHash();
int mcast_count = 0;
for (int cidx=0; cidx < n_comm; cidx++) {
LDCommData& cdata = commData[cidx];
int senderPE, receiverPE;
if (cdata.from_proc())
senderPE = cdata.src_proc;
else {
int idx = getHash(cdata.sender);
if (idx == -1) continue; // sender has just migrated?
senderPE = to_proc[idx];
CmiAssert(senderPE != -1);
}
CmiAssert(senderPE < nprocs() && senderPE >= 0);
// find receiver: point-to-point and multicast two cases
int receiver_type = cdata.receiver.get_type();
if (receiver_type == LD_PROC_MSG || receiver_type == LD_OBJ_MSG) {
if (receiver_type == LD_PROC_MSG)
receiverPE = cdata.receiver.proc();
else { // LD_OBJ_MSG
int idx = getHash(cdata.receiver.get_destObj());
if (idx == -1) { // receiver outside this domain
if (complete_flag) continue;
else receiverPE = -1;
}
else {
receiverPE = to_proc[idx];
CmiAssert(receiverPE < nprocs() && receiverPE >= 0);
}
}
if(senderPE != receiverPE)
{
nmsgs += cdata.messages;
nbytes += cdata.bytes;
}
}
else if (receiver_type == LD_OBJLIST_MSG) {
int nobjs;
LDObjKey *objs = cdata.receiver.get_destObjs(nobjs);
mcast_count ++;
CkVec<int> pes;
for (int i=0; i<nobjs; i++) {
int idx = getHash(objs[i]);
CmiAssert(idx != -1);
if (idx == -1) continue; // receiver has just been removed?
receiverPE = to_proc[idx];
CmiAssert(receiverPE < nprocs() && receiverPE >= 0);
int exist = 0;
for (int p=0; p<pes.size(); p++)
if (receiverPE == pes[p]) { exist=1; break; }
if (exist) continue;
pes.push_back(receiverPE);
if(senderPE != receiverPE)
{
nmsgs += cdata.messages;
nbytes += cdata.bytes;
}
}
}
} // end of for
#endif
}
void readinput(char* filename)
{
FILE *file;
char line[128];
char variable[64];
char value[64];
file = fopen(filename, "r");
if(file == NULL)
{
printf("file read error %s\n", filename);
CkExit();
}
/* parse the header lines to get the number of vertices*/
while(fgets(line, 128, file) != NULL)
{
if(strncmp(line, "DIMENSION", 9) == 0)
{
sscanf(line, "%s : %s", variable, value);
verticesNum = atoi(value);
}else if(strncmp(line, "EDGE_DATA_SECTION", 17) == 0)
{
break;
}
}
vector<int> verticeNbs;
verticeNbs.resize(verticesNum);
/* get the edges, src dest */
int src, dest;
int previous=-1;
int countptr = 0;
int edgeNum = 0;
while(fgets(line, sizeof line, file) != NULL && strncmp(line, "-1", 2) != 0)
{
edgeNum += 1;
sscanf(line, "%d %d", &src, &dest);
if(src != previous)
{
inputGraph.push_back(src-1);
//CkPrintf("\nSource: %d %d:", src, inputGraph[countptr]);
previous = src;
countptr = inputGraph.size();
inputGraph.push_back(1);
}else
{
inputGraph[countptr]++;
}
//CkPrintf(" %d ", dest);
inputGraph.push_back(dest-1);
}
#ifdef YHDEBUG
CkPrintf("\n");
for(int i=0; i<inputGraph.size(); i++)
{
CkPrintf(" %d ", inputGraph[i]);
}
CkPrintf("+++++++++++++===\n");
#endif
fclose(file);
}
void HybridBaseLB::CollectInfo(Location *loc, int n, int fromlevel)
{
int atlevel = fromlevel + 1;
LevelData *lData = levelData[atlevel];
lData->info_recved++;
CkVec<Location> &matchedObjs = lData->matchedObjs;
std::map<LDObjKey, int> &unmatchedObjs = lData->unmatchedObjs;
// sort into matched and unmatched list
#if 0
for (int i=0; i<n; i++) {
// search and see if we have answer, put to matched
// store in unknown
int found = 0;
for (int obj=0; obj<unmatchedObjs.size(); obj++) {
if (loc[i].key == unmatchedObjs[obj].key) {
// answer must exist
CmiAssert(unmatchedObjs[obj].loc != -1 || loc[i].loc != -1);
if (unmatchedObjs[obj].loc == -1) unmatchedObjs[obj].loc = loc[i].loc;
matchedObjs.push_back(unmatchedObjs[obj]);
unmatchedObjs.remove(obj);
found = 1;
break;
}
}
if (!found) unmatchedObjs.push_back(loc[i]);
}
#else
for (int i=0; i<n; i++) {
std::map<LDObjKey, int>::iterator iter = unmatchedObjs.find(loc[i].key);
if (iter != unmatchedObjs.end()) {
CmiAssert(iter->second != -1 || loc[i].loc != -1);
if (loc[i].loc == -1) loc[i].loc = iter->second;
matchedObjs.push_back(loc[i]);
unmatchedObjs.erase(iter);
}
else
unmatchedObjs[loc[i].key] = loc[i].loc;
}
#endif
DEBUGF(("[%d] level %d has %d unmatched and %d matched. \n", CkMyPe(), atlevel, unmatchedObjs.size(), matchedObjs.size()));
if (lData->info_recved == lData->nChildren) {
lData->info_recved = 0;
if (_lb_args.debug() > 1)
CkPrintf("[%d] CollectInfo at level %d started at %f\n",
CkMyPe(), atlevel, CkWallTimer());
if (lData->parent != -1) {
// send only unmatched ones up the tree
CkVec<Location> unmatchedbuf;
for(std::map<LDObjKey, int>::const_iterator it = unmatchedObjs.begin(); it != unmatchedObjs.end(); ++it)
{
unmatchedbuf.push_back(Location(it->first, it->second));
}
thisProxy[lData->parent].CollectInfo(unmatchedbuf.getVec(), unmatchedbuf.size(), atlevel);
}
else { // root
// we should have all answers now
CmiAssert(unmatchedObjs.size() == 0);
// start send match list down
thisProxy.PropagateInfo(matchedObjs.getVec(), matchedObjs.size(), atlevel, lData->nChildren, lData->children);
lData->statsData->clear();
}
}
}
