static int DD_Find_Local (Zoltan_DD_Directory *dd, ZOLTAN_ID_PTR gid, /* incoming GID to locate (in) */ ZOLTAN_ID_PTR lid, /* gid's LID (out) */ ZOLTAN_ID_PTR user, /* gid's user data (out) */ int *partition, /* gid's partition number (out) */ int *owner) /* gid's owner (processor number) (out) */ { DD_Node *ptr; int index; char *yo = "DD_Find_Local"; /* input sanity check */ if (dd == NULL || owner == NULL || gid == NULL) { ZOLTAN_PRINT_ERROR ((dd == NULL) ? 0 : dd->my_proc, yo, "Invalid input"); return ZOLTAN_FATAL; } if (dd->debug_level > 5) ZOLTAN_TRACE_IN (dd->my_proc, yo, NULL); /* compute offset into hash table to find head of linked list */ index = Zoltan_DD_Hash2 (gid, dd->gid_length, dd->table_length); /* walk link list until end looking for matching global ID */ for (ptr = dd->table[index]; ptr != NULL; ptr = ptr->next) if (ZOLTAN_EQ_ID (dd->gid_length, gid, ptr->gid) == TRUE) { /* matching global ID found! Return gid's information */ if (lid) ZOLTAN_SET_ID(dd->lid_length, lid, ptr->gid + dd->gid_length); if (user) ZOLTAN_SET_ID(dd->user_data_length, user,ptr->gid + (dd->gid_length + dd->lid_length)); if (owner) *owner = ptr->owner; if (partition) *partition = ptr->partition; if (dd->debug_level > 5) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_OK; } if (owner != NULL) *owner = -1; /* JDT Added -1 owner not found */ if (dd->debug_level > 5) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); if (dd->debug_level > 0) { ZOLTAN_PRINT_INFO(dd->my_proc, yo, "GID not found"); return ZOLTAN_WARN; } return ZOLTAN_OK; }
int Zoltan_DD_GetLocalKeys(Zoltan_DD_Directory *dd, ZOLTAN_ID_PTR *gid, int *size) { int ierr = ZOLTAN_OK; int i, k; DD_Node *ptr; int gid_alloc_size; gid_alloc_size = dd->table_length; (*gid) = (ZOLTAN_ID_PTR)ZOLTAN_MALLOC( gid_alloc_size*dd->gid_length*sizeof(ZOLTAN_ID_TYPE)); k= 0; for (i = 0; i < dd->table_length; i++) for (ptr = dd->table[i]; ptr != NULL; ptr = ptr->next) { if (k >= gid_alloc_size) { gid_alloc_size *= 2; (*gid) = (ZOLTAN_ID_PTR) ZOLTAN_REALLOC((*gid), gid_alloc_size*dd->gid_length*sizeof(ZOLTAN_ID_TYPE)); } ZOLTAN_SET_ID (dd->gid_length, (*gid)+k*dd->gid_length, ptr->gid); k++; } (*size) = k; return (ierr); }
int Zoltan_DD_Remove ( Zoltan_DD_Directory *dd, /* directory state infomation */ ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to remove */ int count) /* Number of GIDs in removal list */ { int *procs = NULL; /* list of processors to contact */ DD_Remove_Msg *ptr = NULL; ZOLTAN_COMM_OBJ *plan = NULL; /* efficient MPI communication */ char *sbuff = NULL; /* send buffer */ char *sbufftmp = NULL;/* pointer into send buffer */ char *rbuff = NULL; /* receive buffer */ char *rbufftmp = NULL;/* pointer into receive buffer */ int nrec; /* number of receives to expect */ int i; int err; /* error condition to return */ int errcount; /* count of GIDs not found */ char str[100]; /* string to build error messages */ char *yo = "Zoltan_DD_Remove"; /* input sanity checks */ if (dd == NULL || count < 0 || (gid == NULL && count > 0)) { ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo, "Invalid input argument"); return ZOLTAN_FATAL; } if (dd->debug_level > 4) ZOLTAN_TRACE_IN (dd->my_proc, yo, NULL); /* allocate memory for processor contact list */ if (count) { procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count); if (procs == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list"); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_MEMERR; } } /* allocate memory for DD_Remove_Msg send buffer */ if (count) { sbuff = (char*)ZOLTAN_MALLOC((size_t)(dd->remove_msg_size)*(size_t)count); if (sbuff == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer"); err = ZOLTAN_MEMERR; goto fini; } } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After proc & sbuff mallocs"); /* for each GID, fill in contact list and then message structure */ sbufftmp = sbuff; for (i = 0; i < count; i++) { procs[i] = dd->hash(gid + i*dd->gid_length, dd->gid_length, dd->nproc, dd->hashdata, dd->hashfn); ptr = (DD_Remove_Msg*) sbufftmp; sbufftmp += dd->remove_msg_size; ptr->owner = dd->my_proc; ZOLTAN_SET_ID (dd->gid_length, ptr->gid, gid + i * dd->gid_length); } /* now create efficient communication plan */ err = Zoltan_Comm_Create (&plan, count, procs, dd->comm, ZOLTAN_DD_REMOVE_MSG_TAG, &nrec); if (err != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Create error"); goto fini; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After Zoltan_Comm_Create"); /* allocate receive buffer for nrec DD_Remove_Msg structures */ if (nrec) { rbuff = (char*)ZOLTAN_MALLOC((size_t)nrec*(size_t)(dd->remove_msg_size)); if (rbuff == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Receive buffer malloc failed"); err = ZOLTAN_MEMERR; goto fini; } } /* send my remove messages & receive removes directed to me */ err = Zoltan_Comm_Do (plan, ZOLTAN_DD_UPDATE_MSG_TAG+1, sbuff, dd->remove_msg_size, rbuff); if (err != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Do error"); goto fini; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After Zoltan_Comm_Do"); /* for each message rec'd, remove local directory info */ errcount = 0; rbufftmp = rbuff; for (i = 0; i < nrec; i++) { ptr = (DD_Remove_Msg*) rbufftmp; rbufftmp += dd->remove_msg_size; err = DD_Remove_Local (dd, ptr->gid); if (err == ZOLTAN_WARN) ++errcount; } err = ZOLTAN_OK; if (dd->debug_level) { sprintf (str, "Processed %d GIDs (%d local), %d GIDs not found", count, nrec, errcount); ZOLTAN_PRINT_INFO (dd->my_proc, yo, str); err = (errcount) ? ZOLTAN_WARN : ZOLTAN_OK; } /* done, now free up things and return */ fini: ZOLTAN_FREE (&procs); ZOLTAN_FREE (&sbuff); ZOLTAN_FREE (&rbuff); Zoltan_Comm_Destroy (&plan); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return err; }
int Zoltan_DD_Update ( Zoltan_DD_Directory *dd, /* directory state information */ ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to update */ ZOLTAN_ID_PTR lid, /* Incoming corresponding LIDs (optional) */ char *user, /* Incoming list of user data (optional) */ int *partition, /* Optional, grouping of GIDs to partitions */ int count) /* Number of GIDs in update list */ { int *procs = NULL; /* list of processors to contact */ DD_Update_Msg *ptr = NULL; ZOLTAN_COMM_OBJ *plan = NULL; /* for efficient MPI communication */ char *sbuff = NULL; /* send buffer */ char *sbufftmp = NULL;/* pointer into send buffer */ char *rbuff = NULL; /* receive buffer */ char *rbufftmp = NULL;/* pointer into receive buffer */ int nrec = 0; /* number of receives to expect */ int i; int err; int errcount = 0; /* count of GIDs not found, added */ char str[100]; /* build error message string */ char *yo = "Zoltan_DD_Update"; /* input sanity checking */ if (dd == NULL || count < 0 || (gid == NULL && count > 0)) { ZOLTAN_PRINT_ERROR ((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc), yo, "Invalid input argument"); return ZOLTAN_FATAL; } if (dd->debug_level > 4) ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL); /* part of initializing the error checking process */ /* for each linked list head, walk its list resetting errcheck */ if (dd->debug_level) for (i = 0; i < dd->table_length; i++) { DD_NodeIdx nodeidx; for (nodeidx = dd->table[i]; nodeidx != -1; nodeidx = dd->nodelist[nodeidx].next) dd->nodelist[nodeidx].errcheck = ZOLTAN_DD_NO_PROC; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After reset errcheck"); /* allocate memory for list of processors to contact */ if (count) { procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count); if (procs == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list"); err = ZOLTAN_MEMERR; goto fini; } } /* allocate memory for DD_Update_Msg send buffer */ if (count) { sbuff = (char*) ZOLTAN_CALLOC (count, dd->update_msg_size); if (sbuff == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer"); err = ZOLTAN_MEMERR; goto fini; } } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs"); /* for each GID given, fill in contact list and then message structure */ sbufftmp = sbuff; for (i = 0; i < count; i++) { procs[i] = dd->hash(gid + i*dd->gid_length, dd->gid_length, dd->nproc, dd->hashdata, dd->hashfn); ptr = (DD_Update_Msg*) sbufftmp; sbufftmp += dd->update_msg_size; ptr->lid_flag = (lid) ? 1 : 0; ptr->user_flag = (user) ? 1 : 0; ptr->partition_flag = (partition) ? 1 : 0; ptr->partition = (partition) ? *(partition + i) : -1; ptr->owner = dd->my_proc; ZOLTAN_SET_ID (dd->gid_length, ptr->gid, gid + i * dd->gid_length); if (lid) { ZOLTAN_SET_ID(dd->lid_length, ptr->gid + dd->gid_length, lid + i * dd->lid_length); } else { memset(ptr->gid + dd->gid_length, 0, dd->lid_length); } if (user) { memcpy(ptr->gid + (dd->gid_length + dd->lid_length), user + (size_t)i * (size_t)(dd->user_data_length), dd->user_data_length); } else { memset(ptr->gid + (dd->gid_length + dd->lid_length), 0, dd->user_data_length); } } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill contact list"); /* now create efficient communication plan */ err = Zoltan_Comm_Create (&plan, count, procs, dd->comm, ZOLTAN_DD_UPDATE_MSG_TAG, &nrec); if (err != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Create error"); goto fini; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create"); /* If dd has no nodes allocated (e.g., first call to DD_Update; * create the nodelist and freelist */ if (nrec && dd->nodelistlen == 0) { DD_Memory_Alloc_Nodelist(dd, (DD_NodeIdx) nrec, 0.); /* TODO Add overalloc parameter */ } /* allocate receive buffer for nrec DD_Update_Msg structures */ if (nrec) { rbuff = (char*)ZOLTAN_MALLOC((size_t)nrec*(size_t)(dd->update_msg_size)); if (rbuff == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Receive buffer malloc failed"); err = ZOLTAN_MEMERR; goto fini; } } /* send my update messages & receive updates directed to me */ err = Zoltan_Comm_Do (plan, ZOLTAN_DD_UPDATE_MSG_TAG+1, sbuff, dd->update_msg_size, rbuff); if (err != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Do error"); goto fini; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do"); /* for each message rec'd, update local directory information */ errcount = 0; rbufftmp = rbuff; for (i = 0; i < nrec; i++) { ptr = (DD_Update_Msg *) rbufftmp; rbufftmp += dd->update_msg_size; err = DD_Update_Local (dd, ptr->gid, (ptr->lid_flag) ? (ptr->gid + dd->gid_length) : NULL, (char*)((ptr->user_flag)?(ptr->gid+(dd->gid_length+dd->lid_length)):NULL), (ptr->partition_flag) ? (ptr->partition) : -1, /* illegal partition */ ptr->owner); if (err != ZOLTAN_OK) ++errcount; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Local update"); err = ZOLTAN_OK; if (dd->debug_level) /* overwrite error return if extra checking is on */ err = (errcount) ? ZOLTAN_WARN : ZOLTAN_OK; fini: ZOLTAN_FREE (&procs); ZOLTAN_FREE (&sbuff); ZOLTAN_FREE (&rbuff); Zoltan_Comm_Destroy (&plan); if (dd->debug_level) { sprintf (str, "Processed %d GIDs (%d local), %d GID errors", count, nrec, errcount); ZOLTAN_PRINT_INFO (dd->my_proc, yo, str); } if (dd->debug_level > 4) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return err; }
static int DD_Update_Local (Zoltan_DD_Directory *dd, ZOLTAN_ID_PTR gid, /* GID to update (in) */ ZOLTAN_ID_PTR lid, /* gid's LID (in), NULL if not needed */ char *user, /* gid's user data (in), NULL if not needed */ int partition, /* gid's partition (in), -1 if not used */ int owner) /* gid's current owner (proc number) (in) */ { int index; char *yo = "DD_Update_Local"; DD_NodeIdx nodeidx; DD_Node *ptr; /* input sanity checking */ if (dd == NULL || owner < 0 || owner >= dd->nproc || gid == NULL) { ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo, "Invalid input parameter"); return ZOLTAN_FATAL; } if (dd->debug_level > 5) ZOLTAN_TRACE_IN (dd->my_proc, yo, NULL); /* compute offset into hash table to find head of linked list */ index = Zoltan_DD_Hash2 (gid, dd->gid_length, dd->table_length, dd->hashdata, NULL); /* walk linked list until end looking for matching gid */ for (nodeidx = dd->table[index]; nodeidx != -1; nodeidx = dd->nodelist[nodeidx].next) { ptr = dd->nodelist + nodeidx; if (ZOLTAN_EQ_ID (dd->gid_length, gid, ptr->gid) == TRUE) { /* found match, update directory information */ if (lid) ZOLTAN_SET_ID (dd->lid_length,ptr->gid + dd->gid_length, lid); if (user) memcpy(ptr->gid + (dd->gid_length + dd->lid_length), user, dd->user_data_length); ptr->owner = owner; if (partition != -1) ptr->partition = partition; /* Response to multiple updates to a gid in 1 update cycle */ if (dd->debug_level > 0 && ptr->errcheck != owner) { ZOLTAN_PRINT_INFO (dd->my_proc, yo, "Multiply defined GID"); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_WARN; } ptr->errcheck = owner; if (dd->debug_level > 5) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_OK; /* ignore all errors */ } } /* gid not found. Create new DD_Node and fill it in */ nodeidx = DD_Memory_Alloc_Node(dd); ptr = dd->nodelist + nodeidx; ZOLTAN_SET_ID (dd->gid_length, ptr->gid, gid); if (lid) { ZOLTAN_SET_ID(dd->lid_length,ptr->gid + dd->gid_length, lid); } else { memset(ptr->gid + dd->gid_length, 0, dd->lid_length*sizeof(ZOLTAN_ID_TYPE)); } if (user) { memcpy(ptr->gid + (dd->gid_length + dd->lid_length), user, dd->user_data_length); } else { memset(ptr->gid + (dd->gid_length+dd->lid_length), 0, dd->user_data_length); } ptr->partition = partition; ptr->owner = owner; ptr->errcheck = owner; /* Add node to the linked list */ ptr->next = dd->table[index]; dd->table[index] = nodeidx; if (dd->debug_level > 6) ZOLTAN_PRINT_INFO (dd->my_proc, yo, "Created new directory item"); if (dd->debug_level > 5) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_OK; }
int main (int argc, char *argv[]) { Zoltan_DD_Directory *dd; ZTIMER *zt; int myproc; /* MPI rank, my processor's MPI ID */ int nproc; /* MPI size, number of processors */ ZOLTAN_ID_PTR glist = NULL; /* list of GIDs */ ZOLTAN_ID_PTR llist = NULL; /* list of LIDs */ ZOLTAN_ID_PTR ulist = NULL; /* list of user data of type ZOLTAN_ID_TYPE */ int *plist = NULL; /* list of partitions */ int *olist = NULL; /* list of owners */ Param param ; /* program's adjustable parameters */ char *store = NULL; /* non directory storage of test data */ Data *data = NULL; /* pointer into store */ /* these are all temporary variables */ int new_owner; int count; int i; char *p; int err; int errcount; int loop; char str[100]; /* for building output messages */ static int timer[7] = {-1,-1,-1,-1,-1,-1,-1}; char *yo = "DD_Main"; /* initialize MPI communications, ignore errors */ MPI_Init (&argc, &argv); MPI_Comm_rank (MPI_COMM_WORLD, &myproc); MPI_Comm_size (MPI_COMM_WORLD, &nproc); get_params (¶m); /* read input parameters */ zt = Zoltan_Timer_Create(1); MACRO_TIMER_START(5, "Total time", 0); MACRO_TIMER_START(0, "DD_Create time", 0); err = Zoltan_DD_Create (&dd, MPI_COMM_WORLD, param.glen, param.llen, param.ulen, param.tlen, param.debug_level); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Create"); MACRO_TIMER_STOP(0); param.slen = sizeof (Data) + sizeof(ZOLTAN_ID_TYPE) * (param.glen + param.llen + param.ulen); param.slen = Zoltan_Align(param.slen); store = (char*) ZOLTAN_MALLOC (param.count * param.slen); /* allocate storage for various lists */ glist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC(sizeof(ZOLTAN_ID_TYPE) * param.count * param.glen); plist = (int*) ZOLTAN_MALLOC (sizeof(int) * param.count); olist = (int*) ZOLTAN_MALLOC (sizeof(int) * param.count); if (param.llen) llist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC (sizeof (ZOLTAN_ID_TYPE) * param.count * param.llen); if (param.ulen) ulist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC (sizeof (ZOLTAN_ID_TYPE) * param.count * param.ulen) ; if (store == NULL || glist == NULL || (param.llen != 0 && llist == NULL) || (param.ulen != 0 && ulist == NULL) || plist == NULL || olist == NULL) { ZOLTAN_PRINT_ERROR (myproc, yo, "Unable to malloc storage lists"); return ZOLTAN_MEMERR; } initialize_data (¶m, store, nproc); /* create & update directory with myproc's initial simulated GIDs */ count = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) if (((Data *)p)->new_owner == myproc) { ZOLTAN_SET_ID (param.glen, glist + count * param.glen, ((Data*)p)->id); if (param.llen) ZOLTAN_SET_ID (param.llen, llist + count * param.llen, ((Data *)p)->id + param.glen); if (param.ulen) ZOLTAN_SET_ID (param.ulen, ulist + count * param.ulen, ((Data *)p)->id + (param.glen + param.llen)); plist [count] = ((Data *)p)->partition; ++count; } MACRO_TIMER_START (1, "DD_Update timer", 0); err = Zoltan_DD_Update (dd, glist, llist, ulist, plist, count); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update"); MACRO_TIMER_STOP(1); i = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) { ZOLTAN_SET_ID (param.glen, glist + i * param.glen, ((Data *)p)->id); ++i; } MACRO_TIMER_START(2, "DD_Find timer", 0); err = Zoltan_DD_Find (dd, glist, llist, ulist, plist, param.count, olist); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find"); MACRO_TIMER_STOP(2); errcount = 0; for (i = 0; i < param.count; i++) if (olist[i] != ((Data *) (store + i * param.slen))->new_owner) ++errcount; if (errcount) sprintf (str, "FIRST TEST FAILED, errcount is %d", errcount); else sprintf (str, "FIRST TEST SUCCESSFUL"); ZOLTAN_PRINT_INFO (myproc, yo, str); /* repeatedly simulate moving "dots" around the system */ for (loop = 0; loop < param.nloops; loop++) { for (p = store; p < store + param.count * param.slen; p += param.slen) ((Data*) p)->old_owner = ((Data*) p)->new_owner; /* randomly exchange some dots and randomly reassign others */ for (i = 0; i < (param.pmove * param.count)/100; i++) { Data *d1, *d2; d1 = (Data*) (store + param.slen * (rand() % param.count)); d2 = (Data*) (store + param.slen * (rand() % param.count)); new_owner = d1->new_owner; d1->new_owner = d2->new_owner; d2->new_owner = new_owner; } for (i = 0; i < (param.count * param.pscatter)/100; i++) ((Data*) (store + param.slen *(rand() % param.count)))->new_owner = rand() % nproc; /* determine which new GIDs myproc gained, and update directory */ count = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) if (((Data*)p)->new_owner == myproc) { ((Data*)p)->id[param.glen] = count; /* set LID */ ZOLTAN_SET_ID (param.glen, glist + count * param.glen, ((Data *)p)->id); if (param.llen) ZOLTAN_SET_ID (param.llen, llist + count * param.llen, ((Data*)p)->id + param.glen); if (param.ulen) ZOLTAN_SET_ID (param.ulen, ulist + count * param.ulen, ((Data*)p)->id + (param.glen + param.llen)); plist [count] = ((Data *)p)->partition; ++count; } MACRO_TIMER_START(1, "DD_Update", 0); err = Zoltan_DD_Update (dd, glist, llist, ulist, plist, count); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update"); MACRO_TIMER_STOP(1); /* use directory to "find" GIDs */ i = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) { ZOLTAN_SET_ID (param.glen, glist + i * param.glen, ((Data *)p)->id); ++i; } MACRO_TIMER_START(2, "DD_Find timer", 0); if (loop % 5 == 0) err = Zoltan_DD_Find(dd,glist,NULL,ulist,plist,param.count,olist); else if (loop % 7 == 0) err = Zoltan_DD_Find(dd,glist,llist,NULL,plist,param.count,olist); else if (loop % 9 == 0) err = Zoltan_DD_Find(dd,glist,llist,ulist,NULL,param.count,olist); else if (loop % 2 == 0) err = Zoltan_DD_Find(dd,glist,llist,ulist,plist,param.count,NULL); else err = Zoltan_DD_Find(dd,glist,llist,ulist,plist,param.count,olist); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find"); MACRO_TIMER_STOP(2); if (loop % 2) { errcount = 0 ; for (i = 0; i < param.count; i++) if (olist[i] != ((Data *)(store + i * param.slen))->new_owner) ++errcount; if (errcount) sprintf (str,"LOOP %d TEST FAILED, errcount is %d",loop,errcount); else sprintf (str, "LOOP %d TEST SUCCESSFUL", loop); ZOLTAN_PRINT_INFO (myproc, yo, str) ; } else { sprintf (str, "LOOP %d Completed", loop); ZOLTAN_PRINT_INFO (myproc, yo, str); } /* randomly remove a percentage of GIDs from the directory */ count = 0; for (i = 0; i < (param.count * param.pdelete)/100; i++) { data = (Data*) (store + param.slen * (rand() % param.count)); if (data->new_owner == myproc) { ZOLTAN_SET_ID (param.glen, glist + count * param.glen, data->id); ++count; } data->new_owner = NO_PROC; } MACRO_TIMER_START(3, "DD_Remove timer", 0); err = Zoltan_DD_Remove (dd, glist, count); MACRO_TIMER_STOP(3); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Remove"); /* update directory (put directory entries back in) */ for (p = store; p < store + param.count * param.slen; p += param.slen) if (((Data*)p)->new_owner == NO_PROC) ((Data*)p)->new_owner = loop % nproc; /* place in new location */ count = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) if (((Data*)p)->new_owner == myproc) { ZOLTAN_SET_ID(param.glen,glist+count*param.glen,((Data*)p)->id); if (param.llen) ZOLTAN_SET_ID (param.llen, llist + count * param.llen, ((Data*)p)->id + param.glen); if (param.ulen) ZOLTAN_SET_ID(param.ulen, ulist + count * param.ulen, ((Data *)p)->id + (param.glen + param.llen)); plist [count] = ((Data*)p)->partition; ++count; } MACRO_TIMER_START(1, "DD_Update timer", 0); err = Zoltan_DD_Update (dd, glist, NULL, NULL, NULL, count); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update"); MACRO_TIMER_STOP(1); } /* now Find directory info for GIDs myproc now owns and validate */ count = 0; for (i = 0; i < param.count; i++) { data = (Data *) (store + i * param.slen); if (data->new_owner == myproc) { ZOLTAN_SET_ID (param.glen, glist + count * param.glen, data->id); ++count; } } MACRO_TIMER_START(2, "DD_Find", 0); err = Zoltan_DD_Find (dd, glist, NULL, NULL, NULL, count, olist); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find"); MACRO_TIMER_STOP(2); errcount = 0; for (i = 0; i < count; i++) if (olist[i] != myproc) ++errcount; if (errcount) { sprintf (str, "errcount is %d", errcount); ZOLTAN_PRINT_ERROR (myproc, yo, str); } else ZOLTAN_PRINT_INFO (myproc, yo, "TEST SUCCESSFUL"); Zoltan_DD_Stats (dd); /* done, now free memory, stop MPI & directory, return */ ZOLTAN_FREE (&store); ZOLTAN_FREE (&glist); ZOLTAN_FREE (&plist); ZOLTAN_FREE (&olist); if (param.llen) ZOLTAN_FREE (&llist); if (param.ulen) ZOLTAN_FREE (&ulist); ZOLTAN_PRINT_INFO (myproc, yo, "Completing program"); MACRO_TIMER_START(4, "DD_Destroy", 0); Zoltan_DD_Destroy (&dd); MACRO_TIMER_STOP(4); MACRO_TIMER_STOP(5); Zoltan_Timer_PrintAll(zt, 0, MPI_COMM_WORLD, stderr); MPI_Finalize (); return ZOLTAN_OK; }
int Zoltan_DD_Find ( Zoltan_DD_Directory *dd, /* contains directory state information */ ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to get owners proc */ ZOLTAN_ID_PTR lid, /* Outgoing corresponding list of LIDs */ char *data, /* Outgoing optional corresponding user data */ int *partition, /* Outgoing optional partition information */ int count, /* Count of GIDs in above list (in) */ int *owner) /* Outgoing optional list of data owners */ { ZOLTAN_COMM_OBJ *plan = NULL; /* efficient MPI communication */ char *rbuff = NULL; /* receive buffer */ char *rbufftmp = NULL; /* pointer into receive buffer */ char *sbuff = NULL; /* send buffer */ char *sbufftmp = NULL; /* pointer into send buffer */ int *procs = NULL; /* list of processors to contact */ DD_Find_Msg *ptr = NULL; int i; int nrec; /* number of messages to receive */ int err = ZOLTAN_OK; /* return error condition */ int errcount; /* count of GIDs not found */ char *yo = "Zoltan_DD_Find"; /* input sanity check */ if (dd == NULL || count < 0 || (gid == NULL && count > 0)) { ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo, "Invalid input argument"); return ZOLTAN_FATAL; } if (dd->debug_level > 4) ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL); /* allocate memory for processors to contact for directory info */ if (count) { procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count); if (procs == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list"); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return ZOLTAN_MEMERR; } } /* allocate memory for DD_Find_Msg send buffer */ if (count) { sbuff = (char*) ZOLTAN_CALLOC (count, dd->find_msg_size); if (sbuff == NULL) { ZOLTAN_FREE (&procs); ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer"); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return ZOLTAN_MEMERR; } } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs"); /* for each GID, fill DD_Find_Msg buffer and contact list */ sbufftmp = sbuff; for (i = 0; i < count; i++) { procs[i] = dd->hash (gid + i*dd->gid_length, dd->gid_length, dd->nproc, dd->hashdata, dd->hashfn); ptr = (DD_Find_Msg*) sbufftmp; sbufftmp += dd->find_msg_size; ptr->index = i; ptr->proc = procs[i]; ZOLTAN_SET_ID (dd->gid_length, ptr->id, gid + i*dd->gid_length); } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill"); /* create efficient communication plan */ err = Zoltan_Comm_Create (&plan, count, procs, dd->comm, ZOLTAN_DD_FIND_MSG_TAG, &nrec); if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create"); if (err != ZOLTAN_OK) goto fini; /* allocate receive buffer */ if (nrec) { rbuff = (char*) ZOLTAN_MALLOC ((size_t)nrec*(size_t)(dd->find_msg_size)); if (rbuff == NULL) { err = ZOLTAN_MEMERR; goto fini; } } /* send out find messages across entire system */ err = Zoltan_Comm_Do (plan, ZOLTAN_DD_FIND_MSG_TAG+1, sbuff, dd->find_msg_size, rbuff); if (err != ZOLTAN_OK) goto fini; if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do"); /* get find messages directed to me, fill in return information */ errcount = 0; rbufftmp = rbuff; for (i = 0; i < nrec; i++) { ptr = (DD_Find_Msg*) rbufftmp; rbufftmp += dd->find_msg_size; err = DD_Find_Local (dd, ptr->id, ptr->id, (char *)(ptr->id + dd->max_id_length), &ptr->partition, &ptr->proc); if (err == ZOLTAN_WARN) ++errcount; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill in return info"); /* send return information back to requester */ err = Zoltan_Comm_Do_Reverse(plan, ZOLTAN_DD_FIND_MSG_TAG+2, rbuff, dd->find_msg_size, NULL, sbuff); if (err != ZOLTAN_OK) goto fini; if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Reverse"); /* fill in user supplied lists with returned information */ sbufftmp = sbuff; for (i = 0; i < count; i++) { ptr = (DD_Find_Msg*) sbufftmp; sbufftmp += dd->find_msg_size; if (owner) owner[ptr->index] = ptr->proc; if (partition) partition[ptr->index] = ptr->partition ; if (lid) ZOLTAN_SET_ID(dd->lid_length,lid+ptr->index*dd->lid_length,ptr->id); if (data) memcpy(data + (size_t)(ptr->index) * (size_t)(dd->user_data_length), ptr->id + dd->max_id_length, dd->user_data_length); } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill return lists"); /* err = ZOLTAN_OK; */ MPI_Allreduce(&errcount, &err, 1, MPI_INT, MPI_SUM, dd->comm); err = (err) ? ZOLTAN_WARN : ZOLTAN_OK; /* if at least one GID was not found, potentially notify caller of error */ if (dd->debug_level > 0) { char str[100]; /* diagnostic message string */ sprintf (str, "Processed %d GIDs, GIDs not found: %d", count, errcount); ZOLTAN_PRINT_INFO (dd->my_proc, yo, str); } fini: ZOLTAN_FREE (&sbuff); ZOLTAN_FREE (&rbuff); ZOLTAN_FREE (&procs) ; Zoltan_Comm_Destroy (&plan); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return err; }
int Zoltan_DD_Find ( Zoltan_DD_Directory *dd, /* contains directory state information */ ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to get owners proc */ ZOLTAN_ID_PTR lid, /* Outgoing corresponding list of LIDs */ ZOLTAN_ID_PTR data, /* Outgoing optional corresponding user data */ int *partition, /* Outgoing optional partition information */ int count, /* Count of GIDs in above list (in) */ int *owner) /* Outgoing corresponding list of data locations */ { ZOLTAN_COMM_OBJ *plan = NULL ; /* efficient MPI communication */ char *rbuff = NULL ; /* receive buffer */ char *sbuff = NULL ; /* send buffer */ int *procs = NULL ; /* list of processors to contact */ DD_Find_Msg *ptr = NULL ; int i ; int nrec ; /* number of messages to receive */ int err ; /* return error condition */ int errcount ; /* count of GIDs not found */ char *yo = "Zoltan_DD_Find" ; if (dd != NULL && dd->debug_level > 1) ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL); /* input sanity check */ if (dd == NULL || count < 0 || ((owner == NULL || gid == NULL) && count > 0)) { ZOLTAN_PRINT_ERROR ((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc), yo, "Invalid input argument.") ; if (dd != NULL && dd->debug_level > 1) ZOLTAN_TRACE_OUT((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc), yo, NULL); return ZOLTAN_DD_INPUT_ERROR ; } /* allocate memory for processors to contact for directory info */ if (count > 0) { procs = (int *) ZOLTAN_MALLOC (sizeof (int) * count) ; if (procs == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list.") ; if (dd->debug_level > 1) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return ZOLTAN_DD_MEMORY_ERROR ; } } /* allocate memory for DD_Find_Msg send buffer */ if (count > 0) { sbuff = (char *) ZOLTAN_MALLOC (dd->find_msg_size * count) ; if (sbuff == NULL) { ZOLTAN_FREE (&procs) ; ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer.") ; if (dd->debug_level > 1) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return ZOLTAN_DD_MEMORY_ERROR ; } } if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs."); /* for each GID, fill DD_Find_Msg buffer and contact list */ for (i = 0 ; i < count ; i++) { procs[i] = dd->hash (gid + i*dd->gid_length, dd->gid_length, dd->nproc) ; ptr = (DD_Find_Msg *) (sbuff + i * dd->find_msg_size) ; ptr->index = i ; ptr->proc = procs[i] ; ZOLTAN_SET_ID (dd->gid_length, ptr->id, gid + i*dd->gid_length) ; } if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill."); /* create efficient communication plan */ err = Zoltan_Comm_Create (&plan, count, procs, dd->comm, ZOLTAN_DD_FIND_MSG_TAG, &nrec) ; if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create."); if (err != ZOLTAN_OK) goto fini ; /* allocate receive buffer */ if (nrec > 0) { rbuff = (char *) ZOLTAN_MALLOC (nrec * dd->find_msg_size) ; if (rbuff == NULL) { err = ZOLTAN_DD_MEMORY_ERROR ; goto fini ; } } /* send out find messages across entire system */ err = Zoltan_Comm_Do (plan, ZOLTAN_DD_FIND_MSG_TAG+1, sbuff, dd->find_msg_size, rbuff) ; if (err != ZOLTAN_OK) goto fini ; if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do."); /* get find messages directed to me, fill in return information */ errcount = 0 ; for (i = 0 ; i < nrec ; i++) { ptr = (DD_Find_Msg *) (rbuff + i*dd->find_msg_size) ; err = DD_Find_Local (dd, ptr->id, ptr->id, ptr->id + dd->max_id_length, (partition) ? (&ptr->partition) : NULL, &ptr->proc) ; if (err == ZOLTAN_DD_GID_NOT_FOUND_ERROR) errcount++ ; } if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill in return info."); /* send return information back to requester */ err = Zoltan_Comm_Do_Reverse(plan, ZOLTAN_DD_FIND_MSG_TAG+2, rbuff, dd->find_msg_size, NULL, sbuff) ; if (err != ZOLTAN_OK) goto fini ; if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Reverse."); /* fill in user supplied lists with returned information */ for (i = 0 ; i < count ; i++) { ptr = (DD_Find_Msg *) (sbuff + i*dd->find_msg_size) ; owner[ptr->index] = ptr->proc ; if (partition != NULL) partition[ptr->index] = ptr->partition ; if (lid != NULL) ZOLTAN_SET_ID (dd->lid_length, lid + ptr->index * dd->lid_length, ptr->id) ; if (data != NULL) ZOLTAN_SET_ID (dd->user_data_length, data + ptr->index * dd->user_data_length, ptr->id + dd->max_id_length) ; } /* if at least one GID was not found, notify caller of error */ err = (errcount == 0) ? ZOLTAN_DD_NORMAL_RETURN : ZOLTAN_DD_GID_NOT_FOUND_ERROR ; if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill return lists."); fini: ZOLTAN_FREE (&sbuff) ; ZOLTAN_FREE (&rbuff) ; ZOLTAN_FREE (&procs) ; Zoltan_Comm_Destroy (&plan) ; if (err != ZOLTAN_DD_NORMAL_RETURN) ZOLTAN_PRINT_WARN (dd->my_proc, yo, "Return is not normal.") ; if (dd->debug_level > 0) { char str[100] ; /* diagnostic message string */ sprintf (str, "Processed %d GIDs, GIDs not found: %d", count, errcount) ; ZOLTAN_PRINT_INFO (dd->my_proc, yo, str) ; } if (dd->debug_level > 1) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return err ; }
static int DD_Update_Local (Zoltan_DD_Directory *dd, ZOLTAN_ID_PTR gid, /* GID to update (in) */ ZOLTAN_ID_PTR lid, /* gid's LID (in), NULL if not needed */ ZOLTAN_ID_PTR user, /* gid's user data (in), NULL if not needed */ int partition, /* gid's partition (in), -1 if not used */ int owner) /* gid's current owner (proc number) (in) */ { DD_Node **ptr; int index; char *yo = "DD_Update_Local"; /* input sanity checking */ if (dd == NULL || owner < 0 || owner >= dd->nproc || gid == NULL) { ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo, "Invalid input parameter"); return ZOLTAN_FATAL; } if (dd->debug_level > 5) ZOLTAN_TRACE_IN (dd->my_proc, yo, NULL); /* compute offset into hash table to find head of linked list */ index = Zoltan_DD_Hash2 (gid, dd->gid_length, dd->table_length, dd->hashdata); /* walk linked list until end looking for matching gid */ for (ptr = dd->table+index; *ptr != NULL; ptr = &((*ptr)->next)) if (ZOLTAN_EQ_ID (dd->gid_length, gid, (*ptr)->gid) == TRUE) { /* found match, update directory information */ if (lid) ZOLTAN_SET_ID (dd->lid_length,(*ptr)->gid + dd->gid_length, lid); if (user) ZOLTAN_SET_ID (dd->user_data_length, (*ptr)->gid + (dd->gid_length + dd->lid_length), user); (*ptr)->owner = owner; if (partition != -1) (*ptr)->partition = partition; /* Response to multiple updates to a gid in 1 update cycle */ if (dd->debug_level > 0 && (*ptr)->errcheck != owner) { ZOLTAN_PRINT_INFO (dd->my_proc, yo, "Multiply defined GID"); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_WARN; } (*ptr)->errcheck = owner; if (dd->debug_level > 5) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_OK; /* ignore all errors */ } /* gid not found. Create new DD_Node and fill it in */ *ptr = (DD_Node*) ZOLTAN_MALLOC (dd->node_size); if (*ptr == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc new Node"); if (dd->debug_level > 5) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_MEMERR; } ZOLTAN_SET_ID (dd->gid_length, (*ptr)->gid, gid); if (lid) { ZOLTAN_SET_ID(dd->lid_length,(*ptr)->gid + dd->gid_length, lid); } else { memset((*ptr)->gid + dd->gid_length, 0, dd->lid_length*sizeof(ZOLTAN_ID_TYPE)); } if (user) { ZOLTAN_SET_ID(dd->user_data_length, (*ptr)->gid + (dd->gid_length + dd->lid_length), user); } else { memset((*ptr)->gid + (dd->gid_length+dd->lid_length), 0, dd->user_data_length*sizeof(ZOLTAN_ID_TYPE)); } (*ptr)->partition = partition ; (*ptr)->next = NULL; (*ptr)->owner = owner; (*ptr)->errcheck = owner; if (dd->debug_level > 6) ZOLTAN_PRINT_INFO (dd->my_proc, yo, "Created new directory item"); if (dd->debug_level > 5) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_OK; }