static int DD_Find_Local (Zoltan_DD_Directory *dd,
ZOLTAN_ID_PTR gid, /* incoming GID to locate (in) */
ZOLTAN_ID_PTR lid, /* gid's LID (out) */
ZOLTAN_ID_PTR user, /* gid's user data (out) */
int *partition, /* gid's partition number (out) */
int *owner) /* gid's owner (processor number) (out) */
{
DD_Node *ptr;
int index;
char *yo = "DD_Find_Local";
/* input sanity check */
if (dd == NULL || owner == NULL || gid == NULL) {
ZOLTAN_PRINT_ERROR ((dd == NULL) ? 0 : dd->my_proc, yo, "Invalid input");
return ZOLTAN_FATAL;
}
if (dd->debug_level > 5)
ZOLTAN_TRACE_IN (dd->my_proc, yo, NULL);
/* compute offset into hash table to find head of linked list */
index = Zoltan_DD_Hash2 (gid, dd->gid_length, dd->table_length);
/* walk link list until end looking for matching global ID */
for (ptr = dd->table[index]; ptr != NULL; ptr = ptr->next)
if (ZOLTAN_EQ_ID (dd->gid_length, gid, ptr->gid) == TRUE) {
/* matching global ID found! Return gid's information */
if (lid) ZOLTAN_SET_ID(dd->lid_length, lid, ptr->gid + dd->gid_length);
if (user) ZOLTAN_SET_ID(dd->user_data_length, user,ptr->gid
+ (dd->gid_length + dd->lid_length));
if (owner) *owner = ptr->owner;
if (partition) *partition = ptr->partition;
if (dd->debug_level > 5)
ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL);
return ZOLTAN_OK;
}
if (owner != NULL)
*owner = -1; /* JDT Added -1 owner not found */
if (dd->debug_level > 5)
ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL);
if (dd->debug_level > 0) {
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "GID not found");
return ZOLTAN_WARN;
}
return ZOLTAN_OK;
}
int Zoltan_DD_GetLocalKeys(Zoltan_DD_Directory *dd,
ZOLTAN_ID_PTR *gid,
int *size)
{
int ierr = ZOLTAN_OK;
int i, k;
DD_Node *ptr;
int gid_alloc_size;
gid_alloc_size = dd->table_length;
(*gid) = (ZOLTAN_ID_PTR)ZOLTAN_MALLOC(
gid_alloc_size*dd->gid_length*sizeof(ZOLTAN_ID_TYPE));
k= 0;
for (i = 0; i < dd->table_length; i++)
for (ptr = dd->table[i]; ptr != NULL; ptr = ptr->next) {
if (k >= gid_alloc_size) {
gid_alloc_size *= 2;
(*gid) = (ZOLTAN_ID_PTR)
ZOLTAN_REALLOC((*gid),
gid_alloc_size*dd->gid_length*sizeof(ZOLTAN_ID_TYPE));
}
ZOLTAN_SET_ID (dd->gid_length, (*gid)+k*dd->gid_length, ptr->gid);
k++;
}
(*size) = k;
return (ierr);
}
int Zoltan_DD_Remove (
Zoltan_DD_Directory *dd, /* directory state infomation */
ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to remove */
int count) /* Number of GIDs in removal list */
{
int *procs = NULL; /* list of processors to contact */
DD_Remove_Msg *ptr = NULL;
ZOLTAN_COMM_OBJ *plan = NULL; /* efficient MPI communication */
char *sbuff = NULL; /* send buffer */
char *sbufftmp = NULL;/* pointer into send buffer */
char *rbuff = NULL; /* receive buffer */
char *rbufftmp = NULL;/* pointer into receive buffer */
int nrec; /* number of receives to expect */
int i;
int err; /* error condition to return */
int errcount; /* count of GIDs not found */
char str[100]; /* string to build error messages */
char *yo = "Zoltan_DD_Remove";
/* input sanity checks */
if (dd == NULL || count < 0 || (gid == NULL && count > 0)) {
ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo,
"Invalid input argument");
return ZOLTAN_FATAL;
}
if (dd->debug_level > 4)
ZOLTAN_TRACE_IN (dd->my_proc, yo, NULL);
/* allocate memory for processor contact list */
if (count) {
procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count);
if (procs == NULL) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list");
if (dd->debug_level > 4)
ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL);
return ZOLTAN_MEMERR;
}
}
/* allocate memory for DD_Remove_Msg send buffer */
if (count) {
sbuff = (char*)ZOLTAN_MALLOC((size_t)(dd->remove_msg_size)*(size_t)count);
if (sbuff == NULL) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer");
err = ZOLTAN_MEMERR;
goto fini;
}
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After proc & sbuff mallocs");
/* for each GID, fill in contact list and then message structure */
sbufftmp = sbuff;
for (i = 0; i < count; i++) {
procs[i] = dd->hash(gid + i*dd->gid_length, dd->gid_length, dd->nproc,
dd->hashdata, dd->hashfn);
ptr = (DD_Remove_Msg*) sbufftmp;
sbufftmp += dd->remove_msg_size;
ptr->owner = dd->my_proc;
ZOLTAN_SET_ID (dd->gid_length, ptr->gid, gid + i * dd->gid_length);
}
/* now create efficient communication plan */
err = Zoltan_Comm_Create (&plan, count, procs, dd->comm,
ZOLTAN_DD_REMOVE_MSG_TAG, &nrec);
if (err != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Create error");
goto fini;
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After Zoltan_Comm_Create");
/* allocate receive buffer for nrec DD_Remove_Msg structures */
if (nrec) {
rbuff = (char*)ZOLTAN_MALLOC((size_t)nrec*(size_t)(dd->remove_msg_size));
if (rbuff == NULL) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Receive buffer malloc failed");
err = ZOLTAN_MEMERR;
goto fini;
}
}
/* send my remove messages & receive removes directed to me */
err = Zoltan_Comm_Do (plan, ZOLTAN_DD_UPDATE_MSG_TAG+1, sbuff,
dd->remove_msg_size, rbuff);
if (err != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Do error");
goto fini;
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After Zoltan_Comm_Do");
/* for each message rec'd, remove local directory info */
errcount = 0;
rbufftmp = rbuff;
for (i = 0; i < nrec; i++) {
ptr = (DD_Remove_Msg*) rbufftmp;
rbufftmp += dd->remove_msg_size;
err = DD_Remove_Local (dd, ptr->gid);
if (err == ZOLTAN_WARN)
++errcount;
}
err = ZOLTAN_OK;
if (dd->debug_level) {
sprintf (str, "Processed %d GIDs (%d local), %d GIDs not found",
count, nrec, errcount);
ZOLTAN_PRINT_INFO (dd->my_proc, yo, str);
err = (errcount) ? ZOLTAN_WARN : ZOLTAN_OK;
}
/* done, now free up things and return */
fini:
ZOLTAN_FREE (&procs);
ZOLTAN_FREE (&sbuff);
ZOLTAN_FREE (&rbuff);
Zoltan_Comm_Destroy (&plan);
if (dd->debug_level > 4)
ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL);
return err;
}
int Zoltan_DD_Update (
Zoltan_DD_Directory *dd, /* directory state information */
ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to update */
ZOLTAN_ID_PTR lid, /* Incoming corresponding LIDs (optional) */
char *user, /* Incoming list of user data (optional) */
int *partition, /* Optional, grouping of GIDs to partitions */
int count) /* Number of GIDs in update list */
{
int *procs = NULL; /* list of processors to contact */
DD_Update_Msg *ptr = NULL;
ZOLTAN_COMM_OBJ *plan = NULL; /* for efficient MPI communication */
char *sbuff = NULL; /* send buffer */
char *sbufftmp = NULL;/* pointer into send buffer */
char *rbuff = NULL; /* receive buffer */
char *rbufftmp = NULL;/* pointer into receive buffer */
int nrec = 0; /* number of receives to expect */
int i;
int err;
int errcount = 0; /* count of GIDs not found, added */
char str[100]; /* build error message string */
char *yo = "Zoltan_DD_Update";
/* input sanity checking */
if (dd == NULL || count < 0 || (gid == NULL && count > 0)) {
ZOLTAN_PRINT_ERROR ((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc), yo,
"Invalid input argument");
return ZOLTAN_FATAL;
}
if (dd->debug_level > 4)
ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL);
/* part of initializing the error checking process */
/* for each linked list head, walk its list resetting errcheck */
if (dd->debug_level)
for (i = 0; i < dd->table_length; i++) {
DD_NodeIdx nodeidx;
for (nodeidx = dd->table[i]; nodeidx != -1;
nodeidx = dd->nodelist[nodeidx].next)
dd->nodelist[nodeidx].errcheck = ZOLTAN_DD_NO_PROC;
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After reset errcheck");
/* allocate memory for list of processors to contact */
if (count) {
procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count);
if (procs == NULL) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list");
err = ZOLTAN_MEMERR;
goto fini;
}
}
/* allocate memory for DD_Update_Msg send buffer */
if (count) {
sbuff = (char*) ZOLTAN_CALLOC (count, dd->update_msg_size);
if (sbuff == NULL) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer");
err = ZOLTAN_MEMERR;
goto fini;
}
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs");
/* for each GID given, fill in contact list and then message structure */
sbufftmp = sbuff;
for (i = 0; i < count; i++) {
procs[i] = dd->hash(gid + i*dd->gid_length, dd->gid_length, dd->nproc,
dd->hashdata, dd->hashfn);
ptr = (DD_Update_Msg*) sbufftmp;
sbufftmp += dd->update_msg_size;
ptr->lid_flag = (lid) ? 1 : 0;
ptr->user_flag = (user) ? 1 : 0;
ptr->partition_flag = (partition) ? 1 : 0;
ptr->partition = (partition) ? *(partition + i) : -1;
ptr->owner = dd->my_proc;
ZOLTAN_SET_ID (dd->gid_length, ptr->gid, gid + i * dd->gid_length);
if (lid) {
ZOLTAN_SET_ID(dd->lid_length, ptr->gid + dd->gid_length,
lid + i * dd->lid_length);
}
else {
memset(ptr->gid + dd->gid_length, 0, dd->lid_length);
}
if (user) {
memcpy(ptr->gid + (dd->gid_length + dd->lid_length),
user + (size_t)i * (size_t)(dd->user_data_length),
dd->user_data_length);
}
else {
memset(ptr->gid + (dd->gid_length + dd->lid_length), 0,
dd->user_data_length);
}
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill contact list");
/* now create efficient communication plan */
err = Zoltan_Comm_Create (&plan, count, procs, dd->comm,
ZOLTAN_DD_UPDATE_MSG_TAG, &nrec);
if (err != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Create error");
goto fini;
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create");
/* If dd has no nodes allocated (e.g., first call to DD_Update;
* create the nodelist and freelist
*/
if (nrec && dd->nodelistlen == 0) {
DD_Memory_Alloc_Nodelist(dd, (DD_NodeIdx) nrec, 0.);
/* TODO Add overalloc parameter */
}
/* allocate receive buffer for nrec DD_Update_Msg structures */
if (nrec) {
rbuff = (char*)ZOLTAN_MALLOC((size_t)nrec*(size_t)(dd->update_msg_size));
if (rbuff == NULL) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Receive buffer malloc failed");
err = ZOLTAN_MEMERR;
goto fini;
}
}
/* send my update messages & receive updates directed to me */
err = Zoltan_Comm_Do (plan, ZOLTAN_DD_UPDATE_MSG_TAG+1, sbuff,
dd->update_msg_size, rbuff);
if (err != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Do error");
goto fini;
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do");
/* for each message rec'd, update local directory information */
errcount = 0;
rbufftmp = rbuff;
for (i = 0; i < nrec; i++) {
ptr = (DD_Update_Msg *) rbufftmp;
rbufftmp += dd->update_msg_size;
err = DD_Update_Local (dd, ptr->gid,
(ptr->lid_flag) ? (ptr->gid + dd->gid_length) : NULL,
(char*)((ptr->user_flag)?(ptr->gid+(dd->gid_length+dd->lid_length)):NULL),
(ptr->partition_flag) ? (ptr->partition) : -1, /* illegal partition */
ptr->owner);
if (err != ZOLTAN_OK)
++errcount;
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Local update");
err = ZOLTAN_OK;
if (dd->debug_level) /* overwrite error return if extra checking is on */
err = (errcount) ? ZOLTAN_WARN : ZOLTAN_OK;
fini:
ZOLTAN_FREE (&procs);
ZOLTAN_FREE (&sbuff);
ZOLTAN_FREE (&rbuff);
Zoltan_Comm_Destroy (&plan);
if (dd->debug_level) {
sprintf (str, "Processed %d GIDs (%d local), %d GID errors", count,
nrec, errcount);
ZOLTAN_PRINT_INFO (dd->my_proc, yo, str);
}
if (dd->debug_level > 4)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return err;
}
static int DD_Update_Local (Zoltan_DD_Directory *dd,
ZOLTAN_ID_PTR gid, /* GID to update (in) */
ZOLTAN_ID_PTR lid, /* gid's LID (in), NULL if not needed */
char *user, /* gid's user data (in), NULL if not needed */
int partition, /* gid's partition (in), -1 if not used */
int owner) /* gid's current owner (proc number) (in) */
{
int index;
char *yo = "DD_Update_Local";
DD_NodeIdx nodeidx;
DD_Node *ptr;
/* input sanity checking */
if (dd == NULL || owner < 0 || owner >= dd->nproc || gid == NULL) {
ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo,
"Invalid input parameter");
return ZOLTAN_FATAL;
}
if (dd->debug_level > 5)
ZOLTAN_TRACE_IN (dd->my_proc, yo, NULL);
/* compute offset into hash table to find head of linked list */
index = Zoltan_DD_Hash2 (gid, dd->gid_length, dd->table_length,
dd->hashdata, NULL);
/* walk linked list until end looking for matching gid */
for (nodeidx = dd->table[index]; nodeidx != -1;
nodeidx = dd->nodelist[nodeidx].next) {
ptr = dd->nodelist + nodeidx;
if (ZOLTAN_EQ_ID (dd->gid_length, gid, ptr->gid) == TRUE) {
/* found match, update directory information */
if (lid)
ZOLTAN_SET_ID (dd->lid_length,ptr->gid + dd->gid_length, lid);
if (user)
memcpy(ptr->gid + (dd->gid_length + dd->lid_length), user,
dd->user_data_length);
ptr->owner = owner;
if (partition != -1)
ptr->partition = partition;
/* Response to multiple updates to a gid in 1 update cycle */
if (dd->debug_level > 0 && ptr->errcheck != owner) {
ZOLTAN_PRINT_INFO (dd->my_proc, yo, "Multiply defined GID");
if (dd->debug_level > 4)
ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL);
return ZOLTAN_WARN;
}
ptr->errcheck = owner;
if (dd->debug_level > 5)
ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL);
return ZOLTAN_OK; /* ignore all errors */
}
}
/* gid not found. Create new DD_Node and fill it in */
nodeidx = DD_Memory_Alloc_Node(dd);
ptr = dd->nodelist + nodeidx;
ZOLTAN_SET_ID (dd->gid_length, ptr->gid, gid);
if (lid) {
ZOLTAN_SET_ID(dd->lid_length,ptr->gid + dd->gid_length, lid);
}
else {
memset(ptr->gid + dd->gid_length, 0,
dd->lid_length*sizeof(ZOLTAN_ID_TYPE));
}
if (user) {
memcpy(ptr->gid + (dd->gid_length + dd->lid_length), user,
dd->user_data_length);
}
else {
memset(ptr->gid + (dd->gid_length+dd->lid_length), 0,
dd->user_data_length);
}
ptr->partition = partition;
ptr->owner = owner;
ptr->errcheck = owner;
/* Add node to the linked list */
ptr->next = dd->table[index];
dd->table[index] = nodeidx;
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO (dd->my_proc, yo, "Created new directory item");
if (dd->debug_level > 5)
ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL);
return ZOLTAN_OK;
}
int main (int argc, char *argv[])
{
Zoltan_DD_Directory *dd;
ZTIMER *zt;
int myproc; /* MPI rank, my processor's MPI ID */
int nproc; /* MPI size, number of processors */
ZOLTAN_ID_PTR glist = NULL; /* list of GIDs */
ZOLTAN_ID_PTR llist = NULL; /* list of LIDs */
ZOLTAN_ID_PTR ulist = NULL; /* list of user data of type
ZOLTAN_ID_TYPE */
int *plist = NULL; /* list of partitions */
int *olist = NULL; /* list of owners */
Param param ; /* program's adjustable parameters */
char *store = NULL; /* non directory storage of test data */
Data *data = NULL; /* pointer into store */
/* these are all temporary variables */
int new_owner;
int count;
int i;
char *p;
int err;
int errcount;
int loop;
char str[100]; /* for building output messages */
static int timer[7] = {-1,-1,-1,-1,-1,-1,-1};
char *yo = "DD_Main";
/* initialize MPI communications, ignore errors */
MPI_Init (&argc, &argv);
MPI_Comm_rank (MPI_COMM_WORLD, &myproc);
MPI_Comm_size (MPI_COMM_WORLD, &nproc);
get_params (¶m); /* read input parameters */
zt = Zoltan_Timer_Create(1);
MACRO_TIMER_START(5, "Total time", 0);
MACRO_TIMER_START(0, "DD_Create time", 0);
err = Zoltan_DD_Create (&dd, MPI_COMM_WORLD, param.glen, param.llen,
param.ulen, param.tlen, param.debug_level);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Create");
MACRO_TIMER_STOP(0);
param.slen = sizeof (Data) + sizeof(ZOLTAN_ID_TYPE)
* (param.glen + param.llen + param.ulen);
param.slen = Zoltan_Align(param.slen);
store = (char*) ZOLTAN_MALLOC (param.count * param.slen);
/* allocate storage for various lists */
glist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC(sizeof(ZOLTAN_ID_TYPE) * param.count
* param.glen);
plist = (int*) ZOLTAN_MALLOC (sizeof(int) * param.count);
olist = (int*) ZOLTAN_MALLOC (sizeof(int) * param.count);
if (param.llen)
llist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC (sizeof (ZOLTAN_ID_TYPE)
* param.count * param.llen);
if (param.ulen)
ulist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC (sizeof (ZOLTAN_ID_TYPE)
* param.count * param.ulen) ;
if (store == NULL || glist == NULL || (param.llen != 0 && llist == NULL)
|| (param.ulen != 0 && ulist == NULL) || plist == NULL || olist == NULL) {
ZOLTAN_PRINT_ERROR (myproc, yo, "Unable to malloc storage lists");
return ZOLTAN_MEMERR;
}
initialize_data (¶m, store, nproc);
/* create & update directory with myproc's initial simulated GIDs */
count = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen)
if (((Data *)p)->new_owner == myproc) {
ZOLTAN_SET_ID (param.glen, glist + count * param.glen, ((Data*)p)->id);
if (param.llen)
ZOLTAN_SET_ID (param.llen, llist + count * param.llen,
((Data *)p)->id + param.glen);
if (param.ulen)
ZOLTAN_SET_ID (param.ulen, ulist + count * param.ulen,
((Data *)p)->id + (param.glen + param.llen));
plist [count] = ((Data *)p)->partition;
++count;
}
MACRO_TIMER_START (1, "DD_Update timer", 0);
err = Zoltan_DD_Update (dd, glist, llist, ulist, plist, count);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update");
MACRO_TIMER_STOP(1);
i = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen) {
ZOLTAN_SET_ID (param.glen, glist + i * param.glen, ((Data *)p)->id);
++i;
}
MACRO_TIMER_START(2, "DD_Find timer", 0);
err = Zoltan_DD_Find (dd, glist, llist, ulist, plist, param.count, olist);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find");
MACRO_TIMER_STOP(2);
errcount = 0;
for (i = 0; i < param.count; i++)
if (olist[i] != ((Data *) (store + i * param.slen))->new_owner)
++errcount;
if (errcount)
sprintf (str, "FIRST TEST FAILED, errcount is %d", errcount);
else
sprintf (str, "FIRST TEST SUCCESSFUL");
ZOLTAN_PRINT_INFO (myproc, yo, str);
/* repeatedly simulate moving "dots" around the system */
for (loop = 0; loop < param.nloops; loop++)
{
for (p = store; p < store + param.count * param.slen; p += param.slen)
((Data*) p)->old_owner = ((Data*) p)->new_owner;
/* randomly exchange some dots and randomly reassign others */
for (i = 0; i < (param.pmove * param.count)/100; i++) {
Data *d1, *d2;
d1 = (Data*) (store + param.slen * (rand() % param.count));
d2 = (Data*) (store + param.slen * (rand() % param.count));
new_owner = d1->new_owner;
d1->new_owner = d2->new_owner;
d2->new_owner = new_owner;
}
for (i = 0; i < (param.count * param.pscatter)/100; i++)
((Data*) (store + param.slen *(rand() % param.count)))->new_owner
= rand() % nproc;
/* determine which new GIDs myproc gained, and update directory */
count = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen)
if (((Data*)p)->new_owner == myproc) {
((Data*)p)->id[param.glen] = count; /* set LID */
ZOLTAN_SET_ID (param.glen, glist + count * param.glen,
((Data *)p)->id);
if (param.llen)
ZOLTAN_SET_ID (param.llen, llist + count * param.llen,
((Data*)p)->id + param.glen);
if (param.ulen)
ZOLTAN_SET_ID (param.ulen, ulist + count * param.ulen,
((Data*)p)->id + (param.glen + param.llen));
plist [count] = ((Data *)p)->partition;
++count;
}
MACRO_TIMER_START(1, "DD_Update", 0);
err = Zoltan_DD_Update (dd, glist, llist, ulist, plist, count);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update");
MACRO_TIMER_STOP(1);
/* use directory to "find" GIDs */
i = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen) {
ZOLTAN_SET_ID (param.glen, glist + i * param.glen, ((Data *)p)->id);
++i;
}
MACRO_TIMER_START(2, "DD_Find timer", 0);
if (loop % 5 == 0)
err = Zoltan_DD_Find(dd,glist,NULL,ulist,plist,param.count,olist);
else if (loop % 7 == 0)
err = Zoltan_DD_Find(dd,glist,llist,NULL,plist,param.count,olist);
else if (loop % 9 == 0)
err = Zoltan_DD_Find(dd,glist,llist,ulist,NULL,param.count,olist);
else if (loop % 2 == 0)
err = Zoltan_DD_Find(dd,glist,llist,ulist,plist,param.count,NULL);
else
err = Zoltan_DD_Find(dd,glist,llist,ulist,plist,param.count,olist);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find");
MACRO_TIMER_STOP(2);
if (loop % 2) {
errcount = 0 ;
for (i = 0; i < param.count; i++)
if (olist[i] != ((Data *)(store + i * param.slen))->new_owner)
++errcount;
if (errcount)
sprintf (str,"LOOP %d TEST FAILED, errcount is %d",loop,errcount);
else
sprintf (str, "LOOP %d TEST SUCCESSFUL", loop);
ZOLTAN_PRINT_INFO (myproc, yo, str) ;
}
else {
sprintf (str, "LOOP %d Completed", loop);
ZOLTAN_PRINT_INFO (myproc, yo, str);
}
/* randomly remove a percentage of GIDs from the directory */
count = 0;
for (i = 0; i < (param.count * param.pdelete)/100; i++) {
data = (Data*) (store + param.slen * (rand() % param.count));
if (data->new_owner == myproc) {
ZOLTAN_SET_ID (param.glen, glist + count * param.glen, data->id);
++count;
}
data->new_owner = NO_PROC;
}
MACRO_TIMER_START(3, "DD_Remove timer", 0);
err = Zoltan_DD_Remove (dd, glist, count);
MACRO_TIMER_STOP(3);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Remove");
/* update directory (put directory entries back in) */
for (p = store; p < store + param.count * param.slen; p += param.slen)
if (((Data*)p)->new_owner == NO_PROC)
((Data*)p)->new_owner = loop % nproc; /* place in new location */
count = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen)
if (((Data*)p)->new_owner == myproc) {
ZOLTAN_SET_ID(param.glen,glist+count*param.glen,((Data*)p)->id);
if (param.llen)
ZOLTAN_SET_ID (param.llen, llist + count * param.llen,
((Data*)p)->id + param.glen);
if (param.ulen)
ZOLTAN_SET_ID(param.ulen, ulist + count * param.ulen,
((Data *)p)->id + (param.glen + param.llen));
plist [count] = ((Data*)p)->partition;
++count;
}
MACRO_TIMER_START(1, "DD_Update timer", 0);
err = Zoltan_DD_Update (dd, glist, NULL, NULL, NULL, count);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update");
MACRO_TIMER_STOP(1);
}
/* now Find directory info for GIDs myproc now owns and validate */
count = 0;
for (i = 0; i < param.count; i++) {
data = (Data *) (store + i * param.slen);
if (data->new_owner == myproc) {
ZOLTAN_SET_ID (param.glen, glist + count * param.glen, data->id);
++count;
}
}
MACRO_TIMER_START(2, "DD_Find", 0);
err = Zoltan_DD_Find (dd, glist, NULL, NULL, NULL, count, olist);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find");
MACRO_TIMER_STOP(2);
errcount = 0;
for (i = 0; i < count; i++)
if (olist[i] != myproc)
++errcount;
if (errcount) {
sprintf (str, "errcount is %d", errcount);
ZOLTAN_PRINT_ERROR (myproc, yo, str);
}
else
ZOLTAN_PRINT_INFO (myproc, yo, "TEST SUCCESSFUL");
Zoltan_DD_Stats (dd);
/* done, now free memory, stop MPI & directory, return */
ZOLTAN_FREE (&store);
ZOLTAN_FREE (&glist);
ZOLTAN_FREE (&plist);
ZOLTAN_FREE (&olist);
if (param.llen) ZOLTAN_FREE (&llist);
if (param.ulen) ZOLTAN_FREE (&ulist);
ZOLTAN_PRINT_INFO (myproc, yo, "Completing program");
MACRO_TIMER_START(4, "DD_Destroy", 0);
Zoltan_DD_Destroy (&dd);
MACRO_TIMER_STOP(4);
MACRO_TIMER_STOP(5);
Zoltan_Timer_PrintAll(zt, 0, MPI_COMM_WORLD, stderr);
MPI_Finalize ();
return ZOLTAN_OK;
}
int Zoltan_DD_Find (
Zoltan_DD_Directory *dd, /* contains directory state information */
ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to get owners proc */
ZOLTAN_ID_PTR lid, /* Outgoing corresponding list of LIDs */
char *data, /* Outgoing optional corresponding user data */
int *partition, /* Outgoing optional partition information */
int count, /* Count of GIDs in above list (in) */
int *owner) /* Outgoing optional list of data owners */
{
ZOLTAN_COMM_OBJ *plan = NULL; /* efficient MPI communication */
char *rbuff = NULL; /* receive buffer */
char *rbufftmp = NULL; /* pointer into receive buffer */
char *sbuff = NULL; /* send buffer */
char *sbufftmp = NULL; /* pointer into send buffer */
int *procs = NULL; /* list of processors to contact */
DD_Find_Msg *ptr = NULL;
int i;
int nrec; /* number of messages to receive */
int err = ZOLTAN_OK; /* return error condition */
int errcount; /* count of GIDs not found */
char *yo = "Zoltan_DD_Find";
/* input sanity check */
if (dd == NULL || count < 0 || (gid == NULL && count > 0)) {
ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo,
"Invalid input argument");
return ZOLTAN_FATAL;
}
if (dd->debug_level > 4)
ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL);
/* allocate memory for processors to contact for directory info */
if (count) {
procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count);
if (procs == NULL) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list");
if (dd->debug_level > 4)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return ZOLTAN_MEMERR;
}
}
/* allocate memory for DD_Find_Msg send buffer */
if (count) {
sbuff = (char*) ZOLTAN_CALLOC (count, dd->find_msg_size);
if (sbuff == NULL) {
ZOLTAN_FREE (&procs);
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer");
if (dd->debug_level > 4)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return ZOLTAN_MEMERR;
}
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs");
/* for each GID, fill DD_Find_Msg buffer and contact list */
sbufftmp = sbuff;
for (i = 0; i < count; i++) {
procs[i] = dd->hash (gid + i*dd->gid_length, dd->gid_length, dd->nproc,
dd->hashdata, dd->hashfn);
ptr = (DD_Find_Msg*) sbufftmp;
sbufftmp += dd->find_msg_size;
ptr->index = i;
ptr->proc = procs[i];
ZOLTAN_SET_ID (dd->gid_length, ptr->id, gid + i*dd->gid_length);
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill");
/* create efficient communication plan */
err = Zoltan_Comm_Create (&plan, count, procs, dd->comm,
ZOLTAN_DD_FIND_MSG_TAG, &nrec);
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create");
if (err != ZOLTAN_OK)
goto fini;
/* allocate receive buffer */
if (nrec) {
rbuff = (char*) ZOLTAN_MALLOC ((size_t)nrec*(size_t)(dd->find_msg_size));
if (rbuff == NULL) {
err = ZOLTAN_MEMERR;
goto fini;
}
}
/* send out find messages across entire system */
err = Zoltan_Comm_Do (plan, ZOLTAN_DD_FIND_MSG_TAG+1, sbuff,
dd->find_msg_size, rbuff);
if (err != ZOLTAN_OK)
goto fini;
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do");
/* get find messages directed to me, fill in return information */
errcount = 0;
rbufftmp = rbuff;
for (i = 0; i < nrec; i++) {
ptr = (DD_Find_Msg*) rbufftmp;
rbufftmp += dd->find_msg_size;
err = DD_Find_Local (dd, ptr->id, ptr->id,
(char *)(ptr->id + dd->max_id_length),
&ptr->partition, &ptr->proc);
if (err == ZOLTAN_WARN)
++errcount;
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill in return info");
/* send return information back to requester */
err = Zoltan_Comm_Do_Reverse(plan, ZOLTAN_DD_FIND_MSG_TAG+2, rbuff,
dd->find_msg_size, NULL, sbuff);
if (err != ZOLTAN_OK)
goto fini;
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Reverse");
/* fill in user supplied lists with returned information */
sbufftmp = sbuff;
for (i = 0; i < count; i++) {
ptr = (DD_Find_Msg*) sbufftmp;
sbufftmp += dd->find_msg_size;
if (owner)
owner[ptr->index] = ptr->proc;
if (partition)
partition[ptr->index] = ptr->partition ;
if (lid)
ZOLTAN_SET_ID(dd->lid_length,lid+ptr->index*dd->lid_length,ptr->id);
if (data)
memcpy(data + (size_t)(ptr->index) * (size_t)(dd->user_data_length),
ptr->id + dd->max_id_length, dd->user_data_length);
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill return lists");
/* err = ZOLTAN_OK; */
MPI_Allreduce(&errcount, &err, 1, MPI_INT, MPI_SUM, dd->comm);
err = (err) ? ZOLTAN_WARN : ZOLTAN_OK;
/* if at least one GID was not found, potentially notify caller of error */
if (dd->debug_level > 0) {
char str[100]; /* diagnostic message string */
sprintf (str, "Processed %d GIDs, GIDs not found: %d", count, errcount);
ZOLTAN_PRINT_INFO (dd->my_proc, yo, str);
}
fini:
ZOLTAN_FREE (&sbuff);
ZOLTAN_FREE (&rbuff);
ZOLTAN_FREE (&procs) ;
Zoltan_Comm_Destroy (&plan);
if (dd->debug_level > 4)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return err;
}
int Zoltan_DD_Find (
Zoltan_DD_Directory *dd, /* contains directory state information */
ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to get owners proc */
ZOLTAN_ID_PTR lid, /* Outgoing corresponding list of LIDs */
ZOLTAN_ID_PTR data, /* Outgoing optional corresponding user data */
int *partition, /* Outgoing optional partition information */
int count, /* Count of GIDs in above list (in) */
int *owner) /* Outgoing corresponding list of data locations */
{
ZOLTAN_COMM_OBJ *plan = NULL ; /* efficient MPI communication */
char *rbuff = NULL ; /* receive buffer */
char *sbuff = NULL ; /* send buffer */
int *procs = NULL ; /* list of processors to contact */
DD_Find_Msg *ptr = NULL ;
int i ;
int nrec ; /* number of messages to receive */
int err ; /* return error condition */
int errcount ; /* count of GIDs not found */
char *yo = "Zoltan_DD_Find" ;
if (dd != NULL && dd->debug_level > 1)
ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL);
/* input sanity check */
if (dd == NULL || count < 0 || ((owner == NULL || gid == NULL) && count > 0))
{
ZOLTAN_PRINT_ERROR ((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc),
yo, "Invalid input argument.") ;
if (dd != NULL && dd->debug_level > 1)
ZOLTAN_TRACE_OUT((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc),
yo, NULL);
return ZOLTAN_DD_INPUT_ERROR ;
}
/* allocate memory for processors to contact for directory info */
if (count > 0)
{
procs = (int *) ZOLTAN_MALLOC (sizeof (int) * count) ;
if (procs == NULL)
{
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list.") ;
if (dd->debug_level > 1)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return ZOLTAN_DD_MEMORY_ERROR ;
}
}
/* allocate memory for DD_Find_Msg send buffer */
if (count > 0)
{
sbuff = (char *) ZOLTAN_MALLOC (dd->find_msg_size * count) ;
if (sbuff == NULL)
{
ZOLTAN_FREE (&procs) ;
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer.") ;
if (dd->debug_level > 1)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return ZOLTAN_DD_MEMORY_ERROR ;
}
}
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs.");
/* for each GID, fill DD_Find_Msg buffer and contact list */
for (i = 0 ; i < count ; i++)
{
procs[i] = dd->hash (gid + i*dd->gid_length, dd->gid_length, dd->nproc) ;
ptr = (DD_Find_Msg *) (sbuff + i * dd->find_msg_size) ;
ptr->index = i ;
ptr->proc = procs[i] ;
ZOLTAN_SET_ID (dd->gid_length, ptr->id, gid + i*dd->gid_length) ;
}
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill.");
/* create efficient communication plan */
err = Zoltan_Comm_Create (&plan, count, procs, dd->comm,
ZOLTAN_DD_FIND_MSG_TAG, &nrec) ;
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create.");
if (err != ZOLTAN_OK)
goto fini ;
/* allocate receive buffer */
if (nrec > 0)
{
rbuff = (char *) ZOLTAN_MALLOC (nrec * dd->find_msg_size) ;
if (rbuff == NULL)
{
err = ZOLTAN_DD_MEMORY_ERROR ;
goto fini ;
}
}
/* send out find messages across entire system */
err = Zoltan_Comm_Do (plan, ZOLTAN_DD_FIND_MSG_TAG+1, sbuff,
dd->find_msg_size, rbuff) ;
if (err != ZOLTAN_OK)
goto fini ;
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do.");
/* get find messages directed to me, fill in return information */
errcount = 0 ;
for (i = 0 ; i < nrec ; i++)
{
ptr = (DD_Find_Msg *) (rbuff + i*dd->find_msg_size) ;
err = DD_Find_Local (dd, ptr->id, ptr->id, ptr->id + dd->max_id_length,
(partition) ? (&ptr->partition) : NULL, &ptr->proc) ;
if (err == ZOLTAN_DD_GID_NOT_FOUND_ERROR)
errcount++ ;
}
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill in return info.");
/* send return information back to requester */
err = Zoltan_Comm_Do_Reverse(plan, ZOLTAN_DD_FIND_MSG_TAG+2, rbuff,
dd->find_msg_size, NULL, sbuff) ;
if (err != ZOLTAN_OK)
goto fini ;
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Reverse.");
/* fill in user supplied lists with returned information */
for (i = 0 ; i < count ; i++)
{
ptr = (DD_Find_Msg *) (sbuff + i*dd->find_msg_size) ;
owner[ptr->index] = ptr->proc ;
if (partition != NULL)
partition[ptr->index] = ptr->partition ;
if (lid != NULL)
ZOLTAN_SET_ID (dd->lid_length, lid + ptr->index * dd->lid_length,
ptr->id) ;
if (data != NULL)
ZOLTAN_SET_ID (dd->user_data_length, data + ptr->index
* dd->user_data_length, ptr->id + dd->max_id_length) ;
}
/* if at least one GID was not found, notify caller of error */
err = (errcount == 0) ? ZOLTAN_DD_NORMAL_RETURN
: ZOLTAN_DD_GID_NOT_FOUND_ERROR ;
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill return lists.");
fini:
ZOLTAN_FREE (&sbuff) ;
ZOLTAN_FREE (&rbuff) ;
ZOLTAN_FREE (&procs) ;
Zoltan_Comm_Destroy (&plan) ;
if (err != ZOLTAN_DD_NORMAL_RETURN)
ZOLTAN_PRINT_WARN (dd->my_proc, yo, "Return is not normal.") ;
if (dd->debug_level > 0)
{
char str[100] ; /* diagnostic message string */
sprintf (str, "Processed %d GIDs, GIDs not found: %d", count, errcount) ;
ZOLTAN_PRINT_INFO (dd->my_proc, yo, str) ;
}
if (dd->debug_level > 1)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return err ;
}
static int DD_Update_Local (Zoltan_DD_Directory *dd,
ZOLTAN_ID_PTR gid, /* GID to update (in) */
ZOLTAN_ID_PTR lid, /* gid's LID (in), NULL if not needed */
ZOLTAN_ID_PTR user, /* gid's user data (in), NULL if not needed */
int partition, /* gid's partition (in), -1 if not used */
int owner) /* gid's current owner (proc number) (in) */
{
DD_Node **ptr;
int index;
char *yo = "DD_Update_Local";
/* input sanity checking */
if (dd == NULL || owner < 0 || owner >= dd->nproc || gid == NULL) {
ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo,
"Invalid input parameter");
return ZOLTAN_FATAL;
}
if (dd->debug_level > 5)
ZOLTAN_TRACE_IN (dd->my_proc, yo, NULL);
/* compute offset into hash table to find head of linked list */
index = Zoltan_DD_Hash2 (gid, dd->gid_length, dd->table_length, dd->hashdata);
/* walk linked list until end looking for matching gid */
for (ptr = dd->table+index; *ptr != NULL; ptr = &((*ptr)->next))
if (ZOLTAN_EQ_ID (dd->gid_length, gid, (*ptr)->gid) == TRUE) {
/* found match, update directory information */
if (lid)
ZOLTAN_SET_ID (dd->lid_length,(*ptr)->gid + dd->gid_length, lid);
if (user)
ZOLTAN_SET_ID (dd->user_data_length, (*ptr)->gid + (dd->gid_length
+ dd->lid_length), user);
(*ptr)->owner = owner;
if (partition != -1)
(*ptr)->partition = partition;
/* Response to multiple updates to a gid in 1 update cycle */
if (dd->debug_level > 0 && (*ptr)->errcheck != owner) {
ZOLTAN_PRINT_INFO (dd->my_proc, yo, "Multiply defined GID");
if (dd->debug_level > 4)
ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL);
return ZOLTAN_WARN;
}
(*ptr)->errcheck = owner;
if (dd->debug_level > 5)
ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL);
return ZOLTAN_OK; /* ignore all errors */
}
/* gid not found. Create new DD_Node and fill it in */
*ptr = (DD_Node*) ZOLTAN_MALLOC (dd->node_size);
if (*ptr == NULL) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc new Node");
if (dd->debug_level > 5)
ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL);
return ZOLTAN_MEMERR;
}
ZOLTAN_SET_ID (dd->gid_length, (*ptr)->gid, gid);
if (lid) {
ZOLTAN_SET_ID(dd->lid_length,(*ptr)->gid + dd->gid_length, lid);
}
else {
memset((*ptr)->gid + dd->gid_length, 0,
dd->lid_length*sizeof(ZOLTAN_ID_TYPE));
}
if (user) {
ZOLTAN_SET_ID(dd->user_data_length,
(*ptr)->gid + (dd->gid_length + dd->lid_length), user);
}
else {
memset((*ptr)->gid + (dd->gid_length+dd->lid_length), 0,
dd->user_data_length*sizeof(ZOLTAN_ID_TYPE));
}
(*ptr)->partition = partition ;
(*ptr)->next = NULL;
(*ptr)->owner = owner;
(*ptr)->errcheck = owner;
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO (dd->my_proc, yo, "Created new directory item");
if (dd->debug_level > 5)
ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL);
return ZOLTAN_OK;
}
