void Zoltan_DD_Stats ( Zoltan_DD_Directory *dd) /* directory state information */ { int node_count = 0 ; /* counts Nodes in local directory */ int maxlength = 0; /* length of longest linked list */ int list_count = 0 ; /* number of linked lints in hash table */ int length ; int i ; DD_Node *ptr ; char str[100] ; /* used to build message string */ char *yo = "Zoltan_DD_Stats" ; /* Input sanity check */ if (dd == NULL) { ZOLTAN_PRINT_ERROR (0, yo, "Invalid input argument.") ; return ; } if (dd->debug_level > 4) ZOLTAN_TRACE_IN (dd->my_proc, yo, NULL) ; /* walk down each list in hash table to find every Node */ for (i = 0 ; i < dd->table_length ; i++) { length = 0 ; /* reset length for next count */ if (dd->table[i] != NULL) list_count++ ; /* count of distict linked lists */ for (ptr = dd->table[i] ; ptr != NULL ; ptr = ptr->next) { if (dd->debug_level > 6) { sprintf (str, "GID %4u, Owner %d, Table Index %d.", *ptr->gid, ptr->owner, i) ; ZOLTAN_PRINT_INFO (dd->my_proc, yo, str) ; } length++ ; /* linked list length */ node_count++ ; /* count of Nodes */ } if (length > maxlength) maxlength = length ; /* save length of longest linked list */ } sprintf (str, "Hash table size %d, %d nodes on %d lists, max list length %d.", dd->table_length, node_count, list_count, maxlength) ; ZOLTAN_PRINT_INFO (dd->my_proc, yo, str) ; if (dd->debug_level > 4) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL) ; }
static int DD_Find_Local (Zoltan_DD_Directory *dd, ZOLTAN_ID_PTR gid, /* incoming GID to locate (in) */ ZOLTAN_ID_PTR lid, /* gid's LID (out) */ char *user, /* gid's user data (out) */ int *partition, /* gid's partition number (out) */ int *owner) /* gid's owner (processor number) (out) */ { DD_Node *ptr; DD_NodeIdx nodeidx; int index; char *yo = "DD_Find_Local"; /* input sanity check */ if (dd == NULL || owner == NULL || gid == NULL) { ZOLTAN_PRINT_ERROR ((dd == NULL) ? 0 : dd->my_proc, yo, "Invalid input"); return ZOLTAN_FATAL; } if (dd->debug_level > 5) ZOLTAN_TRACE_IN (dd->my_proc, yo, NULL); /* compute offset into hash table to find head of linked list */ index = Zoltan_DD_Hash2 (gid, dd->gid_length, dd->table_length, dd->hashdata, NULL); /* walk link list until end looking for matching global ID */ for (nodeidx = dd->table[index]; nodeidx != -1; nodeidx = dd->nodelist[nodeidx].next) { ptr = dd->nodelist + nodeidx; if (ZOLTAN_EQ_ID (dd->gid_length, gid, ptr->gid) == TRUE) { /* matching global ID found! Return gid's information */ if (lid) ZOLTAN_SET_ID(dd->lid_length, lid, ptr->gid + dd->gid_length); if (user) memcpy(user, ptr->gid + (dd->gid_length + dd->lid_length), dd->user_data_length); if (owner) *owner = ptr->owner; if (partition) *partition = ptr->partition; if (dd->debug_level > 5) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_OK; } } if (owner != NULL) *owner = -1; /* JDT Added -1 owner not found */ if (dd->debug_level > 5) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); if (dd->debug_level > 0) { ZOLTAN_PRINT_INFO(dd->my_proc, yo, "GID not found"); return ZOLTAN_WARN; } return ZOLTAN_WARN; }
static int DD_Find_Local (Zoltan_DD_Directory *dd, ZOLTAN_ID_PTR gid, /* incoming GID to locate (in) */ ZOLTAN_ID_PTR lid, /* gid's LID (out) */ ZOLTAN_ID_PTR user, /* gid's user data (out) */ int *partition, /* gid's partition number (out) */ int *owner) /* gid's owner (processor number) (out) */ { DD_Node *ptr ; int index ; char *yo = "DD_Find_Local" ; /* input sanity check */ if (dd == NULL || owner == NULL || gid == NULL) { ZOLTAN_PRINT_ERROR ((dd == NULL) ? 0 : dd->my_proc, yo, "Invalid input argument.") ; return ZOLTAN_DD_INPUT_ERROR ; } if (dd->debug_level > 2) ZOLTAN_TRACE_IN (dd->my_proc, yo, NULL) ; /* compute offset into hash table to find head of linked list */ index = Zoltan_DD_Hash2 (gid, dd->gid_length, dd->table_length) ; /* walk link list until end looking for matching global ID */ for (ptr = dd->table[index] ; ptr != NULL ; ptr = ptr->next) if (ZOLTAN_EQ_ID (dd->gid_length, gid, ptr->gid) == TRUE) { /* matching global ID found! Return gid's information */ if (lid) ZOLTAN_SET_ID(dd->lid_length, lid, ptr->gid + dd->gid_length); if (user) ZOLTAN_SET_ID(dd->user_data_length, user, ptr->gid + (dd->gid_length + dd->lid_length)); if (owner != NULL) *owner = ptr->owner ; if (partition != NULL) *partition = ptr->partition ; if (dd->debug_level > 2) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL) ; return ZOLTAN_DD_NORMAL_RETURN ; } if (dd->debug_level > 0) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "GID not found."); if (dd->debug_level > 2) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL) ; return ZOLTAN_DD_GID_NOT_FOUND_ERROR ; }
int Zoltan_DD_Set_Hash_Fn ( Zoltan_DD_Directory *dd, /* directory state information */ ZOLTAN_HASH_FN *hash) { char *yo = "Zoltan_DD_Set_Hash_Fn"; /* input sanity checking */ if (dd == NULL || hash == NULL) { ZOLTAN_PRINT_ERROR (0, yo, "Invalid input argument"); return ZOLTAN_FATAL ; } dd->hash = (DD_Hash_fn*)dd_hash_user; dd->hashdata = NULL; dd->hashfn = hash; dd->cleanup = (DD_Cleanup_fn*) NULL; if (dd->debug_level > 0) ZOLTAN_PRINT_INFO (dd->my_proc, yo, "Successful"); return ZOLTAN_OK; }
int Zoltan_DD_Remove ( Zoltan_DD_Directory *dd, /* directory state infomation */ ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to remove */ int count) /* Number of GIDs in removal list */ { int *procs = NULL; /* list of processors to contact */ DD_Remove_Msg *ptr = NULL; ZOLTAN_COMM_OBJ *plan = NULL; /* efficient MPI communication */ char *sbuff = NULL; /* send buffer */ char *sbufftmp = NULL;/* pointer into send buffer */ char *rbuff = NULL; /* receive buffer */ char *rbufftmp = NULL;/* pointer into receive buffer */ int nrec; /* number of receives to expect */ int i; int err; /* error condition to return */ int errcount; /* count of GIDs not found */ char str[100]; /* string to build error messages */ char *yo = "Zoltan_DD_Remove"; /* input sanity checks */ if (dd == NULL || count < 0 || (gid == NULL && count > 0)) { ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo, "Invalid input argument"); return ZOLTAN_FATAL; } if (dd->debug_level > 4) ZOLTAN_TRACE_IN (dd->my_proc, yo, NULL); /* allocate memory for processor contact list */ if (count) { procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count); if (procs == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list"); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_MEMERR; } } /* allocate memory for DD_Remove_Msg send buffer */ if (count) { sbuff = (char*)ZOLTAN_MALLOC((size_t)(dd->remove_msg_size)*(size_t)count); if (sbuff == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer"); err = ZOLTAN_MEMERR; goto fini; } } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After proc & sbuff mallocs"); /* for each GID, fill in contact list and then message structure */ sbufftmp = sbuff; for (i = 0; i < count; i++) { procs[i] = dd->hash(gid + i*dd->gid_length, dd->gid_length, dd->nproc, dd->hashdata, dd->hashfn); ptr = (DD_Remove_Msg*) sbufftmp; sbufftmp += dd->remove_msg_size; ptr->owner = dd->my_proc; ZOLTAN_SET_ID (dd->gid_length, ptr->gid, gid + i * dd->gid_length); } /* now create efficient communication plan */ err = Zoltan_Comm_Create (&plan, count, procs, dd->comm, ZOLTAN_DD_REMOVE_MSG_TAG, &nrec); if (err != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Create error"); goto fini; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After Zoltan_Comm_Create"); /* allocate receive buffer for nrec DD_Remove_Msg structures */ if (nrec) { rbuff = (char*)ZOLTAN_MALLOC((size_t)nrec*(size_t)(dd->remove_msg_size)); if (rbuff == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Receive buffer malloc failed"); err = ZOLTAN_MEMERR; goto fini; } } /* send my remove messages & receive removes directed to me */ err = Zoltan_Comm_Do (plan, ZOLTAN_DD_UPDATE_MSG_TAG+1, sbuff, dd->remove_msg_size, rbuff); if (err != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Do error"); goto fini; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After Zoltan_Comm_Do"); /* for each message rec'd, remove local directory info */ errcount = 0; rbufftmp = rbuff; for (i = 0; i < nrec; i++) { ptr = (DD_Remove_Msg*) rbufftmp; rbufftmp += dd->remove_msg_size; err = DD_Remove_Local (dd, ptr->gid); if (err == ZOLTAN_WARN) ++errcount; } err = ZOLTAN_OK; if (dd->debug_level) { sprintf (str, "Processed %d GIDs (%d local), %d GIDs not found", count, nrec, errcount); ZOLTAN_PRINT_INFO (dd->my_proc, yo, str); err = (errcount) ? ZOLTAN_WARN : ZOLTAN_OK; } /* done, now free up things and return */ fini: ZOLTAN_FREE (&procs); ZOLTAN_FREE (&sbuff); ZOLTAN_FREE (&rbuff); Zoltan_Comm_Destroy (&plan); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return err; }
int Zoltan_Get_Coordinates( ZZ *zz, int num_obj, /* Input: number of objects */ ZOLTAN_ID_PTR global_ids, /* Input: global IDs of objects */ ZOLTAN_ID_PTR local_ids, /* Input: local IDs of objects; may be NULL. */ int *num_dim, /* Output: dimension of coordinates */ double **coords /* Output: array of coordinates; malloc'ed by fn if NULL upon input. */ ) { char *yo = "Zoltan_Get_Coordinates"; int i,j,rc; int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; int alloced_coords = 0; ZOLTAN_ID_PTR lid; /* Temporary pointers to local IDs; used to pass NULL to query functions when NUM_LID_ENTRIES == 0. */ double dist[3]; double im[3][3]; double deg_ratio; double x; int order[3]; int reduce_dimensions, d, axis_aligned; int target_dim; int ierr = ZOLTAN_OK; char msg[256]; ZZ_Transform *tran; ZOLTAN_TRACE_ENTER(zz, yo); /* Error check -- make sure needed callback functions are registered. */ if (zz->Get_Num_Geom == NULL || (zz->Get_Geom == NULL && zz->Get_Geom_Multi == NULL)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_GEOM_FN and " "either ZOLTAN_GEOM_MULTI_FN or ZOLTAN_GEOM_FN"); goto End; } /* Get problem dimension. */ *num_dim = zz->Get_Num_Geom(zz->Get_Num_Geom_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from ZOLTAN_GET_NUM_GEOM_FN"); goto End; } if (*num_dim < 0 || *num_dim > 3) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Invalid dimension returned from ZOLTAN_NUM_GEOM_FN"); goto End; } /* Get coordinates for object; allocate memory if not already provided. */ if (*num_dim > 0 && num_obj > 0) { if (*coords == NULL) { alloced_coords = 1; *coords = (double *) ZOLTAN_MALLOC(num_obj * (*num_dim) * sizeof(double)); if (*coords == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error"); goto End; } } if (zz->Get_Geom_Multi != NULL) { zz->Get_Geom_Multi(zz->Get_Geom_Multi_Data, zz->Num_GID, zz->Num_LID, num_obj, global_ids, local_ids, *num_dim, *coords, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from ZOLTAN_GET_GEOM_MULTI_FN"); goto End; } } else { for (i = 0; i < num_obj; i++) { lid = (num_lid_entries ? &(local_ids[i*num_lid_entries]) : NULL); zz->Get_Geom(zz->Get_Geom_Data, num_gid_entries, num_lid_entries, global_ids + i*num_gid_entries, lid, (*coords) + i*(*num_dim), &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from ZOLTAN_GET_GEOM_FN"); goto End; } } } } /* * For RCB, RIB, and HSFC: if REDUCE_DIMENSIONS was selected, compute the * center of mass and inertial matrix of the coordinates. * * For 3D problems: If the geometry is "flat", transform the points so the * two primary directions lie along the X and Y coordinate axes and project * to the Z=0 plane. If in addition the geometry is "skinny", project to * the X axis. (This creates a 2D or 1D problem respectively.) * * For 2D problems: If the geometry is essentially a line, transform it's * primary direction to the X axis and project to the X axis, yielding a * 1D problem. * * Return these points to the partitioning algorithm, in effect partitioning * in only the 2 (or 1) significant dimensions. */ if (((*num_dim == 3) || (*num_dim == 2)) && ((zz->LB.Method==RCB) || (zz->LB.Method==RIB) || (zz->LB.Method==HSFC))){ Zoltan_Bind_Param(Reduce_Dim_Params, "KEEP_CUTS", (void *)&i); Zoltan_Bind_Param(Reduce_Dim_Params, "REDUCE_DIMENSIONS", (void *)&reduce_dimensions); Zoltan_Bind_Param(Reduce_Dim_Params, "DEGENERATE_RATIO", (void *)°_ratio); i = 0; reduce_dimensions = 0; deg_ratio = 10.0; Zoltan_Assign_Param_Vals(zz->Params, Reduce_Dim_Params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); if (reduce_dimensions == 0){ goto End; } if (deg_ratio <= 1){ if (zz->Proc == 0){ ZOLTAN_PRINT_WARN(0, yo, "DEGENERATE_RATIO <= 1, setting it to 10.0"); } deg_ratio = 10.0; } if (zz->LB.Method == RCB){ tran = &(((RCB_STRUCT *)(zz->LB.Data_Structure))->Tran); } else if (zz->LB.Method == RIB){ tran = &(((RIB_STRUCT *)(zz->LB.Data_Structure))->Tran); } else{ tran = &(((HSFC_Data*)(zz->LB.Data_Structure))->tran); } d = *num_dim; if (tran->Target_Dim >= 0){ /* * On a previous load balancing call, we determined whether * or not the geometry was degenerate. If the geometry was * determined to be not degenerate, then we assume it is still * not degenerate, and we skip the degeneracy calculation. */ if (tran->Target_Dim > 0){ /* * The geometry *was* degenerate. We test the extent * of the geometry along the principal directions determined * last time to determine if it is still degenerate with that * orientation. If so, we transform the coordinates using the * same transformation we used last time. If not, we do the * entire degeneracy calculation again. */ if ((tran->Axis_Order[0] >= 0) && (tran->Axis_Order[1] >= 0) && (tran->Axis_Order[2] >= 0)){ axis_aligned = 1; } else{ axis_aligned = 0; } projected_distances(zz, *coords, num_obj, tran->CM, tran->Evecs, dist, d, axis_aligned, tran->Axis_Order); target_dim = get_target_dimension(dist, order, deg_ratio, d); if (target_dim > 0){ transform_coordinates(*coords, num_obj, d, tran); } else{ /* Set's Target_Dim to -1, flag to recompute degeneracy */ Zoltan_Initialize_Transformation(tran); } } } if (tran->Target_Dim < 0){ tran->Target_Dim = 0; /* * Get the center of mass and inertial matrix of coordinates. Ignore * vertex weights, we are only interested in geometry. Global operation. */ if (d == 2){ inertial_matrix2D(zz, *coords, num_obj, tran->CM, im); } else{ inertial_matrix3D(zz, *coords, num_obj, tran->CM, im); } /* * The inertial matrix is a 3x3 or 2x2 real symmetric matrix. Get its * three or two orthonormal eigenvectors. These usually indicate the * orientation of the geometry. */ rc = eigenvectors(im, tran->Evecs, d); if (rc){ if (zz->Proc == 0){ ZOLTAN_PRINT_WARN(0, yo, "REDUCE_DIMENSIONS calculation failed"); } goto End; } /* * Here we check to see if the eigenvectors are very close * to the coordinate axes. If so, we can more quickly * determine whether the geometry is degenerate, and also more * quickly transform the geometry to the lower dimensional * space. */ axis_aligned = 0; for (i=0; i<d; i++){ tran->Axis_Order[i] = -1; } for (j=0; j<d; j++){ for (i=0; i<d; i++){ x = fabs(tran->Evecs[i][j]); if (NEAR_ONE(x)){ tran->Axis_Order[j] = i; /* e'vector j is very close to i axis */ break; } } if (tran->Axis_Order[j] < 0){ break; } } if ((tran->Axis_Order[0] >= 0) && (tran->Axis_Order[1] >= 0) && (tran->Axis_Order[2] >= 0)){ axis_aligned = 1; } /* * Calculate the extent of the geometry along the three lines defined * by the direction of the eigenvectors through the center of mass. */ projected_distances(zz, *coords, num_obj, tran->CM, tran->Evecs, dist, d, axis_aligned, tran->Axis_Order); /* * Decide whether these distances indicate the geometry is * very flat in one or two directions. */ target_dim = get_target_dimension(dist, order, deg_ratio, d); if (target_dim > 0){ /* * Yes, geometry is degenerate */ if ((zz->Debug_Level > 0) && (zz->Proc == 0)){ if (d == 2){ sprintf(msg, "Geometry (~%lf x %lf), exceeds %lf to 1.0 ratio", dist[order[0]], dist[order[1]], deg_ratio); } else{ sprintf(msg, "Geometry (~%lf x %lf x %lf), exceeds %lf to 1.0 ratio", dist[order[0]], dist[order[1]], dist[order[2]], deg_ratio); } ZOLTAN_PRINT_INFO(zz->Proc, yo, msg); sprintf(msg, "We'll treat it as %d dimensional",target_dim); ZOLTAN_PRINT_INFO(zz->Proc, yo, msg); } if (axis_aligned){ /* ** Create new geometry, transforming the primary direction ** to the X-axis, and the secondary to the Y-axis. */ tran->Permutation[0] = tran->Axis_Order[order[0]]; if (target_dim == 2){ tran->Permutation[1] = tran->Axis_Order[order[1]]; } } else{ /* * Reorder the eigenvectors (they're the columns of evecs) from * longest projected distance to shorted projected distance. Compute * the transpose (the inverse) of the matrix. This will transform * the geometry to align along the X-Y plane, or along the X axis. */ for (i=0; i< target_dim; i++){ tran->Transformation[i][2] = 0.0; for (j=0; j<d; j++){ tran->Transformation[i][j] = tran->Evecs[j][order[i]]; } } for (i=target_dim; i< 3; i++){ for (j=0; j<3; j++){ tran->Transformation[i][j] = 0.0; } } } tran->Target_Dim = target_dim; transform_coordinates(*coords, num_obj, d, tran); } /* If geometry is very flat */ } /* If REDUCE_DIMENSIONS is true */ } /* If 2-D or 3-D rcb, rib or hsfc */ End: if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error found; no coordinates returned."); if (alloced_coords) ZOLTAN_FREE(coords); } ZOLTAN_TRACE_EXIT(zz, yo); return ierr; }
int Zoltan_DD_Update ( Zoltan_DD_Directory *dd, /* directory state information */ ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to update */ ZOLTAN_ID_PTR lid, /* Incoming corresponding LIDs (optional) */ char *user, /* Incoming list of user data (optional) */ int *partition, /* Optional, grouping of GIDs to partitions */ int count) /* Number of GIDs in update list */ { int *procs = NULL; /* list of processors to contact */ DD_Update_Msg *ptr = NULL; ZOLTAN_COMM_OBJ *plan = NULL; /* for efficient MPI communication */ char *sbuff = NULL; /* send buffer */ char *sbufftmp = NULL;/* pointer into send buffer */ char *rbuff = NULL; /* receive buffer */ char *rbufftmp = NULL;/* pointer into receive buffer */ int nrec = 0; /* number of receives to expect */ int i; int err; int errcount = 0; /* count of GIDs not found, added */ char str[100]; /* build error message string */ char *yo = "Zoltan_DD_Update"; /* input sanity checking */ if (dd == NULL || count < 0 || (gid == NULL && count > 0)) { ZOLTAN_PRINT_ERROR ((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc), yo, "Invalid input argument"); return ZOLTAN_FATAL; } if (dd->debug_level > 4) ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL); /* part of initializing the error checking process */ /* for each linked list head, walk its list resetting errcheck */ if (dd->debug_level) for (i = 0; i < dd->table_length; i++) { DD_NodeIdx nodeidx; for (nodeidx = dd->table[i]; nodeidx != -1; nodeidx = dd->nodelist[nodeidx].next) dd->nodelist[nodeidx].errcheck = ZOLTAN_DD_NO_PROC; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After reset errcheck"); /* allocate memory for list of processors to contact */ if (count) { procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count); if (procs == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list"); err = ZOLTAN_MEMERR; goto fini; } } /* allocate memory for DD_Update_Msg send buffer */ if (count) { sbuff = (char*) ZOLTAN_CALLOC (count, dd->update_msg_size); if (sbuff == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer"); err = ZOLTAN_MEMERR; goto fini; } } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs"); /* for each GID given, fill in contact list and then message structure */ sbufftmp = sbuff; for (i = 0; i < count; i++) { procs[i] = dd->hash(gid + i*dd->gid_length, dd->gid_length, dd->nproc, dd->hashdata, dd->hashfn); ptr = (DD_Update_Msg*) sbufftmp; sbufftmp += dd->update_msg_size; ptr->lid_flag = (lid) ? 1 : 0; ptr->user_flag = (user) ? 1 : 0; ptr->partition_flag = (partition) ? 1 : 0; ptr->partition = (partition) ? *(partition + i) : -1; ptr->owner = dd->my_proc; ZOLTAN_SET_ID (dd->gid_length, ptr->gid, gid + i * dd->gid_length); if (lid) { ZOLTAN_SET_ID(dd->lid_length, ptr->gid + dd->gid_length, lid + i * dd->lid_length); } else { memset(ptr->gid + dd->gid_length, 0, dd->lid_length); } if (user) { memcpy(ptr->gid + (dd->gid_length + dd->lid_length), user + (size_t)i * (size_t)(dd->user_data_length), dd->user_data_length); } else { memset(ptr->gid + (dd->gid_length + dd->lid_length), 0, dd->user_data_length); } } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill contact list"); /* now create efficient communication plan */ err = Zoltan_Comm_Create (&plan, count, procs, dd->comm, ZOLTAN_DD_UPDATE_MSG_TAG, &nrec); if (err != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Create error"); goto fini; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create"); /* If dd has no nodes allocated (e.g., first call to DD_Update; * create the nodelist and freelist */ if (nrec && dd->nodelistlen == 0) { DD_Memory_Alloc_Nodelist(dd, (DD_NodeIdx) nrec, 0.); /* TODO Add overalloc parameter */ } /* allocate receive buffer for nrec DD_Update_Msg structures */ if (nrec) { rbuff = (char*)ZOLTAN_MALLOC((size_t)nrec*(size_t)(dd->update_msg_size)); if (rbuff == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Receive buffer malloc failed"); err = ZOLTAN_MEMERR; goto fini; } } /* send my update messages & receive updates directed to me */ err = Zoltan_Comm_Do (plan, ZOLTAN_DD_UPDATE_MSG_TAG+1, sbuff, dd->update_msg_size, rbuff); if (err != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Do error"); goto fini; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do"); /* for each message rec'd, update local directory information */ errcount = 0; rbufftmp = rbuff; for (i = 0; i < nrec; i++) { ptr = (DD_Update_Msg *) rbufftmp; rbufftmp += dd->update_msg_size; err = DD_Update_Local (dd, ptr->gid, (ptr->lid_flag) ? (ptr->gid + dd->gid_length) : NULL, (char*)((ptr->user_flag)?(ptr->gid+(dd->gid_length+dd->lid_length)):NULL), (ptr->partition_flag) ? (ptr->partition) : -1, /* illegal partition */ ptr->owner); if (err != ZOLTAN_OK) ++errcount; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Local update"); err = ZOLTAN_OK; if (dd->debug_level) /* overwrite error return if extra checking is on */ err = (errcount) ? ZOLTAN_WARN : ZOLTAN_OK; fini: ZOLTAN_FREE (&procs); ZOLTAN_FREE (&sbuff); ZOLTAN_FREE (&rbuff); Zoltan_Comm_Destroy (&plan); if (dd->debug_level) { sprintf (str, "Processed %d GIDs (%d local), %d GID errors", count, nrec, errcount); ZOLTAN_PRINT_INFO (dd->my_proc, yo, str); } if (dd->debug_level > 4) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return err; }
static int DD_Update_Local (Zoltan_DD_Directory *dd, ZOLTAN_ID_PTR gid, /* GID to update (in) */ ZOLTAN_ID_PTR lid, /* gid's LID (in), NULL if not needed */ char *user, /* gid's user data (in), NULL if not needed */ int partition, /* gid's partition (in), -1 if not used */ int owner) /* gid's current owner (proc number) (in) */ { int index; char *yo = "DD_Update_Local"; DD_NodeIdx nodeidx; DD_Node *ptr; /* input sanity checking */ if (dd == NULL || owner < 0 || owner >= dd->nproc || gid == NULL) { ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo, "Invalid input parameter"); return ZOLTAN_FATAL; } if (dd->debug_level > 5) ZOLTAN_TRACE_IN (dd->my_proc, yo, NULL); /* compute offset into hash table to find head of linked list */ index = Zoltan_DD_Hash2 (gid, dd->gid_length, dd->table_length, dd->hashdata, NULL); /* walk linked list until end looking for matching gid */ for (nodeidx = dd->table[index]; nodeidx != -1; nodeidx = dd->nodelist[nodeidx].next) { ptr = dd->nodelist + nodeidx; if (ZOLTAN_EQ_ID (dd->gid_length, gid, ptr->gid) == TRUE) { /* found match, update directory information */ if (lid) ZOLTAN_SET_ID (dd->lid_length,ptr->gid + dd->gid_length, lid); if (user) memcpy(ptr->gid + (dd->gid_length + dd->lid_length), user, dd->user_data_length); ptr->owner = owner; if (partition != -1) ptr->partition = partition; /* Response to multiple updates to a gid in 1 update cycle */ if (dd->debug_level > 0 && ptr->errcheck != owner) { ZOLTAN_PRINT_INFO (dd->my_proc, yo, "Multiply defined GID"); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_WARN; } ptr->errcheck = owner; if (dd->debug_level > 5) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_OK; /* ignore all errors */ } } /* gid not found. Create new DD_Node and fill it in */ nodeidx = DD_Memory_Alloc_Node(dd); ptr = dd->nodelist + nodeidx; ZOLTAN_SET_ID (dd->gid_length, ptr->gid, gid); if (lid) { ZOLTAN_SET_ID(dd->lid_length,ptr->gid + dd->gid_length, lid); } else { memset(ptr->gid + dd->gid_length, 0, dd->lid_length*sizeof(ZOLTAN_ID_TYPE)); } if (user) { memcpy(ptr->gid + (dd->gid_length + dd->lid_length), user, dd->user_data_length); } else { memset(ptr->gid + (dd->gid_length+dd->lid_length), 0, dd->user_data_length); } ptr->partition = partition; ptr->owner = owner; ptr->errcheck = owner; /* Add node to the linked list */ ptr->next = dd->table[index]; dd->table[index] = nodeidx; if (dd->debug_level > 6) ZOLTAN_PRINT_INFO (dd->my_proc, yo, "Created new directory item"); if (dd->debug_level > 5) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_OK; }
int main (int argc, char *argv[]) { Zoltan_DD_Directory *dd; ZTIMER *zt; int myproc; /* MPI rank, my processor's MPI ID */ int nproc; /* MPI size, number of processors */ ZOLTAN_ID_PTR glist = NULL; /* list of GIDs */ ZOLTAN_ID_PTR llist = NULL; /* list of LIDs */ ZOLTAN_ID_PTR ulist = NULL; /* list of user data of type ZOLTAN_ID_TYPE */ int *plist = NULL; /* list of partitions */ int *olist = NULL; /* list of owners */ Param param ; /* program's adjustable parameters */ char *store = NULL; /* non directory storage of test data */ Data *data = NULL; /* pointer into store */ /* these are all temporary variables */ int new_owner; int count; int i; char *p; int err; int errcount; int loop; char str[100]; /* for building output messages */ static int timer[7] = {-1,-1,-1,-1,-1,-1,-1}; char *yo = "DD_Main"; /* initialize MPI communications, ignore errors */ MPI_Init (&argc, &argv); MPI_Comm_rank (MPI_COMM_WORLD, &myproc); MPI_Comm_size (MPI_COMM_WORLD, &nproc); get_params (¶m); /* read input parameters */ zt = Zoltan_Timer_Create(1); MACRO_TIMER_START(5, "Total time", 0); MACRO_TIMER_START(0, "DD_Create time", 0); err = Zoltan_DD_Create (&dd, MPI_COMM_WORLD, param.glen, param.llen, param.ulen, param.tlen, param.debug_level); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Create"); MACRO_TIMER_STOP(0); param.slen = sizeof (Data) + sizeof(ZOLTAN_ID_TYPE) * (param.glen + param.llen + param.ulen); param.slen = Zoltan_Align(param.slen); store = (char*) ZOLTAN_MALLOC (param.count * param.slen); /* allocate storage for various lists */ glist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC(sizeof(ZOLTAN_ID_TYPE) * param.count * param.glen); plist = (int*) ZOLTAN_MALLOC (sizeof(int) * param.count); olist = (int*) ZOLTAN_MALLOC (sizeof(int) * param.count); if (param.llen) llist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC (sizeof (ZOLTAN_ID_TYPE) * param.count * param.llen); if (param.ulen) ulist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC (sizeof (ZOLTAN_ID_TYPE) * param.count * param.ulen) ; if (store == NULL || glist == NULL || (param.llen != 0 && llist == NULL) || (param.ulen != 0 && ulist == NULL) || plist == NULL || olist == NULL) { ZOLTAN_PRINT_ERROR (myproc, yo, "Unable to malloc storage lists"); return ZOLTAN_MEMERR; } initialize_data (¶m, store, nproc); /* create & update directory with myproc's initial simulated GIDs */ count = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) if (((Data *)p)->new_owner == myproc) { ZOLTAN_SET_ID (param.glen, glist + count * param.glen, ((Data*)p)->id); if (param.llen) ZOLTAN_SET_ID (param.llen, llist + count * param.llen, ((Data *)p)->id + param.glen); if (param.ulen) ZOLTAN_SET_ID (param.ulen, ulist + count * param.ulen, ((Data *)p)->id + (param.glen + param.llen)); plist [count] = ((Data *)p)->partition; ++count; } MACRO_TIMER_START (1, "DD_Update timer", 0); err = Zoltan_DD_Update (dd, glist, llist, ulist, plist, count); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update"); MACRO_TIMER_STOP(1); i = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) { ZOLTAN_SET_ID (param.glen, glist + i * param.glen, ((Data *)p)->id); ++i; } MACRO_TIMER_START(2, "DD_Find timer", 0); err = Zoltan_DD_Find (dd, glist, llist, ulist, plist, param.count, olist); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find"); MACRO_TIMER_STOP(2); errcount = 0; for (i = 0; i < param.count; i++) if (olist[i] != ((Data *) (store + i * param.slen))->new_owner) ++errcount; if (errcount) sprintf (str, "FIRST TEST FAILED, errcount is %d", errcount); else sprintf (str, "FIRST TEST SUCCESSFUL"); ZOLTAN_PRINT_INFO (myproc, yo, str); /* repeatedly simulate moving "dots" around the system */ for (loop = 0; loop < param.nloops; loop++) { for (p = store; p < store + param.count * param.slen; p += param.slen) ((Data*) p)->old_owner = ((Data*) p)->new_owner; /* randomly exchange some dots and randomly reassign others */ for (i = 0; i < (param.pmove * param.count)/100; i++) { Data *d1, *d2; d1 = (Data*) (store + param.slen * (rand() % param.count)); d2 = (Data*) (store + param.slen * (rand() % param.count)); new_owner = d1->new_owner; d1->new_owner = d2->new_owner; d2->new_owner = new_owner; } for (i = 0; i < (param.count * param.pscatter)/100; i++) ((Data*) (store + param.slen *(rand() % param.count)))->new_owner = rand() % nproc; /* determine which new GIDs myproc gained, and update directory */ count = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) if (((Data*)p)->new_owner == myproc) { ((Data*)p)->id[param.glen] = count; /* set LID */ ZOLTAN_SET_ID (param.glen, glist + count * param.glen, ((Data *)p)->id); if (param.llen) ZOLTAN_SET_ID (param.llen, llist + count * param.llen, ((Data*)p)->id + param.glen); if (param.ulen) ZOLTAN_SET_ID (param.ulen, ulist + count * param.ulen, ((Data*)p)->id + (param.glen + param.llen)); plist [count] = ((Data *)p)->partition; ++count; } MACRO_TIMER_START(1, "DD_Update", 0); err = Zoltan_DD_Update (dd, glist, llist, ulist, plist, count); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update"); MACRO_TIMER_STOP(1); /* use directory to "find" GIDs */ i = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) { ZOLTAN_SET_ID (param.glen, glist + i * param.glen, ((Data *)p)->id); ++i; } MACRO_TIMER_START(2, "DD_Find timer", 0); if (loop % 5 == 0) err = Zoltan_DD_Find(dd,glist,NULL,ulist,plist,param.count,olist); else if (loop % 7 == 0) err = Zoltan_DD_Find(dd,glist,llist,NULL,plist,param.count,olist); else if (loop % 9 == 0) err = Zoltan_DD_Find(dd,glist,llist,ulist,NULL,param.count,olist); else if (loop % 2 == 0) err = Zoltan_DD_Find(dd,glist,llist,ulist,plist,param.count,NULL); else err = Zoltan_DD_Find(dd,glist,llist,ulist,plist,param.count,olist); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find"); MACRO_TIMER_STOP(2); if (loop % 2) { errcount = 0 ; for (i = 0; i < param.count; i++) if (olist[i] != ((Data *)(store + i * param.slen))->new_owner) ++errcount; if (errcount) sprintf (str,"LOOP %d TEST FAILED, errcount is %d",loop,errcount); else sprintf (str, "LOOP %d TEST SUCCESSFUL", loop); ZOLTAN_PRINT_INFO (myproc, yo, str) ; } else { sprintf (str, "LOOP %d Completed", loop); ZOLTAN_PRINT_INFO (myproc, yo, str); } /* randomly remove a percentage of GIDs from the directory */ count = 0; for (i = 0; i < (param.count * param.pdelete)/100; i++) { data = (Data*) (store + param.slen * (rand() % param.count)); if (data->new_owner == myproc) { ZOLTAN_SET_ID (param.glen, glist + count * param.glen, data->id); ++count; } data->new_owner = NO_PROC; } MACRO_TIMER_START(3, "DD_Remove timer", 0); err = Zoltan_DD_Remove (dd, glist, count); MACRO_TIMER_STOP(3); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Remove"); /* update directory (put directory entries back in) */ for (p = store; p < store + param.count * param.slen; p += param.slen) if (((Data*)p)->new_owner == NO_PROC) ((Data*)p)->new_owner = loop % nproc; /* place in new location */ count = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) if (((Data*)p)->new_owner == myproc) { ZOLTAN_SET_ID(param.glen,glist+count*param.glen,((Data*)p)->id); if (param.llen) ZOLTAN_SET_ID (param.llen, llist + count * param.llen, ((Data*)p)->id + param.glen); if (param.ulen) ZOLTAN_SET_ID(param.ulen, ulist + count * param.ulen, ((Data *)p)->id + (param.glen + param.llen)); plist [count] = ((Data*)p)->partition; ++count; } MACRO_TIMER_START(1, "DD_Update timer", 0); err = Zoltan_DD_Update (dd, glist, NULL, NULL, NULL, count); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update"); MACRO_TIMER_STOP(1); } /* now Find directory info for GIDs myproc now owns and validate */ count = 0; for (i = 0; i < param.count; i++) { data = (Data *) (store + i * param.slen); if (data->new_owner == myproc) { ZOLTAN_SET_ID (param.glen, glist + count * param.glen, data->id); ++count; } } MACRO_TIMER_START(2, "DD_Find", 0); err = Zoltan_DD_Find (dd, glist, NULL, NULL, NULL, count, olist); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find"); MACRO_TIMER_STOP(2); errcount = 0; for (i = 0; i < count; i++) if (olist[i] != myproc) ++errcount; if (errcount) { sprintf (str, "errcount is %d", errcount); ZOLTAN_PRINT_ERROR (myproc, yo, str); } else ZOLTAN_PRINT_INFO (myproc, yo, "TEST SUCCESSFUL"); Zoltan_DD_Stats (dd); /* done, now free memory, stop MPI & directory, return */ ZOLTAN_FREE (&store); ZOLTAN_FREE (&glist); ZOLTAN_FREE (&plist); ZOLTAN_FREE (&olist); if (param.llen) ZOLTAN_FREE (&llist); if (param.ulen) ZOLTAN_FREE (&ulist); ZOLTAN_PRINT_INFO (myproc, yo, "Completing program"); MACRO_TIMER_START(4, "DD_Destroy", 0); Zoltan_DD_Destroy (&dd); MACRO_TIMER_STOP(4); MACRO_TIMER_STOP(5); Zoltan_Timer_PrintAll(zt, 0, MPI_COMM_WORLD, stderr); MPI_Finalize (); return ZOLTAN_OK; }
int Zoltan_DD_Find ( Zoltan_DD_Directory *dd, /* contains directory state information */ ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to get owners proc */ ZOLTAN_ID_PTR lid, /* Outgoing corresponding list of LIDs */ char *data, /* Outgoing optional corresponding user data */ int *partition, /* Outgoing optional partition information */ int count, /* Count of GIDs in above list (in) */ int *owner) /* Outgoing optional list of data owners */ { ZOLTAN_COMM_OBJ *plan = NULL; /* efficient MPI communication */ char *rbuff = NULL; /* receive buffer */ char *rbufftmp = NULL; /* pointer into receive buffer */ char *sbuff = NULL; /* send buffer */ char *sbufftmp = NULL; /* pointer into send buffer */ int *procs = NULL; /* list of processors to contact */ DD_Find_Msg *ptr = NULL; int i; int nrec; /* number of messages to receive */ int err = ZOLTAN_OK; /* return error condition */ int errcount; /* count of GIDs not found */ char *yo = "Zoltan_DD_Find"; /* input sanity check */ if (dd == NULL || count < 0 || (gid == NULL && count > 0)) { ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo, "Invalid input argument"); return ZOLTAN_FATAL; } if (dd->debug_level > 4) ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL); /* allocate memory for processors to contact for directory info */ if (count) { procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count); if (procs == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list"); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return ZOLTAN_MEMERR; } } /* allocate memory for DD_Find_Msg send buffer */ if (count) { sbuff = (char*) ZOLTAN_CALLOC (count, dd->find_msg_size); if (sbuff == NULL) { ZOLTAN_FREE (&procs); ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer"); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return ZOLTAN_MEMERR; } } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs"); /* for each GID, fill DD_Find_Msg buffer and contact list */ sbufftmp = sbuff; for (i = 0; i < count; i++) { procs[i] = dd->hash (gid + i*dd->gid_length, dd->gid_length, dd->nproc, dd->hashdata, dd->hashfn); ptr = (DD_Find_Msg*) sbufftmp; sbufftmp += dd->find_msg_size; ptr->index = i; ptr->proc = procs[i]; ZOLTAN_SET_ID (dd->gid_length, ptr->id, gid + i*dd->gid_length); } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill"); /* create efficient communication plan */ err = Zoltan_Comm_Create (&plan, count, procs, dd->comm, ZOLTAN_DD_FIND_MSG_TAG, &nrec); if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create"); if (err != ZOLTAN_OK) goto fini; /* allocate receive buffer */ if (nrec) { rbuff = (char*) ZOLTAN_MALLOC ((size_t)nrec*(size_t)(dd->find_msg_size)); if (rbuff == NULL) { err = ZOLTAN_MEMERR; goto fini; } } /* send out find messages across entire system */ err = Zoltan_Comm_Do (plan, ZOLTAN_DD_FIND_MSG_TAG+1, sbuff, dd->find_msg_size, rbuff); if (err != ZOLTAN_OK) goto fini; if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do"); /* get find messages directed to me, fill in return information */ errcount = 0; rbufftmp = rbuff; for (i = 0; i < nrec; i++) { ptr = (DD_Find_Msg*) rbufftmp; rbufftmp += dd->find_msg_size; err = DD_Find_Local (dd, ptr->id, ptr->id, (char *)(ptr->id + dd->max_id_length), &ptr->partition, &ptr->proc); if (err == ZOLTAN_WARN) ++errcount; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill in return info"); /* send return information back to requester */ err = Zoltan_Comm_Do_Reverse(plan, ZOLTAN_DD_FIND_MSG_TAG+2, rbuff, dd->find_msg_size, NULL, sbuff); if (err != ZOLTAN_OK) goto fini; if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Reverse"); /* fill in user supplied lists with returned information */ sbufftmp = sbuff; for (i = 0; i < count; i++) { ptr = (DD_Find_Msg*) sbufftmp; sbufftmp += dd->find_msg_size; if (owner) owner[ptr->index] = ptr->proc; if (partition) partition[ptr->index] = ptr->partition ; if (lid) ZOLTAN_SET_ID(dd->lid_length,lid+ptr->index*dd->lid_length,ptr->id); if (data) memcpy(data + (size_t)(ptr->index) * (size_t)(dd->user_data_length), ptr->id + dd->max_id_length, dd->user_data_length); } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill return lists"); /* err = ZOLTAN_OK; */ MPI_Allreduce(&errcount, &err, 1, MPI_INT, MPI_SUM, dd->comm); err = (err) ? ZOLTAN_WARN : ZOLTAN_OK; /* if at least one GID was not found, potentially notify caller of error */ if (dd->debug_level > 0) { char str[100]; /* diagnostic message string */ sprintf (str, "Processed %d GIDs, GIDs not found: %d", count, errcount); ZOLTAN_PRINT_INFO (dd->my_proc, yo, str); } fini: ZOLTAN_FREE (&sbuff); ZOLTAN_FREE (&rbuff); ZOLTAN_FREE (&procs) ; Zoltan_Comm_Destroy (&plan); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return err; }
int Zoltan_DD_Find ( Zoltan_DD_Directory *dd, /* contains directory state information */ ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to get owners proc */ ZOLTAN_ID_PTR lid, /* Outgoing corresponding list of LIDs */ ZOLTAN_ID_PTR data, /* Outgoing optional corresponding user data */ int *partition, /* Outgoing optional partition information */ int count, /* Count of GIDs in above list (in) */ int *owner) /* Outgoing corresponding list of data locations */ { ZOLTAN_COMM_OBJ *plan = NULL ; /* efficient MPI communication */ char *rbuff = NULL ; /* receive buffer */ char *sbuff = NULL ; /* send buffer */ int *procs = NULL ; /* list of processors to contact */ DD_Find_Msg *ptr = NULL ; int i ; int nrec ; /* number of messages to receive */ int err ; /* return error condition */ int errcount ; /* count of GIDs not found */ char *yo = "Zoltan_DD_Find" ; if (dd != NULL && dd->debug_level > 1) ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL); /* input sanity check */ if (dd == NULL || count < 0 || ((owner == NULL || gid == NULL) && count > 0)) { ZOLTAN_PRINT_ERROR ((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc), yo, "Invalid input argument.") ; if (dd != NULL && dd->debug_level > 1) ZOLTAN_TRACE_OUT((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc), yo, NULL); return ZOLTAN_DD_INPUT_ERROR ; } /* allocate memory for processors to contact for directory info */ if (count > 0) { procs = (int *) ZOLTAN_MALLOC (sizeof (int) * count) ; if (procs == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list.") ; if (dd->debug_level > 1) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return ZOLTAN_DD_MEMORY_ERROR ; } } /* allocate memory for DD_Find_Msg send buffer */ if (count > 0) { sbuff = (char *) ZOLTAN_MALLOC (dd->find_msg_size * count) ; if (sbuff == NULL) { ZOLTAN_FREE (&procs) ; ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer.") ; if (dd->debug_level > 1) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return ZOLTAN_DD_MEMORY_ERROR ; } } if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs."); /* for each GID, fill DD_Find_Msg buffer and contact list */ for (i = 0 ; i < count ; i++) { procs[i] = dd->hash (gid + i*dd->gid_length, dd->gid_length, dd->nproc) ; ptr = (DD_Find_Msg *) (sbuff + i * dd->find_msg_size) ; ptr->index = i ; ptr->proc = procs[i] ; ZOLTAN_SET_ID (dd->gid_length, ptr->id, gid + i*dd->gid_length) ; } if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill."); /* create efficient communication plan */ err = Zoltan_Comm_Create (&plan, count, procs, dd->comm, ZOLTAN_DD_FIND_MSG_TAG, &nrec) ; if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create."); if (err != ZOLTAN_OK) goto fini ; /* allocate receive buffer */ if (nrec > 0) { rbuff = (char *) ZOLTAN_MALLOC (nrec * dd->find_msg_size) ; if (rbuff == NULL) { err = ZOLTAN_DD_MEMORY_ERROR ; goto fini ; } } /* send out find messages across entire system */ err = Zoltan_Comm_Do (plan, ZOLTAN_DD_FIND_MSG_TAG+1, sbuff, dd->find_msg_size, rbuff) ; if (err != ZOLTAN_OK) goto fini ; if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do."); /* get find messages directed to me, fill in return information */ errcount = 0 ; for (i = 0 ; i < nrec ; i++) { ptr = (DD_Find_Msg *) (rbuff + i*dd->find_msg_size) ; err = DD_Find_Local (dd, ptr->id, ptr->id, ptr->id + dd->max_id_length, (partition) ? (&ptr->partition) : NULL, &ptr->proc) ; if (err == ZOLTAN_DD_GID_NOT_FOUND_ERROR) errcount++ ; } if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill in return info."); /* send return information back to requester */ err = Zoltan_Comm_Do_Reverse(plan, ZOLTAN_DD_FIND_MSG_TAG+2, rbuff, dd->find_msg_size, NULL, sbuff) ; if (err != ZOLTAN_OK) goto fini ; if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Reverse."); /* fill in user supplied lists with returned information */ for (i = 0 ; i < count ; i++) { ptr = (DD_Find_Msg *) (sbuff + i*dd->find_msg_size) ; owner[ptr->index] = ptr->proc ; if (partition != NULL) partition[ptr->index] = ptr->partition ; if (lid != NULL) ZOLTAN_SET_ID (dd->lid_length, lid + ptr->index * dd->lid_length, ptr->id) ; if (data != NULL) ZOLTAN_SET_ID (dd->user_data_length, data + ptr->index * dd->user_data_length, ptr->id + dd->max_id_length) ; } /* if at least one GID was not found, notify caller of error */ err = (errcount == 0) ? ZOLTAN_DD_NORMAL_RETURN : ZOLTAN_DD_GID_NOT_FOUND_ERROR ; if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill return lists."); fini: ZOLTAN_FREE (&sbuff) ; ZOLTAN_FREE (&rbuff) ; ZOLTAN_FREE (&procs) ; Zoltan_Comm_Destroy (&plan) ; if (err != ZOLTAN_DD_NORMAL_RETURN) ZOLTAN_PRINT_WARN (dd->my_proc, yo, "Return is not normal.") ; if (dd->debug_level > 0) { char str[100] ; /* diagnostic message string */ sprintf (str, "Processed %d GIDs, GIDs not found: %d", count, errcount) ; ZOLTAN_PRINT_INFO (dd->my_proc, yo, str) ; } if (dd->debug_level > 1) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return err ; }
static int DD_Update_Local (Zoltan_DD_Directory *dd, ZOLTAN_ID_PTR gid, /* GID to update (in) */ ZOLTAN_ID_PTR lid, /* gid's LID (in), NULL if not needed */ ZOLTAN_ID_PTR user, /* gid's user data (in), NULL if not needed */ int partition, /* gid's partition (in), -1 if not used */ int owner) /* gid's current owner (proc number) (in) */ { DD_Node **ptr; int index; char *yo = "DD_Update_Local"; /* input sanity checking */ if (dd == NULL || owner < 0 || owner >= dd->nproc || gid == NULL) { ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo, "Invalid input parameter"); return ZOLTAN_FATAL; } if (dd->debug_level > 5) ZOLTAN_TRACE_IN (dd->my_proc, yo, NULL); /* compute offset into hash table to find head of linked list */ index = Zoltan_DD_Hash2 (gid, dd->gid_length, dd->table_length, dd->hashdata); /* walk linked list until end looking for matching gid */ for (ptr = dd->table+index; *ptr != NULL; ptr = &((*ptr)->next)) if (ZOLTAN_EQ_ID (dd->gid_length, gid, (*ptr)->gid) == TRUE) { /* found match, update directory information */ if (lid) ZOLTAN_SET_ID (dd->lid_length,(*ptr)->gid + dd->gid_length, lid); if (user) ZOLTAN_SET_ID (dd->user_data_length, (*ptr)->gid + (dd->gid_length + dd->lid_length), user); (*ptr)->owner = owner; if (partition != -1) (*ptr)->partition = partition; /* Response to multiple updates to a gid in 1 update cycle */ if (dd->debug_level > 0 && (*ptr)->errcheck != owner) { ZOLTAN_PRINT_INFO (dd->my_proc, yo, "Multiply defined GID"); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_WARN; } (*ptr)->errcheck = owner; if (dd->debug_level > 5) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_OK; /* ignore all errors */ } /* gid not found. Create new DD_Node and fill it in */ *ptr = (DD_Node*) ZOLTAN_MALLOC (dd->node_size); if (*ptr == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc new Node"); if (dd->debug_level > 5) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_MEMERR; } ZOLTAN_SET_ID (dd->gid_length, (*ptr)->gid, gid); if (lid) { ZOLTAN_SET_ID(dd->lid_length,(*ptr)->gid + dd->gid_length, lid); } else { memset((*ptr)->gid + dd->gid_length, 0, dd->lid_length*sizeof(ZOLTAN_ID_TYPE)); } if (user) { ZOLTAN_SET_ID(dd->user_data_length, (*ptr)->gid + (dd->gid_length + dd->lid_length), user); } else { memset((*ptr)->gid + (dd->gid_length+dd->lid_length), 0, dd->user_data_length*sizeof(ZOLTAN_ID_TYPE)); } (*ptr)->partition = partition ; (*ptr)->next = NULL; (*ptr)->owner = owner; (*ptr)->errcheck = owner; if (dd->debug_level > 6) ZOLTAN_PRINT_INFO (dd->my_proc, yo, "Created new directory item"); if (dd->debug_level > 5) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_OK; }