/*
* Function that initializes the region data structure. It uses the
* global ID, coordinates and weight provided by the application.
*/
static void initialize_region(ZZ *zz, pRegion *ret, ZOLTAN_ID_PTR global_id,
ZOLTAN_ID_PTR local_id, int wgtflag, float wgt,
int num_dim, double *geom_vec)
{
pRegion reg;
int i;
reg = (pRegion) ZOLTAN_MALLOC(sizeof(Region));
*ret = reg;
reg->Global_ID = ZOLTAN_MALLOC_GID(zz);
reg->Local_ID = ZOLTAN_MALLOC_LID(zz);
ZOLTAN_SET_GID(zz, reg->Global_ID, global_id);
ZOLTAN_SET_LID(zz, reg->Local_ID, local_id);
reg->Proc = zz->Proc;
/* reg->Proc = 0; */
reg->Coord[0] = reg->Coord[1] = reg->Coord[2] = 0.0;
for (i = 0; i < num_dim; i++)
reg->Coord[i] = geom_vec[i];
#if 0
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
fprintf(stderr, "Result info on %d: %d %d %d %lf %lf %lf\n", zz->Proc,
reg->Local_ID, reg->Global_ID, reg->Proc,
reg->Coord[0], reg->Coord[1], reg->Coord[2]);
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
#endif
if (wgtflag)
reg->Weight = wgt;
else
reg->Weight = 1;
reg->next = NULL;
}
/*
* void Zoltan_Oct_addRegion(pOctant octant, pRegion region)
* add a region to oct's list
*/
int Zoltan_Oct_addRegion(ZZ *zz, pOctant oct, pRegion region) {
char *yo = "Zoltan_Oct_addRegion";
pRegion entry; /* pointer to new entry in region list */
if(oct == NULL)
return ZOLTAN_WARN;
entry = (pRegion) ZOLTAN_MALLOC(sizeof(Region)); /* malloc space for region */
if(entry == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Cannot allocated memory for region.");
return ZOLTAN_MEMERR;
}
entry->Global_ID = ZOLTAN_MALLOC_GID(zz);
entry->Local_ID = ZOLTAN_MALLOC_LID(zz);
/* copy region information into the entry */
vector_set(entry->Coord, region->Coord);
entry->Weight = region->Weight;
ZOLTAN_SET_GID(zz, entry->Global_ID, region->Global_ID);
ZOLTAN_SET_LID(zz, entry->Local_ID, region->Local_ID);
entry->Proc = region->Proc;
/* attach region to region list */
entry->next = oct->list;
oct->list = entry;
return ZOLTAN_OK;
}
/*
* int Zoltan_Oct_copy_info(pRegion *destination, pRegion source)
*
* Copies region information from the source to the destination
*/
static int Zoltan_Oct_copy_info(ZZ *zz, pRegion src, pRegion *dest) {
pRegion copy;
char *yo = "Zoltan_Oct_copy_info";
int ierr = ZOLTAN_OK;
/* mallloc space for destination */
copy = (pRegion) ZOLTAN_MALLOC(sizeof(Region));
if(copy == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
copy->Global_ID = ZOLTAN_MALLOC_GID(zz);
copy->Local_ID = ZOLTAN_MALLOC_LID(zz);
if (copy->Global_ID == NULL || (zz->Num_LID && copy->Local_ID == NULL)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
/* set up return pointer */
*dest = copy;
/* copy all important information */
vector_set(copy->Coord, src->Coord);
copy->Weight = src->Weight;
ZOLTAN_SET_GID(zz, copy->Global_ID, src->Global_ID);
ZOLTAN_SET_LID(zz, copy->Local_ID, src->Local_ID);
copy->Proc = src->Proc;
copy->attached = 0;
return ierr;
}
void Zoltan_Reftree_Hash_Insert(ZZ *zz, ZOLTAN_REFTREE *reftree_node,
struct Zoltan_Reftree_hash_node **hashtab, int size)
{
int i;
struct Zoltan_Reftree_hash_node *new_entry;
i = Zoltan_Hash(reftree_node->global_id, zz->Num_GID, (unsigned int)size);
new_entry = (struct Zoltan_Reftree_hash_node *)
ZOLTAN_MALLOC(sizeof(struct Zoltan_Reftree_hash_node));
new_entry->gid = ZOLTAN_MALLOC_GID(zz);
ZOLTAN_SET_GID(zz, new_entry->gid,reftree_node->global_id);
new_entry->reftree_node = reftree_node;
new_entry->next = hashtab[i];
hashtab[i] = new_entry;
}
void Zoltan_Reftree_IntHash_Insert(ZZ *zz, ZOLTAN_ID_PTR gid, int lid,
struct Zoltan_Reftree_inthash_node **hashtab,
int size)
{
int i;
struct Zoltan_Reftree_inthash_node *new_entry;
i = Zoltan_Hash(gid, zz->Num_GID, (unsigned int)size);
new_entry = (struct Zoltan_Reftree_inthash_node *)
ZOLTAN_MALLOC(sizeof(struct Zoltan_Reftree_inthash_node));
new_entry->gid = ZOLTAN_MALLOC_GID(zz);
ZOLTAN_SET_GID(zz, new_entry->gid,gid);
new_entry->lid = lid;
new_entry->next = hashtab[i];
hashtab[i] = new_entry;
}
static int Zoltan_Reftree_Sum_Weights(ZZ *zz)
{
/*
* Function to sum the weights in the refinement tree. On input the
* refinement tree should be valid and have weight set. On output the
* values in summed_weight at each node is the sum of the weights in the
* subtree with that node as root.
* This function also sets assigned_to_me for interior nodes to be
* 1 if the entire subtree is assigned to this processor
* 0 if none of the subtree is assigned to this processor
* -1 if some of the subtree is assigned to this processor
*/
char *yo = "Zoltan_Reftree_Sum_Weights";
ZOLTAN_REFTREE *root; /* Root of the refinement tree */
int wdim; /* Dimension of the weight array */
int i,j; /* loop counters */
int count; /* counter */
ZOLTAN_ID_PTR leaf_list = NULL;
/* leaves for which some proc requests weight */
ZOLTAN_ID_PTR all_leaflist = NULL;
/* leaf_list from all processors */
int reqsize; /* length of leaf_list */
int *reqsize_all; /* reqsize from all processors */
int sum_reqsize; /* sum of all reqsize */
int *displs; /* running sum of all reqsize */
int my_start; /* position in leaf_list of this proc's list */
int nproc; /* number of processors */
ZOLTAN_REFTREE *node; /* a node in the refinement tree */
struct Zoltan_Reftree_hash_node **hashtab; /* hash table */
int hashsize; /* dimension of hash table */
float *send_float; /* sending message of floats */
float *req_weights; /* the requested weights */
int num_gid_entries = zz->Num_GID; /* Number of array entries in a global ID */
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* set the root and hash table
*/
root = ((struct Zoltan_Reftree_data_struct *)zz->LB.Data_Structure)->reftree_root;
if (root == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Refinement tree not defined.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_FATAL);
}
hashtab = ((struct Zoltan_Reftree_data_struct *)zz->LB.Data_Structure)->hash_table;
hashsize = ((struct Zoltan_Reftree_data_struct *)zz->LB.Data_Structure)->hash_table_size;
/*
* Determine the dimension of the weight array
*/
if (zz->Obj_Weight_Dim == 0) {
wdim = 1;
} else {
wdim = zz->Obj_Weight_Dim;
}
/*
* In the first pass, sum the weights of the nodes that are assigned to
* this processor, and count the leaves that are not.
*/
count = 0;
for (i=0; i<root->num_child; i++) {
Zoltan_Reftree_Sum_My_Weights(zz,&(root->children[i]),&count,wdim);
}
root->assigned_to_me = -1;
/*
* Make a list of the leaves that are not assigned to this processor
*/
if (count == 0)
leaf_list = ZOLTAN_MALLOC_GID(zz);
else
leaf_list = ZOLTAN_MALLOC_GID_ARRAY(zz, count);
if (leaf_list == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_MEMERR);
}
count = 0;
Zoltan_Reftree_List_Other_Leaves(zz, root,leaf_list,&count);
/*
* Get the unknown leaf weights from other processors.
*/
nproc = zz->Num_Proc;
reqsize = count;
/*
* Build a list of all processor's request list by concatinating them in
* the order of the processor ranks
*/
/*
* Determine the request size of all processors
*/
reqsize_all = (int *)ZOLTAN_MALLOC(nproc*sizeof(int));
displs = (int *)ZOLTAN_MALLOC(nproc*sizeof(int));
if (reqsize_all == NULL || displs == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
Zoltan_Multifree(__FILE__, __LINE__, 3, &displs,
&reqsize_all,
&leaf_list);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_MEMERR);
}
MPI_Allgather((void *)&reqsize,1,MPI_INT,(void *)reqsize_all,1,MPI_INT,
zz->Communicator);
displs[0] = 0;
for (i=1; i<nproc; i++) displs[i] = displs[i-1]+reqsize_all[i-1];
sum_reqsize = displs[nproc-1] + reqsize_all[nproc-1];
my_start = displs[zz->Proc];
/*
* If sum_reqsize is 0, nothing needs to be communciated
*/
if (sum_reqsize == 0) {
Zoltan_Multifree(__FILE__, __LINE__, 3, &displs,
&reqsize_all,
&leaf_list);
}
else {
/*
* Gather the request list from all processors
*/
all_leaflist = ZOLTAN_MALLOC_GID_ARRAY(zz, sum_reqsize);
if (all_leaflist == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
Zoltan_Multifree(__FILE__, __LINE__, 4, &all_leaflist,
&displs,
&reqsize_all,
&leaf_list);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_MEMERR);
}
/* KDDKDD Changed MPI_BYTE to ZOLTAN_ID_MPI_TYPE */
/* Account for number of array entries in an ID. */
for (i=0; i<nproc; i++) {
reqsize_all[i] = reqsize_all[i]*num_gid_entries;
displs[i] = displs[i]*num_gid_entries;
}
MPI_Allgatherv((void *)leaf_list,reqsize*num_gid_entries,ZOLTAN_ID_MPI_TYPE,
(void *)all_leaflist,reqsize_all,displs,ZOLTAN_ID_MPI_TYPE,
zz->Communicator);
ZOLTAN_FREE(&displs);
ZOLTAN_FREE(&leaf_list);
for (i=0; i<nproc; i++) reqsize_all[i] = reqsize_all[i]/num_gid_entries;
/*
* Create a list with the partial sums this processor has
*/
send_float = (float *) ZOLTAN_MALLOC(sizeof(float)*wdim*sum_reqsize);
if (send_float == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
Zoltan_Multifree(__FILE__, __LINE__, 3, &send_float,
&all_leaflist,
&reqsize_all);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_MEMERR);
}
for (i=0; i<sum_reqsize; i++) {
node = Zoltan_Reftree_hash_lookup(zz, hashtab,
&(all_leaflist[i*num_gid_entries]),
hashsize);
if (node == NULL)
for (j=0; j<wdim; j++) send_float[i*wdim+j] = 0.0;
else
for (j=0; j<wdim; j++) send_float[i*wdim+j] = node->my_sum_weight[j];
}
/*
* Sum the weights over all the processors
*/
if (reqsize == 0)
req_weights = (float *) ZOLTAN_MALLOC(sizeof(float)*wdim);
else
req_weights = (float *) ZOLTAN_MALLOC(sizeof(float)*wdim*reqsize);
if (req_weights == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
Zoltan_Multifree(__FILE__, __LINE__, 4, &req_weights,
&send_float,
&all_leaflist,
&reqsize_all);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_MEMERR);
}
MPI_Reduce_scatter((void *)send_float, (void *)req_weights, reqsize_all,
MPI_FLOAT, MPI_SUM, zz->Communicator);
ZOLTAN_FREE(&send_float);
ZOLTAN_FREE(&reqsize_all);
/*
* Set the weights this processor requested
*/
for (i=0; i<count; i++) {
node = Zoltan_Reftree_hash_lookup(zz, hashtab,
&(all_leaflist[(i+my_start)*num_gid_entries]),
hashsize);
for (j=0; j<wdim; j++) node->summed_weight[j] = req_weights[i*wdim+j];
}
ZOLTAN_FREE(&req_weights);
ZOLTAN_FREE(&all_leaflist);
}
/*
* All the leaves now have summed_weight set.
* Sum the weights throughout the tree.
*/
Zoltan_Reftree_Sum_All_Weights(zz,root,wdim);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_OK);
}
/*
* void malloc_new_objects();
*
* gets the tags being imported into this processor, and sets up the
* import_tags array, and the nrectags array.
*/
static int malloc_new_objects(ZZ *zz, int nsentags, pRegion exported_tags,
ZOLTAN_ID_PTR exported_gids,
ZOLTAN_ID_PTR exported_lids, int *tag_pids,
int *nstags, pRegion *ex_tags,
pRegion prev_tags, ZOLTAN_ID_PTR prev_gids,
ZOLTAN_ID_PTR prev_lids, int npimtags,
float *c3)
{
char *yo = "malloc_new_objects";
int i; /* index counter */
int nreceives; /* number of messages received */
pRegion t_b_exp; /* array of tags to be exported */
pRegion tmp = NULL;
ZOLTAN_ID_PTR tmp_gids = NULL;
ZOLTAN_ID_PTR tmp_lids = NULL;
int msgtag, msgtag2;
int j;
int ierr = ZOLTAN_OK;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
float im_load;
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
im_load = 0;
msgtag = 32767;
ierr = Zoltan_Comm_Create(&comm_plan, nsentags, tag_pids, zz->Communicator,
msgtag, &nreceives);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from Zoltan_Comm_Create.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
if (nreceives > 0) {
tmp = (pRegion) ZOLTAN_MALLOC(nreceives * sizeof(Region));
tmp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, nreceives);
tmp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, nreceives);
if(tmp == NULL || !tmp_gids || (num_lid_entries && !tmp_lids)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
msgtag2 = 32766;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_tags,
sizeof(Region), (char *) tmp);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return(ierr);
}
msgtag2--;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_gids,
sizeof(ZOLTAN_ID_TYPE)*num_gid_entries,
(char *) tmp_gids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
fprintf(stderr, "OCT %s Error %s returned from Zoltan_Comm_Do\n", yo,
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return(ierr);
}
if (num_lid_entries > 0) {
msgtag2--;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_lids,
sizeof(ZOLTAN_ID_TYPE)*num_lid_entries,
(char *) tmp_lids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return(ierr);
}
}
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from Zoltan_Comm_Destroy.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
return(ierr);
}
/* get each message sent, and store region in import array */
j=0;
for (i=0; i<nreceives; i++) {
im_load += tmp[i].Weight;
if(tmp[i].newProc != zz->Proc) {
j++;
}
}
if((j + npimtags) != 0) { /* malloc import array */
if((t_b_exp = (pRegion)ZOLTAN_MALLOC((j+npimtags)*sizeof(Region)))==NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return ZOLTAN_MEMERR;
}
}
else
t_b_exp = NULL;
/* setup return pointer */
(*ex_tags) = t_b_exp;
j=0;
for (i=0; i<nreceives; i++) {
if(tmp[i].newProc != zz->Proc) {
t_b_exp[j] = tmp[i];
t_b_exp[j].Global_ID = ZOLTAN_MALLOC_GID(zz);
t_b_exp[j].Local_ID = ZOLTAN_MALLOC_LID(zz);
if (!(t_b_exp[j].Global_ID) ||
(num_lid_entries && !(t_b_exp[j].Local_ID))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return ZOLTAN_MEMERR;
}
ZOLTAN_SET_GID(zz, t_b_exp[j].Global_ID,
&(tmp_gids[i*num_gid_entries]));
ZOLTAN_SET_LID(zz, t_b_exp[j].Local_ID, &(tmp_lids[i*num_lid_entries]));
j++;
}
}
if(npimtags > 0) {
for(i=0; i<npimtags; i++) {
t_b_exp[j] = prev_tags[i];
t_b_exp[j].Global_ID = ZOLTAN_MALLOC_GID(zz);
t_b_exp[j].Local_ID = ZOLTAN_MALLOC_LID(zz);
if (!(t_b_exp[j].Global_ID) ||
(num_lid_entries && !(t_b_exp[j].Local_ID))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return ZOLTAN_MEMERR;
}
ZOLTAN_SET_GID(zz, t_b_exp[j].Global_ID,
&(prev_gids[i*num_gid_entries]));
ZOLTAN_SET_LID(zz, t_b_exp[j].Local_ID,
&(prev_lids[i*num_lid_entries]));
j++;
}
}
*nstags = j;
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
if((*nstags == 0) && (*ex_tags != NULL)) {
ZOLTAN_TRACE_DETAIL(zz, yo,
"Fatal error, import tags not empty but no tags received\n");
return ZOLTAN_FATAL;
}
*c3 = im_load;
return ierr;
}