コード例 #1
0
ファイル: postprocessing.c プロジェクト: nschloe/seacas
static int Compute_Adjpart(
  ZZ *zz,
  int nvtx,         /* Input:  # vtxs in this processor */
  indextype *vtxdist,     /* Input:  Distribution of vertices across processors */
  indextype *xadj,        /* Input:  Index of adjncy:  adjncy[xadj[i]] to 
                               adjncy[xadj[i]+1] are all edge nbors of vtx i. */
  indextype *adjncy,      /* Input:  Array of nbor vertices. */
  int *adjproc,     /* Input:  adjproc[j] == processor owning adjncy[j]. */
  indextype *part,        /* Input:  Partition assignments of vtxs. */
  int *adjpart      /* Output: adjpart[j] == partition owning adjncy[j] */
)
{
/* Given an adjacency list adjncy, find the partition number of each 
 * vertex in adjncy.  Return it in adjpart.
 */
ZOLTAN_COMM_OBJ *plan;
int i;
indextype start = vtxdist[zz->Proc];  /* First vertex on this processor */
indextype *recv_gno= NULL;
int *send_int=NULL;
int nrecv;
int tag = 24542;

  Zoltan_Comm_Create(&plan, (int)xadj[nvtx], adjproc, zz->Communicator, tag++, &nrecv);

  if (nrecv){
    recv_gno = (indextype *) ZOLTAN_MALLOC(nrecv * sizeof(indextype));
    send_int = (int *) ZOLTAN_MALLOC(nrecv * sizeof(int));
    if (!recv_gno || !send_int) {
      Zoltan_Comm_Destroy(&plan);
      ZOLTAN_FREE(&recv_gno);
      ZOLTAN_FREE(&send_int);
      return ZOLTAN_MEMERR;
    }
  }

  Zoltan_Comm_Do(plan, tag++, (char *) adjncy, sizeof(indextype), (char *) recv_gno);

  for (i = 0; i < nrecv; i++){
    send_int[i] = part[recv_gno[i] - start];
  }

  ZOLTAN_FREE(&recv_gno);

  Zoltan_Comm_Do_Reverse(plan, tag, (char *)send_int, sizeof(int), NULL, (char *) adjpart);

  ZOLTAN_FREE(&send_int);

  Zoltan_Comm_Destroy(&plan);

  return ZOLTAN_OK;
}
コード例 #2
0
int Zoltan_Input_HG_Free(ZHG *zhg)
{
  Zoltan_HG_HGraph_Free(&zhg->HG);

  ZOLTAN_FREE(&(zhg->objWeight));
  ZOLTAN_FREE(&(zhg->objGNO));
  ZOLTAN_FREE(&(zhg->objGID));
  ZOLTAN_FREE(&(zhg->objLID));
  ZOLTAN_FREE(&(zhg->numHEdges));
  ZOLTAN_FREE(&(zhg->coor));
  ZOLTAN_FREE(&(zhg->fixed));
  ZOLTAN_FREE(&(zhg->Input_Parts));
  ZOLTAN_FREE(&(zhg->Output_Parts));
  ZOLTAN_FREE(&(zhg->AppObjSizes));
  ZOLTAN_FREE(&(zhg->edgeGNO));
  ZOLTAN_FREE(&(zhg->Esize));
  ZOLTAN_FREE(&(zhg->Ewgt));
  ZOLTAN_FREE(&(zhg->pinGNO));
  ZOLTAN_FREE(&(zhg->Pin_Procs));
  ZOLTAN_FREE(&(zhg->Recv_GNOs));

  Zoltan_Comm_Destroy(&(zhg->VtxPlan));

  return ZOLTAN_OK;
}
コード例 #3
0
ファイル: phg_comm.c プロジェクト: askhl/octopus-dfrt2
/*
 * gather_by_list
 * input:   a list of processors, a message to be sent to these processors,
 *          and a receive buffer to get data back.
 *        
 *  output: concatenated messages from other processors in rbuff, with
 *          appropriate size info in rbuff_size.
 */
int gather_by_list(int procs_length, int   *procs,
                   int sbuff_size,   char  *sbuff, 
                   int *rbuff_size,  char **rbuff, MPI_Comm *comm)
{
    int i, receive_size, mtag, err, myProc;
    char            *send;
    ZOLTAN_COMM_OBJ *plan;
    static char     *yo = "gather_by_list";

    MPI_Comm_rank(*comm, &myProc);

    if (!(send  = (char*) ZOLTAN_MALLOC(procs_length * sizeof(int)))) {
        ZOLTAN_PRINT_ERROR(myProc, yo, "Insufficient memory");
        return ZOLTAN_MEMERR;
    }
    
    mtag = 0;
    for (i = 0; i < procs_length; ++i)
        ((int*)send)[i] = sbuff_size;
    
    /* create and resize communication plan */
    err = Zoltan_Comm_Create(&plan, procs_length, procs, *comm, 
            mtag++, &receive_size);
    if (err < 0)
        ZOLTAN_PRINT_ERROR(myProc, yo, "Zoltan_Comm_Create failed.");    
    err = Zoltan_Comm_Resize(plan, (int*)send, mtag++, rbuff_size);
    if (err < 0)
        ZOLTAN_PRINT_ERROR(myProc, yo, "Zoltan_Comm_Resize failed.");

    ZOLTAN_FREE(&send);
    
    /* allocate send and receive buffer */
    if ((!(*rbuff = (char*) ZOLTAN_MALLOC(*rbuff_size)) && (*rbuff_size != 0))
      ||!(  send  = (char*) ZOLTAN_MALLOC(sbuff_size * procs_length))) {
        printf("failed allocating %d + %d * %d bytes. . .", *rbuff_size, sbuff_size, procs_length);
        fflush(NULL);
        ZOLTAN_PRINT_ERROR(myProc, yo, "Insufficient Memory");
        return ZOLTAN_MEMERR;
    }

    /* copy message for each processor */
    for (i = 0; i < procs_length; ++i)
        memcpy(send + i * sbuff_size, sbuff, sbuff_size);
    
    /* finally, we can do the communication */
    err = Zoltan_Comm_Do(plan, mtag++, send, 1, *rbuff);
    if (err < 0)
        ZOLTAN_PRINT_ERROR(myProc, yo, "Zoltan_Comm_Do failed.");

    /* clean up */
    Zoltan_Comm_Destroy(&plan);
    ZOLTAN_FREE(&send);
    return err;
}
コード例 #4
0
ファイル: postprocessing.c プロジェクト: nschloe/seacas
static int
Zoltan_Postprocess_UnScatter_Graph (ZZ *zz,
				    ZOLTAN_Third_Graph *gr,
				    ZOLTAN_Third_Part *prt,
				    indextype **rank)
{
  static char * yo = "Zoltan_Postprocess_UnScatter_Graph";

  int ierr = ZOLTAN_FATAL;
  indextype *src;
  indextype *dst;

  if (gr->scatter >0){
    gr->num_obj = gr->num_obj_orig;
    if (*rank) {                        /* We have to project back rank */
      dst = (indextype*) ZOLTAN_MALLOC(gr->num_obj*sizeof(indextype));
      src = *rank;
    }
    else {
      dst = prt->part_orig;
      src = prt->part;
    }

    ierr = Zoltan_Comm_Do_Reverse(gr->comm_plan, TAG2, (char *) src,
				  sizeof(indextype), NULL, (char *) dst);
    if ((ierr == ZOLTAN_FATAL) || (ierr == ZOLTAN_MEMERR)){
      ZOLTAN_THIRD_ERROR(ierr, "Zoltan_Comm_Do_Reverse returned error.");
    }
    Zoltan_Comm_Destroy(&gr->comm_plan); /* Destroy the comm. plan */
    /* We don't need the partition array with the scattered distribution
     * any more */
    ZOLTAN_FREE(&src);
    if (prt) {
      /* part is now the new partition array under the original distribution */
      prt->part = prt->part_orig;
      prt->part_orig = NULL;
    }
    else {
      *rank = dst;
    }

  }
  return (ierr);
}
コード例 #5
0
ファイル: DD_Find.c プロジェクト: abhishek4747/pv-zoltan
int Zoltan_DD_Find (
 Zoltan_DD_Directory *dd, /* contains directory state information        */
 ZOLTAN_ID_PTR gid,       /* Incoming list of GIDs to get owners proc    */
 ZOLTAN_ID_PTR lid,       /* Outgoing corresponding list of LIDs         */
 char *data,              /* Outgoing optional corresponding user data   */
 int *partition,          /* Outgoing optional partition information     */
 int  count,              /* Count of GIDs in above list (in)            */
 int *owner)              /* Outgoing optional list of data owners       */
{
   ZOLTAN_COMM_OBJ *plan  = NULL;     /* efficient MPI communication     */
   char            *rbuff = NULL;     /* receive buffer                  */
   char            *rbufftmp = NULL;  /* pointer into receive buffer     */
   char            *sbuff = NULL;     /* send buffer                     */
   char            *sbufftmp = NULL;  /* pointer into send buffer        */
   int             *procs = NULL;     /* list of processors to contact   */
   DD_Find_Msg     *ptr   = NULL;
   int              i;
   int              nrec;             /* number of messages to receive   */
   int              err = ZOLTAN_OK;  /* return error condition          */
   int              errcount;         /* count of GIDs not found         */
   char            *yo = "Zoltan_DD_Find";


   /* input sanity check */
   if (dd == NULL || count < 0 || (gid == NULL && count > 0))  {
      ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo,
       "Invalid input argument");
      return ZOLTAN_FATAL;
   }
   if (dd->debug_level > 4)
      ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL);

   /* allocate memory for processors to contact for directory info */
   if (count)  {
      procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count);
      if (procs == NULL) {
         ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list");
         if (dd->debug_level > 4)
           ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
         return ZOLTAN_MEMERR;
      }
   }

   /* allocate memory for DD_Find_Msg send buffer */
   if (count)  {
      sbuff = (char*) ZOLTAN_CALLOC (count, dd->find_msg_size);
      if (sbuff == NULL)  {
         ZOLTAN_FREE (&procs);
         ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer");
         if (dd->debug_level > 4)
            ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
         return ZOLTAN_MEMERR;
      }
   }

   if (dd->debug_level > 6)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs");

   /* for each GID, fill DD_Find_Msg buffer and contact list */
   sbufftmp = sbuff;
   for (i = 0; i < count; i++)  {
      procs[i] = dd->hash (gid + i*dd->gid_length, dd->gid_length, dd->nproc,
                           dd->hashdata, dd->hashfn);
      ptr      = (DD_Find_Msg*) sbufftmp;
      sbufftmp += dd->find_msg_size;

      ptr->index = i;
      ptr->proc  = procs[i];
      ZOLTAN_SET_ID (dd->gid_length, ptr->id, gid + i*dd->gid_length);
   }
   if (dd->debug_level > 6)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill");

   /* create efficient communication plan */
   err = Zoltan_Comm_Create (&plan, count, procs, dd->comm,
    ZOLTAN_DD_FIND_MSG_TAG, &nrec);
   if (dd->debug_level > 6)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create");
   if (err != ZOLTAN_OK)
      goto fini;

   /* allocate receive buffer */
   if (nrec)  {
      rbuff = (char*) ZOLTAN_MALLOC ((size_t)nrec*(size_t)(dd->find_msg_size));
      if (rbuff == NULL)  {
         err = ZOLTAN_MEMERR;
         goto fini;
      }
   }

   /* send out find messages across entire system */
   err = Zoltan_Comm_Do (plan, ZOLTAN_DD_FIND_MSG_TAG+1, sbuff,
    dd->find_msg_size, rbuff);
   if (err != ZOLTAN_OK)
      goto fini;

   if (dd->debug_level > 6)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do");

   /* get find messages directed to me, fill in return information */
   errcount = 0;
   rbufftmp = rbuff;
   for (i = 0; i < nrec; i++)  {
      ptr = (DD_Find_Msg*) rbufftmp;
      rbufftmp += dd->find_msg_size;
      err = DD_Find_Local (dd, ptr->id, ptr->id,
                           (char *)(ptr->id + dd->max_id_length),
                           &ptr->partition, &ptr->proc);
      if (err == ZOLTAN_WARN)
          ++errcount;
   }
   if (dd->debug_level > 6)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill in return info");

   /* send return information back to requester */
   err = Zoltan_Comm_Do_Reverse(plan, ZOLTAN_DD_FIND_MSG_TAG+2, rbuff,
    dd->find_msg_size, NULL, sbuff);
   if (err != ZOLTAN_OK)
      goto fini;
   if (dd->debug_level > 6)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Reverse");

   /* fill in user supplied lists with returned information */
   sbufftmp = sbuff;
   for (i = 0; i < count; i++) {
      ptr = (DD_Find_Msg*) sbufftmp;
      sbufftmp += dd->find_msg_size;

      if (owner)
         owner[ptr->index] = ptr->proc;
      if (partition)
         partition[ptr->index] = ptr->partition ;
      if (lid)
         ZOLTAN_SET_ID(dd->lid_length,lid+ptr->index*dd->lid_length,ptr->id);
      if (data)
         memcpy(data + (size_t)(ptr->index) * (size_t)(dd->user_data_length),
                ptr->id + dd->max_id_length, dd->user_data_length);
   }
   if (dd->debug_level > 6)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill return lists");
/*    err = ZOLTAN_OK;     */

   MPI_Allreduce(&errcount, &err, 1, MPI_INT, MPI_SUM, dd->comm);
   err = (err) ? ZOLTAN_WARN : ZOLTAN_OK;

   /* if at least one GID was not found, potentially notify caller of error */
   if (dd->debug_level > 0)  {
      char str[100];      /* diagnostic message string */
      sprintf (str, "Processed %d GIDs, GIDs not found: %d", count, errcount);
      ZOLTAN_PRINT_INFO (dd->my_proc, yo, str);
   }

fini:
   ZOLTAN_FREE (&sbuff);
   ZOLTAN_FREE (&rbuff);
   ZOLTAN_FREE (&procs) ;
   Zoltan_Comm_Destroy (&plan);

   if (dd->debug_level > 4)
      ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
   return err;
}
コード例 #6
0
static int read_comm_map_info(int pexoid, int Proc, PROB_INFO_PTR prob,
                              MESH_INFO_PTR mesh)
{
  /* Local declarations. */
  char *yo = "read_comm_map_info";
  int  ielem, imap, loc_elem, iblk, max_len, offset, index;
  int  nnodei, nnodeb, nnodee, nelemi, nelemb, nncmap;
  int *int_elem, *bor_elem;
  int *proc_ids;
  int *gids, *my_procs, *recv_procs;
  int  ierr, nrecv;
  int  msg = 200;
  int  sid;
  ELEM_INFO_PTR elements = mesh->elements;
  ZOLTAN_COMM_OBJ *comm_obj;

  E_Type etype;

/***************************** BEGIN EXECUTION ******************************/

  DEBUG_TRACE_START(Proc, yo);

  if (ne_get_loadbal_param(pexoid, &nnodei, &nnodeb, &nnodee,
                     &nelemi, &nelemb, &nncmap, &(mesh->necmap), Proc) < 0) {
    Gen_Error(0, "fatal: Error returned from ne_get_loadbal_param");
    return 0;
  }

  /*
   * get the list of the border elements in order to set
   * the border flag in the element structures
   */
  int_elem = (int *) malloc ((nelemi + nelemb) * sizeof(int));
  if (!int_elem) {
    Gen_Error(0, "fatal: insufficient memory");
    return 0;
  }
  bor_elem = int_elem + nelemi;

  if (ne_get_elem_map(pexoid, int_elem, bor_elem, Proc) < 0) {
    Gen_Error(0, "fatal: Error returned from ne_get_elem_map");
    return 0;
  }

  for (ielem = 0; ielem < nelemb; ielem++) {
    elements[bor_elem[ielem]-1].border = 1;
  }

  free(int_elem);

  /*
   * For now, only get the elemental communication maps,
   * since, in the driver, elements are only considered
   * adjacent if they share a face (same definition used
   * in element communication maps). Eventually, the ability
   * to consider elements that are connected by any nodes
   * adjacent will have to be added. When that happens,
   * the nodal communication maps will be needed.
   */
  mesh->ecmap_cnt = (int *) malloc (mesh->necmap * sizeof(int));
  mesh->ecmap_id = (int *) malloc(mesh->necmap * sizeof(int));
  if (!mesh->ecmap_cnt || !mesh->ecmap_id) {
    Gen_Error(0, "fatal: insufficient memory");
    return 0;
  }

  if (ne_get_cmap_params(pexoid, NULL, NULL, mesh->ecmap_id, 
                         mesh->ecmap_cnt, Proc) < 0) {
    Gen_Error(0, "fatal: Error returned from ne_get_cmap_params");
    return 0;
  }

  max_len = 0;
  for (imap = 0; imap < mesh->necmap; imap++)
    max_len += mesh->ecmap_cnt[imap];

  proc_ids = (int *) malloc(4 * max_len * sizeof(int));
  gids = proc_ids + max_len;
  my_procs = gids + max_len;
  recv_procs = my_procs + max_len;
  mesh->ecmap_elemids = (int *) malloc(max_len * sizeof(int));
  mesh->ecmap_sideids = (int *) malloc(max_len * sizeof(int));
  mesh->ecmap_neighids = (int *) malloc(max_len * sizeof(int));
  if (!mesh->ecmap_elemids || !mesh->ecmap_sideids || !mesh->ecmap_neighids) {
    Gen_Error(0, "fatal: insufficient memory");
    return 0;
  }

  offset = 0;
  for (imap = 0; imap < mesh->necmap; imap++) {

    if(ne_get_elem_cmap(pexoid, mesh->ecmap_id[imap],
                        &(mesh->ecmap_elemids[offset]),
                        &(mesh->ecmap_sideids[offset]), 
                        &(proc_ids[offset]), Proc) < 0) {
      Gen_Error(0, "fatal: Error returned from ne_get_elem_cmap");
      return 0;
    }
    offset += mesh->ecmap_cnt[imap];
  } /* End: "for (imap = 0; imap < mesh->necmap; imap++)" */

  /*
   * Decrement the ecmap_elemids by one for zero-based local numbering.
   * Convert the element ids to global ids to send to
   * the neighboring processor.
   */
  for (ielem = 0; ielem < max_len; ielem++) {
    mesh->ecmap_elemids[ielem]--;
    gids[ielem] = elements[mesh->ecmap_elemids[ielem]].globalID;
    my_procs[ielem] = Proc;
  }
  /*
   * Now communicate with other processor to get global IDs
   * for the adjacent elements in this communication map.
   */

  ierr = Zoltan_Comm_Create(&comm_obj, max_len, proc_ids, MPI_COMM_WORLD, 
                            msg, &nrecv);
  if (ierr != ZOLTAN_OK) {
    Gen_Error(0, "fatal: Error returned from Zoltan_Comm_Create");
    return 0;
  }
  if (nrecv != max_len) {
    /* Sanity check; this should never happen. */
    Gen_Error(0, "fatal: Error returned from Zoltan_Comm_Create");
    return 0;
  }

  /* Exchange ids to neighbors.  
   * Assuming messages will be stored in order of processor number in 
   * ecmap_neighids. 
   */
  ierr = Zoltan_Comm_Do(comm_obj, msg+1, (char *) gids, sizeof(int), 
                        (char *) (mesh->ecmap_neighids));

  /* Exchange sanity check information. 
   * Allows to check assumption that messages are stored in order of
   * processor number.
   */
  if (ierr == ZOLTAN_OK)
    ierr = Zoltan_Comm_Do(comm_obj, msg+2, (char *) my_procs, sizeof(int), 
                          (char *) recv_procs);

  if (ierr != ZOLTAN_OK) {
    Gen_Error(0, "fatal: Error returned from Zoltan_Comm_Do");
    return 0;
  }

  ierr = Zoltan_Comm_Destroy(&comm_obj);

  /* Sanity check: messages stored in order of processor number. */
  for (ielem = 0; ielem < max_len; ielem++) {
    if (proc_ids[ielem] != recv_procs[ielem]) {
      Gen_Error(0, "fatal: Sanity check failed; assumption wrong");
      return 0;
    }
  }

  /* now process all of the element ids that have been received */
  offset = 0;
  for (imap = 0; imap < mesh->necmap; imap++) {
    for (ielem = 0; ielem < mesh->ecmap_cnt[imap]; ielem++) {
      index = ielem + offset;
      /* translate from element id in the communication map to local elem id */
      loc_elem = mesh->ecmap_elemids[index];
      iblk = elements[loc_elem].elem_blk;
      etype = (E_Type) (mesh->eb_etypes[iblk]);

      (elements[loc_elem].nadj)++;
      if(elements[loc_elem].nadj > elements[loc_elem].adj_len) {
        /* Shouldn't happen as long as only side adjacencies are used. */
        /* adj_len == number of sides.                                 */
        /* Space should already be allocated for the adjacencies read  */
        /* from the communication maps.                                */
        Gen_Error(0, "fatal: Number of adj greater than adj_len");
        return 0;
      }

      /* Store adjacency info in the adj entry corresponding to this side. */
      sid = mesh->ecmap_sideids[index] - 1;
      elements[loc_elem].adj[sid] = mesh->ecmap_neighids[index];
      elements[loc_elem].adj_proc[sid] = mesh->ecmap_id[imap];
      elements[loc_elem].edge_wgt[sid] =
             (float) get_elem_info(NSNODES, etype, mesh->ecmap_sideids[index]);

    } /* End: "for (ielem = 0; ielem < mesh->ecmap_cnt[imap]; ielem++)" */
    offset += mesh->ecmap_cnt[imap];
  } /* End: "for for (imap = 0; imap < mesh->necmap; imap++)" */
  
  free (proc_ids);

  DEBUG_TRACE_END(Proc, yo);
  return 1;
}
コード例 #7
0
ファイル: migoct.c プロジェクト: haripandey/trilinos
static int Zoltan_Oct_build_global_rootlist(ZZ *zz,Migrate_msg  **ret_rmsg, int *size) {
  int j, k = 0;
  int *despid = NULL;
  int nroots, nreceives;
  pRList  RootList;                  /* list of the local roots */
  pOctant RootOct;
  Migrate_msg *snd_rmsg = NULL;
  Migrate_msg *rcv_rmsg = NULL;
  OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
/*Map *array = OCT_info->map;*/
  ZOLTAN_COMM_OBJ *comm_plan;                /* Object returned by communication routines */

  int ierr = ZOLTAN_OK;
  char *yo = "Zoltan_Oct_build_global_rootlist";
 
  nroots = RL_numRootOctants(Zoltan_Oct_POct_localroots(OCT_info));


  if (nroots > 0) {  /* KDDKDD -- Added test to prevent departure before comm */
  if((despid = (int *) ZOLTAN_MALLOC((zz->Num_Proc)*nroots * sizeof(int))) == NULL) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    return ZOLTAN_MEMERR;
  }

  if((snd_rmsg = (Migrate_msg *) ZOLTAN_MALLOC((zz->Num_Proc)*nroots * sizeof(Migrate_msg))) == NULL) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&despid);
    return ZOLTAN_MEMERR;
  }
  
  k = 0;
  for (j=0; j<zz->Num_Proc; j++) {
    RootList = Zoltan_Oct_POct_localroots(OCT_info);
    while((RootOct = RL_nextRootOctant(&RootList))) {	
 /*      if(array[Zoltan_Oct_mapidx(RootOct)].npid > 0) { */
	FILLMIGRATEMSG(RootOct, RootOct, snd_rmsg[k], zz->Proc);
	despid[k] = j;
	k++;
/*       } */
    }
  }
  }  /* KDDKDD */
  
  ierr = Zoltan_Comm_Create(&comm_plan, k, despid, zz->Communicator,
			RootListCommCreate, &nreceives);

  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&despid);
    ZOLTAN_FREE(&snd_rmsg);
    return (ierr);
  }

  if (nreceives > 0) {
  if((rcv_rmsg = (Migrate_msg *) ZOLTAN_MALLOC(nreceives * sizeof(Migrate_msg))) == NULL) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&despid);
    ZOLTAN_FREE(&snd_rmsg);
    return ZOLTAN_MEMERR;
  }
  }

  

  ierr = Zoltan_Comm_Do(comm_plan, RootListCommDo, (char *) snd_rmsg, sizeof(Migrate_msg), (char *) rcv_rmsg);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&despid);
    ZOLTAN_FREE(&snd_rmsg);
    ZOLTAN_FREE(&rcv_rmsg);
    return (ierr);
  }

  ZOLTAN_FREE(&despid);
  ZOLTAN_FREE(&snd_rmsg);


  ierr = Zoltan_Comm_Destroy(&comm_plan);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    return (ierr);
  }

  *ret_rmsg = rcv_rmsg;
  *size = nreceives;

  return ierr;
}
コード例 #8
0
ファイル: scatter_graph.c プロジェクト: agrippa/Trilinos
int Zoltan_Scatter_Graph(
  indextype **vtxdist,
  indextype **xadj,
  indextype **adjncy,
  weighttype **vwgt,
  indextype **vsize,
  weighttype **adjwgt,
  realtype   **xyz,
  int     ndims,		/* # dimensions of xyz geometry data */
  int     vwgt_dim,
  ZZ      *zz,
  ZOLTAN_COMM_OBJ **plan
)
{
  static char *yo = "Zoltan_Scatter_Graph";
  char     msg[256];
  indextype *old_vtxdist, *old_adjncy;
  indextype *old_xadj;
  indextype *old_vsize;
  weighttype *old_vwgt, *old_adjwgt;
  realtype   *old_xyz;
  int *ptr, *proclist = NULL, *proclist2 = NULL;
  int i, j, num_obj, old_num_obj, num_edges, nrecv;
  int use_graph;	/* do we use graph data, or only the geometry? */
  int use_vsize;	/* do we use the vsize array? */
  int ewgt_dim= zz->Edge_Weight_Dim;
  ZOLTAN_COMM_OBJ *plan2;

  ZOLTAN_TRACE_ENTER(zz, yo);

  /* Save pointers to "old" data distribution */
  old_adjncy = NULL;
  old_xadj = NULL;
  old_vwgt = old_adjwgt = NULL;
  old_vsize = NULL;
  old_xyz = NULL;

  old_vtxdist = *vtxdist;

  if (xadj)
    old_xadj = *xadj;
  if (adjncy)
    old_adjncy = *adjncy;
  if (vwgt)
    old_vwgt = *vwgt;
  if (vsize)
    old_vsize = *vsize;
  if (adjwgt)
    old_adjwgt = *adjwgt;
  if (xyz)
    old_xyz = *xyz;

  old_num_obj = (int)(old_vtxdist[zz->Proc+1] - old_vtxdist[zz->Proc]); 
  if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) 
    printf("[%1d] Debug: Old number of objects = %d\n", zz->Proc, old_num_obj);

  /* Compute new distribution, *vtxdist */
  (*vtxdist) = (indextype *)ZOLTAN_MALLOC((zz->Num_Proc+1)* sizeof(indextype));
  for (i=0; i<=zz->Num_Proc; i++){
    (*vtxdist)[i] = (i*old_vtxdist[zz->Num_Proc])/zz->Num_Proc;
  }

  /* Check if any proc has graph data */
  i = (old_xadj != NULL);
  MPI_Allreduce(&i, &use_graph, 1, MPI_INT, MPI_LOR, zz->Communicator);
  j = (old_vsize != NULL);
  MPI_Allreduce(&j, &use_vsize, 1, MPI_INT, MPI_LOR, zz->Communicator);
  if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) 
    printf("[%1d] Debug: use_graph = %1d, use_vsize = %1d\n", zz->Proc, 
          use_graph, use_vsize);

  /* Reset all data pointers to NULL for now */
  *xadj = NULL;
  *adjncy = NULL;
  *vwgt = *adjwgt = NULL;
  *xyz = NULL;
  if (use_vsize) *vsize = NULL;

  /* Convert the xdj array so that it contains the degree of each vertex */
  if (use_graph){
    for (i=0; i<old_num_obj; i++){
      old_xadj[i] = old_xadj[i+1] - old_xadj[i];
    }
  }

  /* Allocate new space for vertex data */
  num_obj = (int)((*vtxdist)[zz->Proc+1] - (*vtxdist)[zz->Proc]);
  if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) 
    printf("[%1d] Debug: New number of objects = %d\n", zz->Proc, num_obj);
  if (use_graph)
    *xadj = (indextype *) ZOLTAN_MALLOC((num_obj+1)*sizeof(indextype));
  if (vwgt_dim)
    *vwgt = (weighttype *) ZOLTAN_MALLOC(vwgt_dim*num_obj*sizeof(weighttype));
  if (use_vsize)
    *vsize = (indextype *) ZOLTAN_MALLOC(num_obj*sizeof(indextype));
  if (ndims)
    *xyz = (realtype *) ZOLTAN_MALLOC(ndims*num_obj*sizeof(realtype));

  if (old_num_obj > 0) {
    /* Set up the communication plan for the vertex data */
    proclist = (int *) ZOLTAN_MALLOC(old_num_obj * sizeof(int));
    /* Let j be the new owner of vertex old_vtxdist[zz->Proc]+i */
    j = 0;
    while (old_vtxdist[zz->Proc] >= (*vtxdist)[j+1]) j++;
    for (i=0; i<old_num_obj; i++){
      if (old_vtxdist[zz->Proc]+i >= (*vtxdist)[j+1]) j++;
      proclist[i] = j;
    }
  }

  Zoltan_Comm_Create(plan, old_num_obj, proclist, zz->Communicator, TAG1, &nrecv);

  if (nrecv != num_obj){
    sprintf(msg,"Proc %d received %d object but expected %d.",
      zz->Proc, nrecv, num_obj);
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
    /* Free data */
    ZOLTAN_FREE(&proclist);
    ZOLTAN_TRACE_EXIT(zz, yo);
    return ZOLTAN_FATAL;
  }

  /* Do the communication. To save memory, we do not pack all the data into
   * a buffer, but send directly from the old arrays to the new arrays. 
   * We use the vertex communication plan for all the vertex-based arrays
   * and the edge communication plan for all the edge-based arrays.
   */

  if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) 
    printf("[%1d] Debug: Starting vertex-based communication.\n", zz->Proc);

  if (use_graph){
    Zoltan_Comm_Do( *plan, TAG2, (char *) old_xadj, sizeof(indextype), (char *) *xadj);
  }
  if (vwgt_dim){
    Zoltan_Comm_Do( *plan, TAG3, (char *) old_vwgt, vwgt_dim*sizeof(weighttype), (char *) *vwgt);
  }
  if (use_vsize){
    Zoltan_Comm_Do( *plan, TAG4, (char *) old_vsize, sizeof(indextype), (char *) *vsize);
  }
  if (ndims){
    Zoltan_Comm_Do( *plan, TAG5, (char *) old_xyz, ndims*sizeof(realtype), (char *) *xyz);
  }
  if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) 
    printf("[%1d] Debug: Finished vertex-based communication.\n", zz->Proc);

  if (use_graph){

    /* Rebuild xadj from degrees */
    for (i=1; i<num_obj; i++)
      (*xadj)[i] += (*xadj)[i-1];
    for (i=num_obj; i>0; i--)
      (*xadj)[i] = (*xadj)[i-1];
    (*xadj)[0] = 0;
  
    /* Allocate space for new edge data structures */
    num_edges = (*xadj)[num_obj];
    *adjncy = (indextype *) ZOLTAN_MALLOC(num_edges*sizeof(indextype));
  
    if (ewgt_dim)
      *adjwgt = (weighttype *) ZOLTAN_MALLOC(ewgt_dim*num_edges*sizeof(weighttype));
  
    /* Set up the communication plan for the edge data. */
    ptr = proclist2 = (int *) ZOLTAN_MALLOC(old_xadj[old_num_obj] * sizeof(int));
    for (i=0; i<old_num_obj; i++)
      for (j=0; j<old_xadj[i]; j++)
        *ptr++ = proclist[i];
    if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
      printf("[%1d] Debug: Allocated proclist of length " TPL_IDX_SPEC " for edges.\n", 
             zz->Proc, old_xadj[old_num_obj]);
    }

    Zoltan_Comm_Create(&plan2, (int)old_xadj[old_num_obj], proclist2, zz->Communicator, 
                   TAG1, &nrecv);
  
    if (nrecv != num_edges){
      sprintf(msg,"Proc %d received %d edges but expected %d.",
        zz->Proc, nrecv, num_edges);
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
      /* Free data */
      ZOLTAN_FREE(&proclist);
      ZOLTAN_FREE(&proclist2);
      ZOLTAN_FREE(&old_vtxdist);
      ZOLTAN_FREE(&old_xadj);
      ZOLTAN_FREE(&old_adjncy);
      ZOLTAN_FREE(&old_vwgt);
      ZOLTAN_FREE(&old_vsize);
      ZOLTAN_FREE(&old_adjwgt);
      ZOLTAN_FREE(&old_xyz);
      ZOLTAN_TRACE_EXIT(zz, yo);
      return ZOLTAN_FATAL;
    }
  
    if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) 
      printf("[%1d] Debug: Starting edge-based communication.\n", zz->Proc);
  
    /* Do the communication. */
    Zoltan_Comm_Do( plan2, TAG2, (char *) old_adjncy, sizeof(indextype), (char *) *adjncy);
    if (ewgt_dim){
      Zoltan_Comm_Do( plan2, TAG3, (char *) old_adjwgt, ewgt_dim*sizeof(weighttype), (char *) *adjwgt);
    }
  
    if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) 
      printf("[%1d] Debug: Finished edge-based communication.\n", zz->Proc);
  
    /* Free the comm. plan for edge data */
    Zoltan_Comm_Destroy(&plan2);

  } /* end of use_graph */

  /* Free data structures */
  ZOLTAN_FREE(&proclist);
  ZOLTAN_FREE(&proclist2);
  ZOLTAN_FREE(&old_vtxdist);
  ZOLTAN_FREE(&old_xadj);
  ZOLTAN_FREE(&old_adjncy);
  ZOLTAN_FREE(&old_vwgt);
  ZOLTAN_FREE(&old_vsize);
  ZOLTAN_FREE(&old_adjwgt);
  ZOLTAN_FREE(&old_xyz);

  ZOLTAN_TRACE_EXIT(zz, yo);
  return ZOLTAN_OK;
}
コード例 #9
0
static int Zoltan_Oct_migreg_migrate_regions(ZZ *zz, Region *regions, 
                              ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids,
                              int *npids, int nregions, int *c2) 
{
  char *yo = "Zoltan_Oct_migreg_migrate_regions";
  int i;                         /* index counter */
  int ierr = ZOLTAN_OK;
  int n_import;
  ZOLTAN_COMM_OBJ *comm_plan;           /* Object returned by communication routines */
  Region *import_objs = NULL;    /* Array of import objects used to request 
				    the objs from other processors. */
  ZOLTAN_ID_PTR import_gids = NULL;  /* Array of global IDs of import_objs. */
  ZOLTAN_ID_PTR import_lids = NULL;  /* Array of local IDs of import_objs. */
  int num_gid_entries = zz->Num_GID;
  int num_lid_entries = zz->Num_LID;

  ierr = Zoltan_Comm_Create(&comm_plan, nregions, npids, zz->Communicator, 
                        MIGMIGREGCommCreate, &n_import);
  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Create");
    ZOLTAN_TRACE_EXIT(zz, yo);
    return (ierr);
  }
  *c2 = n_import;
  if (n_import > 0) {
    import_objs = (Region *) ZOLTAN_MALLOC(n_import * sizeof(Region));
    import_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, n_import);
    import_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, n_import);

    if (!import_objs || !import_gids || (num_lid_entries && !import_lids)) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
      ZOLTAN_TRACE_EXIT(zz, yo);
      return ZOLTAN_MEMERR;
    }
  }
  ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo, (char *) regions, sizeof(Region), 
                   (char *) import_objs);
  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&import_objs);
    ZOLTAN_FREE(&import_gids);
    ZOLTAN_FREE(&import_lids);
    return (ierr);
  }

  ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-1, (char *) gids, 
                    sizeof(ZOLTAN_ID_TYPE)*num_gid_entries, (char *) import_gids);
  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&import_objs);
    ZOLTAN_FREE(&import_gids);
    ZOLTAN_FREE(&import_lids);
    return (ierr);
  }

  if (num_lid_entries > 0) {
    ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-2, (char *) lids, 
                      sizeof(ZOLTAN_ID_TYPE)*num_lid_entries, (char *) import_lids);
    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
      ZOLTAN_TRACE_EXIT(zz, yo);
      ZOLTAN_FREE(&import_objs);
      ZOLTAN_FREE(&import_gids);
      ZOLTAN_FREE(&import_lids);
      return (ierr);
    }
  }
  for (i=0; i<n_import; i++) {
    import_objs[i].Global_ID = &(import_gids[i*num_gid_entries]);
    import_objs[i].Local_ID = (num_lid_entries 
                                 ? &(import_lids[i*num_lid_entries]) 
                                 : NULL);
    Zoltan_Oct_insert_orphan(zz, import_objs[i]);
  }
  
  ZOLTAN_FREE(&import_objs);
  ZOLTAN_FREE(&import_gids);
  ZOLTAN_FREE(&import_lids);

  ierr = Zoltan_Comm_Destroy(&comm_plan);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    return (ierr);
  }
  return ierr;
}
コード例 #10
0
ファイル: migoct.c プロジェクト: haripandey/trilinos
static int Zoltan_Oct_Final_Migration(
ZZ *zz,
pOctant *octs,      /* octs[nocts]    */
int *newpids,       /* newpids[nocts] */
pOctant *newocts,   /* newocts[nocts] */
int nocts,          /* number of octants leaving this processor */
int nrecocts)       /* number of octants received in this processor */
{
  int i;
  Migrate_msg *msnd = NULL, *mrcv = NULL;
  int remotecount;
  int nsends;
  int nreceives;
  int *despid = NULL;
  int ierr = ZOLTAN_OK;
  ZOLTAN_COMM_OBJ *comm_plan;           /* Object returned by communication routines */
  char *yo = "Zoltan_Oct_Final_Migration";
  OCT_Global_Info *OCT_info = (OCT_Global_Info *) zz->LB.Data_Structure;

  /* count number of sends */
  nsends=0;
  for (i=0; i<nocts; i++)              
    if (newpids[i]!=zz->Proc)
      nsends++;

  if((despid = (int *) ZOLTAN_MALLOC((nocts+10) * sizeof(int))) == NULL) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    return ZOLTAN_MEMERR;
  }

  if((msnd   = (Migrate_msg *) ZOLTAN_MALLOC((nocts+10) * sizeof(Migrate_msg))) == NULL) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&despid);
    return ZOLTAN_MEMERR;
  }

  remotecount = 0;
  for (i=0; i<nocts; i++)                          /* Send and free */
    if (newpids[i]!=zz->Proc)
      { 
	FILLMIGRATEMSG(octs[i], newocts[i], msnd[remotecount], zz->Proc); /* bug */
	despid[remotecount++] = newpids[i];
	Zoltan_Oct_clearRegions(octs[i]);
        /* KDDKDDFREE Change oct to &oct to allow NULL from ZOLTAN_FREE 
         * KDDKDDFREE to propagate back. */
	Zoltan_Oct_POct_free(OCT_info, &(octs[i]));
      }

  ierr = Zoltan_Comm_Create(&comm_plan, remotecount, despid, zz->Communicator,
			MigFinCommCreate, &nreceives);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&despid);
    ZOLTAN_FREE(&msnd);
    return (ierr);
  }

  if((mrcv = (Migrate_msg *) ZOLTAN_MALLOC((nreceives+10) * sizeof(Migrate_msg))) == NULL) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&despid);
    ZOLTAN_FREE(&msnd);
    return ZOLTAN_MEMERR;
  }

  ierr = Zoltan_Comm_Do(comm_plan, MigFinCommDo, (char *) msnd,
		    sizeof(Migrate_msg), (char *) mrcv);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&despid);
    ZOLTAN_FREE(&msnd);
    ZOLTAN_FREE(&mrcv);
    return (ierr);
  }

  for (i=0; i<nreceives; i++) {                     /* Receive new parocts */
/*     Zoltan_Oct_setID(mrcv[i].ptr,mrcv[i].id); */
/*     Zoltan_Oct_POct_setparent(OCT_info, mrcv[i].ptr, mrcv[i].parent, mrcv[i].ppid);  */
    SETOCTFROMMIGRATEMSG(OCT_info, mrcv[i]); 
/*     Zoltan_Oct_setchildnum(mrcv[i].ptr,mrcv[i].childnum);  */
/*     Zoltan_Oct_setchildren(mrcv[i].ptr, mrcv[i].children, mrcv[i].cpids);  */
/*     Zoltan_Oct_setbounds(mrcv[i].ptr, mrcv[i].min, mrcv[i].max);   */
/*     Zoltan_Oct_setDir(mrcv[i].ptr,mrcv[i].dir); */
/*     Zoltan_Oct_setMapIdx(mrcv[i].ptr,mrcv[i].mapidx); */
  }

  ierr = Zoltan_Comm_Destroy(&comm_plan);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&despid);
    ZOLTAN_FREE(&msnd);
    ZOLTAN_FREE(&mrcv);
    return (ierr);
  }

  ZOLTAN_FREE(&despid);
  ZOLTAN_FREE(&msnd);
  ZOLTAN_FREE(&mrcv);

  return ierr;
}
コード例 #11
0
ファイル: DD_Update.c プロジェクト: agrippa/Trilinos
int Zoltan_DD_Update (
 Zoltan_DD_Directory *dd,  /* directory state information              */
 ZOLTAN_ID_PTR gid,        /* Incoming list of GIDs to update          */
 ZOLTAN_ID_PTR lid,        /* Incoming corresponding LIDs (optional)   */
 char *user,               /* Incoming list of user data (optional)    */
 int *partition,           /* Optional, grouping of GIDs to partitions */
 int count)                /* Number of GIDs in update list            */
{
   int             *procs = NULL;   /* list of processors to contact   */
   DD_Update_Msg   *ptr   = NULL;
   ZOLTAN_COMM_OBJ *plan  = NULL;   /* for efficient MPI communication */
   char            *sbuff = NULL;   /* send buffer                     */
   char            *sbufftmp = NULL;/* pointer into send buffer        */
   char            *rbuff = NULL;   /* receive buffer                  */
   char            *rbufftmp = NULL;/* pointer into receive buffer     */
   int              nrec = 0;       /* number of receives to expect    */
   int              i;
   int              err;
   int              errcount = 0;   /* count of GIDs not found, added  */
   char             str[100];       /* build error message string      */
   char            *yo = "Zoltan_DD_Update";


   /* input sanity checking */
   if (dd == NULL || count < 0 || (gid == NULL && count > 0))  {
      ZOLTAN_PRINT_ERROR ((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc), yo,
       "Invalid input argument");
      return ZOLTAN_FATAL;
   }
   if (dd->debug_level > 4)
      ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL);

   /* part of initializing the error checking process             */
   /* for each linked list head, walk its list resetting errcheck */
   if (dd->debug_level)
      for (i = 0; i < dd->table_length; i++) {
         DD_NodeIdx nodeidx;
         for (nodeidx = dd->table[i]; nodeidx != -1;
              nodeidx = dd->nodelist[nodeidx].next)
            dd->nodelist[nodeidx].errcheck = ZOLTAN_DD_NO_PROC;
      }

   if (dd->debug_level > 6)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After reset errcheck");

   /* allocate memory for list of processors to contact */
   if (count) {
      procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count);
      if (procs == NULL)  {
         ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list");
         err = ZOLTAN_MEMERR;
         goto fini;
      }
   }

   /* allocate memory for DD_Update_Msg send buffer */
   if (count)  {
      sbuff = (char*) ZOLTAN_CALLOC (count, dd->update_msg_size);
      if (sbuff == NULL)  {
         ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer");
         err = ZOLTAN_MEMERR;
         goto fini;
      }
   }

   if (dd->debug_level > 6)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs");

   /* for each GID given, fill in contact list and then message structure */
   sbufftmp = sbuff;
   for (i = 0; i < count; i++)  {
      procs[i] = dd->hash(gid + i*dd->gid_length, dd->gid_length, dd->nproc,
                          dd->hashdata, dd->hashfn);
      ptr      = (DD_Update_Msg*) sbufftmp;
      sbufftmp += dd->update_msg_size;

      ptr->lid_flag       = (lid)  ? 1 : 0;
      ptr->user_flag      = (user) ? 1 : 0;
      ptr->partition_flag = (partition) ? 1 : 0;
      ptr->partition      = (partition) ? *(partition + i) :  -1;
      ptr->owner          = dd->my_proc;

      ZOLTAN_SET_ID (dd->gid_length, ptr->gid, gid + i * dd->gid_length);
      if (lid) {
         ZOLTAN_SET_ID(dd->lid_length, ptr->gid + dd->gid_length,
                       lid + i * dd->lid_length);
      }
      else {
         memset(ptr->gid + dd->gid_length, 0, dd->lid_length);
      }
      if (user) {
         memcpy(ptr->gid + (dd->gid_length + dd->lid_length),
                user + (size_t)i * (size_t)(dd->user_data_length),
                dd->user_data_length);
      }
      else {
         memset(ptr->gid + (dd->gid_length + dd->lid_length), 0,
                dd->user_data_length);
      }
   }

   if (dd->debug_level > 6)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill contact list");

   /* now create efficient communication plan */
   err = Zoltan_Comm_Create (&plan, count, procs, dd->comm,
    ZOLTAN_DD_UPDATE_MSG_TAG, &nrec);
   if (err != ZOLTAN_OK)  {
      ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Create error");
      goto fini;
   }

   if (dd->debug_level > 6)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create");

   /* If dd has no nodes allocated (e.g., first call to DD_Update; 
    * create the nodelist and freelist 
    */
   if (nrec && dd->nodelistlen == 0) {
      DD_Memory_Alloc_Nodelist(dd, (DD_NodeIdx) nrec, 0.); 
                               /* TODO Add overalloc parameter */
   }

   /* allocate receive buffer for nrec DD_Update_Msg structures */
   if (nrec)  {
      rbuff = (char*)ZOLTAN_MALLOC((size_t)nrec*(size_t)(dd->update_msg_size));
      if (rbuff == NULL)  {
         ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Receive buffer malloc failed");
         err = ZOLTAN_MEMERR;
         goto fini;
      }
   }

   /* send my update messages & receive updates directed to me */
   err = Zoltan_Comm_Do (plan, ZOLTAN_DD_UPDATE_MSG_TAG+1, sbuff,
    dd->update_msg_size, rbuff);
   if (err != ZOLTAN_OK)  {
      ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Do error");
      goto fini;
   }

   if (dd->debug_level > 6)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do");

   /* for each message rec'd, update local directory information */
   errcount = 0;
   rbufftmp = rbuff;
   for (i = 0; i < nrec; i++)  {
      ptr = (DD_Update_Msg *) rbufftmp;
      rbufftmp += dd->update_msg_size;

      err = DD_Update_Local (dd, ptr->gid,
       (ptr->lid_flag)  ? (ptr->gid + dd->gid_length) : NULL,
       (char*)((ptr->user_flag)?(ptr->gid+(dd->gid_length+dd->lid_length)):NULL),
       (ptr->partition_flag) ? (ptr->partition) : -1,  /* illegal partition */
       ptr->owner);

      if (err != ZOLTAN_OK)
         ++errcount;
   }
   if (dd->debug_level > 6)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Local update");

   err = ZOLTAN_OK;
   if (dd->debug_level)  /* overwrite error return if extra checking is on */
      err = (errcount) ? ZOLTAN_WARN : ZOLTAN_OK;

fini:
   ZOLTAN_FREE (&procs);
   ZOLTAN_FREE (&sbuff);
   ZOLTAN_FREE (&rbuff);
   Zoltan_Comm_Destroy (&plan);

   if (dd->debug_level)  {
      sprintf (str, "Processed %d GIDs (%d local), %d GID errors", count,
       nrec, errcount);
      ZOLTAN_PRINT_INFO (dd->my_proc, yo, str);
   }

   if (dd->debug_level > 4)
      ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
   return err;
}
コード例 #12
0
static int local_HEs_from_export_lists(
  ZZ *zz,
  int remap_type,      /* type of remapping to do:  parts, procs, or none. */
  int nobj,            /* # objs the processor knows about (keep + exports) */
  int *new_proc,       /* On input, new processor assignment for each obj; 
                          Upon return, remapped new proc assignment for
                          each obj. */
  int *old_part,       /* old partition assignments for each objs */
  int *new_part,       /* On input, new partition assignments for each objs.
                          Upon return, remapped new partition assignments */
  int *HEcnt,          /* # of HEs allocated. */
  int **HEinfo         /* Array of HE info; for each HE, two pins and 
                          one edge weight. Stored as a single vector
                          to minimize communication calls.  */
) 
{
/*  Routine to remap partitions (to new processors or new partition numbers)
 *  to reduce data movement.
 *  This routine assumes the load-balancing algorithm built export lists.
 *  Objects described are those that STARTED on zz->Proc due to load balancing.
 *  For all these objects, old_proc == zz->Proc.
 */
char *yo = "local_HEs_from_export_lists";
int ierr = ZOLTAN_OK;
int i, cnt, tmp;
int *tmp_HEinfo;
int my_proc = zz->Proc;       /* This processor's rank. */

int nimp = 0;
int *imp_proc = NULL,         /* Temporary arrays if inversion of export to */
    *imp_old_part = NULL,     /* import lists is needed. */
    *imp_new_part = NULL;

int HEwgt_size;               /* # of HE weights allocated. */
int *HEwgt = NULL;            /* Array of HE weights.  Initially includes
                                 zero weights; later zero-weights are removed.*/

  if (remap_type == ZOLTAN_LB_REMAP_PROCESSORS) {
    /* Build HEs based on processor assignment.
     * We know the old processor for all objects we are keeping and all
     * export objects -- it is my_proc!
     * We also know the new processor number for all objects initially on
     * my_proc (since we built export lists.)
     * This case is a special case of partition remapping; it is easy to 
     * build the hyperedges in this special case.
     */

    HEwgt_size = zz->Num_Proc;
    HEwgt = (int *) ZOLTAN_CALLOC(HEwgt_size, sizeof(int));
    if (!HEwgt) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
      ierr = ZOLTAN_MEMERR;
      goto End;
    }

    for (i = 0; i < nobj; i++) 
      HEwgt[new_proc[i]]++;

    *HEcnt = 0;
    for (i = 0; i < HEwgt_size; i++)
      if (HEwgt[i] != 0) (*HEcnt)++;
   
    ierr = malloc_HEinfo(zz, *HEcnt, HEinfo);
    if (ierr < 0) 
      goto End;
    tmp_HEinfo = *HEinfo;

    cnt = 0;
    for (i = 0; i < HEwgt_size; i++) {
      if (HEwgt[i] != 0) {
        tmp = cnt * HEINFO_ENTRIES;
        tmp_HEinfo[tmp] = my_proc;    /* Old processor number */
        tmp_HEinfo[tmp+1] = i;        /* New processor number */
        tmp_HEinfo[tmp+2] = HEwgt[i]; /* shift non-zero weights down. */
        cnt++;
      }
    }
  }

  else {  /* ZOLTAN_LB_REMAP_PARTS */
    /* Cannot renumber partitions given export lists without summing HE weights
     * across processors.  This summation is not straightforward.  Also, a 
     * potentially large number of HEs may exist 
     * (max_old_partition_number * zz->Num_Global_Parts).  Rather than build
     * this large matrix, just compute import lists from the export lists
     * and run the import-list algorithm.
     */
    ZOLTAN_COMM_OBJ *plan;
    int msg_tag = 22345;

    ierr = Zoltan_Comm_Create(&plan, nobj, new_proc, zz->Communicator,
                              msg_tag, &nimp);

    if (nimp > 0) {
      imp_proc = (int *) ZOLTAN_MALLOC(3 * nimp * sizeof(int));
      imp_old_part = imp_proc + nimp;
      imp_new_part = imp_old_part + nimp;
      if (!imp_proc) {
        ierr = ZOLTAN_MEMERR;
        ZOLTAN_PRINT_ERROR(my_proc, yo, "Memory error.");
        goto End;
      }
    }

    ierr = Zoltan_Comm_Info(plan, NULL, NULL, NULL, NULL, NULL, NULL, 
                            NULL, NULL, NULL, NULL, NULL, imp_proc, NULL);

    msg_tag++;
    ierr = Zoltan_Comm_Do(plan, msg_tag, (char *) old_part, sizeof(int),
                          (char *) imp_old_part);

    msg_tag++;
    ierr = Zoltan_Comm_Do(plan, msg_tag, (char *) new_part, sizeof(int),
                          (char *) imp_new_part);

    Zoltan_Comm_Destroy(&plan);

    ierr = local_HEs_from_import_lists(zz, remap_type, nimp, imp_proc,
                                       imp_old_part, imp_new_part,
                                       HEcnt, HEinfo);
  }

End:

  if (HEwgt) ZOLTAN_FREE(&HEwgt);
  if (imp_proc) ZOLTAN_FREE(&imp_proc);

  return ierr;
}
コード例 #13
0
ファイル: phg_Vcycle.c プロジェクト: 00liujj/trilinos
/*  Main partitioning function for hypergraph partitioning. */
int Zoltan_PHG_Partition (
  ZZ *zz,               /* Zoltan data structure */
  HGraph *hg,           /* Input hypergraph to be partitioned */
  int p,                /* Input:  number partitions to be generated */
  float *part_sizes,    /* Input:  array of length p containing percentages
                           of work to be assigned to each partition */
  Partition parts,      /* Input:  initial partition #s; aligned with vtx 
                           arrays. 
                           Output:  computed partition #s */
  PHGPartParams *hgp)   /* Input:  parameters for hgraph partitioning. */
{

  PHGComm *hgc = hg->comm;
  VCycle  *vcycle=NULL, *del=NULL;
  int  i, err = ZOLTAN_OK, middle;
  ZOLTAN_GNO_TYPE origVpincnt; /* for processor reduction test */
  ZOLTAN_GNO_TYPE prevVcnt     = 2*hg->dist_x[hgc->nProc_x]; /* initialized so that the */
  ZOLTAN_GNO_TYPE prevVedgecnt = 2*hg->dist_y[hgc->nProc_y]; /* while loop will be entered
				 		               before any coarsening */
  ZOLTAN_GNO_TYPE tot_nPins, local_nPins;
  MPI_Datatype zoltan_gno_mpi_type;
  char *yo = "Zoltan_PHG_Partition";
  int do_timing = (hgp->use_timers > 1);
  int fine_timing = (hgp->use_timers > 2);
  int vcycle_timing = (hgp->use_timers > 4 && hgp->ProRedL == 0);
  short refine = 0;
  struct phg_timer_indices *timer = Zoltan_PHG_LB_Data_timers(zz);
  int reset_geometric_matching = 0;
  char reset_geometric_string[4];

  ZOLTAN_TRACE_ENTER(zz, yo);

  zoltan_gno_mpi_type = Zoltan_mpi_gno_type();
    
  if (do_timing) {
    if (timer->vcycle < 0) 
      timer->vcycle = Zoltan_Timer_Init(zz->ZTime, 0, "Vcycle");
    if (timer->procred < 0) 
      timer->procred = Zoltan_Timer_Init(zz->ZTime, 0, "Processor Reduction");
    if (timer->match < 0) 
      timer->match = Zoltan_Timer_Init(zz->ZTime, 1, "Matching");
    if (timer->coarse < 0) 
      timer->coarse = Zoltan_Timer_Init(zz->ZTime, 1, "Coarsening");
    if (timer->coarsepart < 0)
      timer->coarsepart = Zoltan_Timer_Init(zz->ZTime, 1,
                                           "Coarse_Partition");
    if (timer->refine < 0) 
      timer->refine = Zoltan_Timer_Init(zz->ZTime, 1, "Refinement");
    if (timer->project < 0) 
      timer->project = Zoltan_Timer_Init(zz->ZTime, 1, "Project_Up");

    ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
  }

  local_nPins = (ZOLTAN_GNO_TYPE)hg->nPins;

  MPI_Allreduce(&local_nPins,&tot_nPins,1,zoltan_gno_mpi_type,MPI_SUM,hgc->Communicator);

  origVpincnt = tot_nPins;

  if (!(vcycle = newVCycle(zz, hg, parts, NULL, vcycle_timing))) {
    ZOLTAN_PRINT_ERROR (zz->Proc, yo, "VCycle is NULL.");
    ZOLTAN_TRACE_EXIT(zz, yo);
    return ZOLTAN_MEMERR;
  }

  /* For geometric coarsening, hgp->matching pointer and string are reset
   * after geometric_levels of coarsening.  Will need to reset them after
   * this vcycle is completed.  Capture that fact now!  */
  if (!strcasecmp(hgp->redm_str, "rcb") || !strcasecmp(hgp->redm_str, "rib")) {
    reset_geometric_matching = 1;
    strcpy(reset_geometric_string, hgp->redm_str);
  }

  /****** Coarsening ******/    
#define COARSEN_FRACTION_LIMIT 0.9  /* Stop if we don't make much progress */
  while ((hg->redl>0) && (hg->dist_x[hgc->nProc_x] > (ZOLTAN_GNO_TYPE)hg->redl)
	 && ((hg->dist_x[hgc->nProc_x] < (ZOLTAN_GNO_TYPE) (COARSEN_FRACTION_LIMIT * prevVcnt + 0.5)) /* prevVcnt initialized to 2*hg->dist_x[hgc->nProc_x] */
	     || (hg->dist_y[hgc->nProc_y] < (ZOLTAN_GNO_TYPE) (COARSEN_FRACTION_LIMIT * prevVedgecnt + 0.5))) /* prevVedgecnt initialized to 2*hg->dist_y[hgc->nProc_y] */
    && hg->dist_y[hgc->nProc_y] && hgp->matching) {
      ZOLTAN_GNO_TYPE *match = NULL;
      VCycle *coarser=NULL, *redistributed=NULL;
        
      prevVcnt     = hg->dist_x[hgc->nProc_x];
      prevVedgecnt = hg->dist_y[hgc->nProc_y];

#ifdef _DEBUG      
      /* UVC: load balance stats */
      Zoltan_PHG_LoadBalStat(zz, hg);
#endif
      
      if (hgp->output_level >= PHG_DEBUG_LIST) {
          uprintf(hgc,
                  "START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
                  hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl, 
                  hgp->redm_str,
                  hgp->coarsepartition_str, hgp->refinement_str, p);
          if (hgp->output_level > PHG_DEBUG_LIST) {
              err = Zoltan_HG_Info(zz, hg);
              if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
                  goto End;
          }
      }
      if (hgp->output_level >= PHG_DEBUG_PLOT)
        Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
         "coarsening plot");

      if (do_timing) {
        ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
        ZOLTAN_TIMER_START(zz->ZTime, timer->match, hgc->Communicator);
      }
      if (vcycle_timing) {
        if (vcycle->timer_match < 0) {
          char str[80];
          sprintf(str, "VC Matching %d", hg->info);
          vcycle->timer_match = Zoltan_Timer_Init(vcycle->timer, 0, str);
        }
        ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_match,
                           hgc->Communicator);
      }

      /* Allocate and initialize Matching Array */
      if (hg->nVtx && !(match = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC (hg->nVtx*sizeof(ZOLTAN_GNO_TYPE)))) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: Matching array");
        ZOLTAN_TRACE_EXIT(zz, yo);
        return ZOLTAN_MEMERR;
      }
      for (i = 0; i < hg->nVtx; i++)
        match[i] = i;
        
      /* Calculate matching (packing or grouping) */

      err = Zoltan_PHG_Matching (zz, hg, match, hgp);

      if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
        ZOLTAN_FREE (&match);
        goto End;
      }
      if (vcycle_timing)
        ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_match,
                          hgc->Communicator);

      if (do_timing) {
        ZOLTAN_TIMER_STOP(zz->ZTime, timer->match, hgc->Communicator);
        ZOLTAN_TIMER_START(zz->ZTime, timer->coarse, hgc->Communicator);
      }

      if (vcycle_timing) {
        if (vcycle->timer_coarse < 0) {
          char str[80];
          sprintf(str, "VC Coarsening %d", hg->info);
          vcycle->timer_coarse = Zoltan_Timer_Init(vcycle->timer, 0, str);
        }
        ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_coarse,
                           hgc->Communicator);
      }
            
      if (!(coarser = newVCycle(zz, NULL, NULL, vcycle, vcycle_timing))) {
        ZOLTAN_FREE (&match);
        ZOLTAN_PRINT_ERROR (zz->Proc, yo, "coarser is NULL.");
        goto End;
      }

      /* Construct coarse hypergraph and LevelMap */
      err = Zoltan_PHG_Coarsening (zz, hg, match, coarser->hg, vcycle->LevelMap,
       &vcycle->LevelCnt, &vcycle->LevelSndCnt, &vcycle->LevelData, 
       &vcycle->comm_plan, hgp);

      if (err != ZOLTAN_OK && err != ZOLTAN_WARN) 
        goto End;

      if (vcycle_timing)
        ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_coarse,
                          hgc->Communicator);
        
      if (do_timing) {
        ZOLTAN_TIMER_STOP(zz->ZTime, timer->coarse, hgc->Communicator);
        ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
      }

      ZOLTAN_FREE (&match);

      if ((err=allocVCycle(coarser))!= ZOLTAN_OK)
        goto End;
      vcycle = coarser;
      hg = vcycle->hg;

      if (hgc->nProc > 1 && hgp->ProRedL > 0) {
        local_nPins = (ZOLTAN_GNO_TYPE)hg->nPins;
	MPI_Allreduce(&local_nPins, &tot_nPins, 1, zoltan_gno_mpi_type, MPI_SUM,
		      hgc->Communicator);

	if (tot_nPins < (ZOLTAN_GNO_TYPE)(hgp->ProRedL * origVpincnt + 0.5)) {
	  if (do_timing) {
	    ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
	    ZOLTAN_TIMER_START(zz->ZTime, timer->procred, hgc->Communicator);
	  }
	  /* redistribute to half the processors */
	  origVpincnt = tot_nPins; /* update for processor reduction test */

	  if(hg->nVtx&&!(hg->vmap=(int*)ZOLTAN_MALLOC(hg->nVtx*sizeof(int)))) {
	    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: hg->vmap");
            ZOLTAN_TRACE_EXIT(zz, yo);
	    return ZOLTAN_MEMERR;
	  }

	  for (i = 0; i < hg->nVtx; i++)
	    hg->vmap[i] = i;

	  middle = (int)((float) (hgc->nProc-1) * hgp->ProRedL);

	  if (hgp->nProc_x_req!=1&&hgp->nProc_y_req!=1) { /* Want 2D decomp */
	    if ((middle+1) > SMALL_PRIME && Zoltan_PHG_isPrime(middle+1))
	      --middle; /* if it was prime just use one less #procs (since
			   it should be bigger than SMALL_PRIME it is safe to
			   decrement) */
	  }

	  if (!(hgc = (PHGComm*) ZOLTAN_MALLOC (sizeof(PHGComm)))) {
	    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: PHGComm");
            ZOLTAN_TRACE_EXIT(zz, yo);
	    return ZOLTAN_MEMERR;
	  }

	  if (!(redistributed=newVCycle(zz,NULL,NULL,vcycle,vcycle_timing))) {
	    ZOLTAN_FREE (&hgc);
	    ZOLTAN_PRINT_ERROR (zz->Proc, yo, "redistributed is NULL.");
	    goto End;
	  }

	  Zoltan_PHG_Redistribute(zz,hgp,hg,0,middle,hgc, redistributed->hg,
				  &vcycle->vlno,&vcycle->vdest);
	  if (hgp->UseFixedVtx || hgp->UsePrefPart)
            redistributed->hg->bisec_split = hg->bisec_split;

	  if ((err=allocVCycle(redistributed))!= ZOLTAN_OK)
	    goto End;
	  vcycle = redistributed;

	  if (hgc->myProc < 0)
	    /* I'm not in the redistributed part so I should go to uncoarsening
	       refinement and wait */ {
	    if (fine_timing) {
	      if (timer->cpgather < 0)
		timer->cpgather = Zoltan_Timer_Init(zz->ZTime, 1, "CP Gather");
	      if (timer->cprefine < 0)
		timer->cprefine =Zoltan_Timer_Init(zz->ZTime, 0, "CP Refine");
	      if (timer->cpart < 0)
		timer->cpart = Zoltan_Timer_Init(zz->ZTime, 0, "CP Part");
	    }
	    if (do_timing) {
	      ZOLTAN_TIMER_STOP(zz->ZTime, timer->procred, hgc->Communicator);
	      ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
	    }
	    goto Refine;
	  }

	  hg = vcycle->hg;
	  hg->redl = hgp->redl; /* not set with hg creation */
	  if (do_timing) {
	    ZOLTAN_TIMER_STOP(zz->ZTime, timer->procred, hgc->Communicator);
	    ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
	  }
	}
      }
  }

  if (hgp->output_level >= PHG_DEBUG_LIST) {
    uprintf(hgc, "START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
     hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl, 
     hgp->redm_str, hgp->coarsepartition_str, hgp->refinement_str, p);
    if (hgp->output_level > PHG_DEBUG_LIST) {
      err = Zoltan_HG_Info(zz, hg);
      if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
        goto End;
    }
  }
  if (hgp->output_level >= PHG_DEBUG_PLOT)
    Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
     "coarsening plot");

  /* free array that may have been allocated in matching */
  if (hgp->vtx_scal) {
    hgp->vtx_scal_size = 0;
    ZOLTAN_FREE(&(hgp->vtx_scal));
  }

  if (do_timing) {
    ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
    ZOLTAN_TIMER_START(zz->ZTime, timer->coarsepart, hgc->Communicator);
  }

  /****** Coarse Partitioning ******/

  err = Zoltan_PHG_CoarsePartition (zz, hg, p, part_sizes, vcycle->Part, hgp);

  if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
    goto End;

  if (do_timing) {
    ZOLTAN_TIMER_STOP(zz->ZTime, timer->coarsepart, hgc->Communicator);
    ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
  }

Refine:
  del = vcycle;
  refine = 1;

  /****** Uncoarsening/Refinement ******/
  while (vcycle) {
    VCycle *finer = vcycle->finer;
    hg = vcycle->hg;

    if (refine && hgc->myProc >= 0) {
      if (do_timing) {
	ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
	ZOLTAN_TIMER_START(zz->ZTime, timer->refine, hgc->Communicator);
      }
      if (vcycle_timing) {
	if (vcycle->timer_refine < 0) {
	  char str[80];
	  sprintf(str, "VC Refinement %d", hg->info);
	  vcycle->timer_refine = Zoltan_Timer_Init(vcycle->timer, 0, str);
	}
	ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_refine,
			   hgc->Communicator);
      }

      err = Zoltan_PHG_Refinement (zz, hg, p, part_sizes, vcycle->Part, hgp);
        
      if (do_timing) {
	ZOLTAN_TIMER_STOP(zz->ZTime, timer->refine, hgc->Communicator);
	ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
      }
      if (vcycle_timing)
	ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_refine,
			  hgc->Communicator);

                          
      if (hgp->output_level >= PHG_DEBUG_LIST)     
	uprintf(hgc, 
		"FINAL %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d bal=%.2f cutl=%.2f\n",
		hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl, 
		hgp->redm_str,
		hgp->coarsepartition_str, hgp->refinement_str, p,
		Zoltan_PHG_Compute_Balance(zz, hg, part_sizes, 0, p, 
                                           vcycle->Part),
		Zoltan_PHG_Compute_ConCut(hgc, hg, vcycle->Part, p, &err));

      if (hgp->output_level >= PHG_DEBUG_PLOT)
	Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, vcycle->Part,
			"partitioned plot");
    }

    if (finer)  {
      int *rbuffer;
            
      /* Project coarse partition to fine partition */
      if (finer->comm_plan) {
	refine = 1;
	if (do_timing) {
	  ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
	  ZOLTAN_TIMER_START(zz->ZTime, timer->project, hgc->Communicator);
	}
	if (vcycle_timing) {
	  if (vcycle->timer_project < 0) {
	    char str[80];
	    sprintf(str, "VC Project Up %d", hg->info);
	    vcycle->timer_project = Zoltan_Timer_Init(vcycle->timer, 0, str);
	  }
	  ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_project,
			     hgc->Communicator);
	}
        
	/* easy to assign partitions to internal matches */
	for (i = 0; i < finer->hg->nVtx; i++)
	  if (finer->LevelMap[i] >= 0)   /* if considers only the local vertices */
	    finer->Part[i] = vcycle->Part[finer->LevelMap[i]];
          
	/* now that the course partition assignments have been propagated */
	/* upward to the finer level for the local vertices, we need to  */    
	/* fill the LevelData (matched pairs of a local vertex with a    */
	/* off processor vertex) with the partition assignment of the    */
	/* local vertex - can be done totally in the finer level!        */    
	for (i = 0; i < finer->LevelCnt; i++)  {
	  ++i;          /* skip over off processor lno */
	  finer->LevelData[i] = finer->Part[finer->LevelData[i]]; 
	}
            
	/* allocate rec buffer to exchange LevelData information */
	rbuffer = NULL;
	if (finer->LevelSndCnt > 0)  {
	  rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->LevelSndCnt * sizeof(int));
	  if (!rbuffer)    {
	    ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory.");
            ZOLTAN_TRACE_EXIT(zz, yo);
	    return ZOLTAN_MEMERR;
	  }
	}       
      
	/* get partition assignments from owners of externally matched vtxs */  
	Zoltan_Comm_Resize (finer->comm_plan, NULL, COMM_TAG, &i);
	Zoltan_Comm_Do_Reverse (finer->comm_plan, COMM_TAG+1, 
         (char*) finer->LevelData, 2 * sizeof(int), NULL, (char*) rbuffer);

	/* process data to assign partitions to expernal matches */
	for (i = 0; i < 2 * finer->LevelSndCnt;)  {
	  int lno, partition;
	  lno       = rbuffer[i++];
	  partition = rbuffer[i++];      
	  finer->Part[lno] = partition;         
	}

	ZOLTAN_FREE (&rbuffer);                  
	Zoltan_Comm_Destroy (&finer->comm_plan);                   

	if (do_timing) {
	  ZOLTAN_TIMER_STOP(zz->ZTime, timer->project, hgc->Communicator);
	  ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
	}
	if (vcycle_timing)
	  ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_project,
			    hgc->Communicator);
      } else {
	int *sendbuf = NULL, size;
	refine = 0;
	/* ints local and partition numbers */
	if (finer->vlno) {
	  sendbuf = (int*) ZOLTAN_MALLOC (2 * hg->nVtx * sizeof(int));
	  if (!sendbuf) {
	    ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory.");
            ZOLTAN_TRACE_EXIT(zz, yo);
	    return ZOLTAN_MEMERR;
	  }

	  for (i = 0; i < hg->nVtx; ++i) {
	    sendbuf[2 * i] = finer->vlno[i];     /* assign local numbers */
	    sendbuf[2 * i + 1] = vcycle->Part[i];/* assign partition numbers */
	  }
	}

	ZOLTAN_FREE (&hgc);
	hgc = finer->hg->comm; /* updating hgc is required when the processors
				   change */
	/* Create comm plan to unredistributed processors */
	err = Zoltan_Comm_Create(&finer->comm_plan, finer->vlno ? hg->nVtx : 0,
				 finer->vdest, hgc->Communicator, COMM_TAG+2,
				 &size);

	if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
	  ZOLTAN_PRINT_ERROR(hgc->myProc, yo, "Zoltan_Comm_Create failed.");
	  goto End;
	}

	/* allocate rec buffer to exchange sendbuf information */
	rbuffer = NULL;
	if (finer->hg->nVtx) {
	  rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->hg->nVtx * sizeof(int));

	  if (!rbuffer) {
	    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
            ZOLTAN_TRACE_EXIT(zz, yo);
	    return ZOLTAN_MEMERR;
	  }
	}

	/* Use plan to send partitions to the unredistributed processors */

	Zoltan_Comm_Do(finer->comm_plan, COMM_TAG+3, (char *) sendbuf,
		       2*sizeof(int), (char *) rbuffer);

	MPI_Bcast(rbuffer, 2*finer->hg->nVtx, MPI_INT, 0, hgc->col_comm);
	
	/* process data to assign partitions to unredistributed processors */
	for (i = 0; i < 2 * finer->hg->nVtx;) {
	  int lno, partition;
	  lno       = rbuffer[i++];
	  partition = rbuffer[i++];
	  finer->Part[lno] = partition;
	}

	if (finer->vlno)
	  ZOLTAN_FREE (&sendbuf);

	ZOLTAN_FREE (&rbuffer);
	Zoltan_Comm_Destroy (&finer->comm_plan);
      }
    }

    vcycle = finer;
  }       /* while (vcycle) */
    
End:
  vcycle = del;
  while (vcycle) {
    if (vcycle_timing) {
      Zoltan_Timer_PrintAll(vcycle->timer, 0, hgc->Communicator, stdout);
      Zoltan_Timer_Destroy(&vcycle->timer);
    }
    if (vcycle->finer) {   /* cleanup by level */
      Zoltan_HG_HGraph_Free (vcycle->hg);

      if (vcycle->LevelData)
	Zoltan_Multifree (__FILE__, __LINE__, 4, &vcycle->Part,
			  &vcycle->LevelMap, &vcycle->LevelData, &vcycle->hg);
      else if (vcycle->vlno)
	Zoltan_Multifree (__FILE__, __LINE__, 5, &vcycle->Part, &vcycle->vdest,
			  &vcycle->vlno, &vcycle->LevelMap, &vcycle->hg);
      else
	Zoltan_Multifree (__FILE__, __LINE__, 3, &vcycle->Part,
			  &vcycle->LevelMap, &vcycle->hg);
    }
    else                   /* cleanup top level */
      Zoltan_Multifree (__FILE__, __LINE__, 2, &vcycle->LevelMap,
                        &vcycle->LevelData);
    del = vcycle;
    vcycle = vcycle->finer;
    ZOLTAN_FREE(&del);
  }

  if (reset_geometric_matching) {
    strcpy(hgp->redm_str, reset_geometric_string);
    Zoltan_PHG_Set_Matching_Fn(hgp);
  }

  if (do_timing)
    ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
  ZOLTAN_TRACE_EXIT(zz, yo) ;
  return err;
}
コード例 #14
0
ファイル: comm_main.c プロジェクト: haripandey/trilinos
int main(int argc, char *argv[])
{
    FILE     *in_file = NULL;	/* file with data for problems */
    ZOLTAN_COMM_OBJ *plan;	/* communication data structure pointer */
    struct Params params;	/* parameters describing a problem */
    struct Data data;		/* data describing a problem instance */
    struct Data my_send_data;	/* data I initially own */
    struct Answer true_answer;	/* expected outcome of exchange */
    int       nvals_recv;	/* number of vals I own after exchange */
    float    *recv_data;	/* values I own after exchange */
    float    *reverse_data;	/* values I own after reversing communication */
    char      file_name[100];	/* name of input file */
    int       nbytes;		/* size of objects */
    int       my_proc;		/* my processor ID */
    int       nprocs;		/* total number of processors */
    int       flag;		/* return code from comm operations */
    int       more_problems;	/* are there more problems to do? */
    int       i, j;

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &my_proc);
    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);

    Zoltan_Memory_Debug(2);

    if (argc > 1) strcpy(file_name, argv[1]);
    else strcpy(file_name,"comm_input.dat");
    
    if (my_proc == 0) {
	in_file = fopen(file_name, "r");
	if (in_file == NULL) {
	    printf("No input file `%s' found.\n", file_name);
	}
    }

    /* Read some problem descriptors from a file */
    more_problems = read_comm_problem(in_file, &params, &out_level, my_proc);

    while (more_problems) {

	/* Generate full data at random on each proc */
	gen_comm_data(&params, &data, nprocs);

if (out_level > 2) if (my_proc == 0) print_data("DATA", &data);

	/* Figure out from the data what what to expect to receive */
	extract_comm_answer(&data, &true_answer, my_proc, nprocs);

	/* Extract what to send */
	set_up_comm_from_send(&data, &my_send_data, params.blocked, my_proc, nprocs);

if (out_level > 2) print_data("MY_DATA", &my_send_data);

if (out_level > 1) printf("%d: About to call comm_create\n", my_proc);
	/* Call comm routines */
	Zoltan_Comm_Create(&plan, my_send_data.nvals, my_send_data.proc_dest,
	    MPI_COMM_WORLD, 1, &nvals_recv);

if (out_level > 1) printf("%d: About to call comm_info\n", my_proc);
        check_comm_info(plan, &my_send_data, nvals_recv, my_proc);

if (out_level > 2) print_plan("BEFORE RESIZE", plan, my_proc);
	/* "4" reflects the max_sizes value in gen_comm_data */
	recv_data = (float *) ZOLTAN_MALLOC(nvals_recv * 4 * sizeof(float));
	reverse_data = (float *) ZOLTAN_MALLOC(my_send_data.nvals * 4 * sizeof(float));

	if (my_send_data.sizes != NULL) {
if (out_level > 1) printf("%d: About to call comm_resize\n", my_proc);
	    Zoltan_Comm_Resize(plan, my_send_data.sizes, 43, NULL);
	    Zoltan_Comm_Resize(plan, NULL, 43, NULL);
	    Zoltan_Comm_Resize(plan, my_send_data.sizes, 43, NULL);
if (out_level > 2) print_plan("AFTER RESIZE", plan, my_proc);
	}

	if (my_send_data.sizes == NULL) nbytes = my_send_data.same_size * sizeof(float);
	else nbytes = sizeof(float);

if (out_level > 1) printf("%d: About to call comm_do\n", my_proc);
	flag = Zoltan_Comm_Do(plan, 2, (char *) my_send_data.vals, nbytes,
	    (char *) recv_data);

	if (flag == ZOLTAN_OK) {
if (out_level > 1) printf("%d: About to call check_answer\n", my_proc);
	    /* Check answers */
	    check_comm_answer(&true_answer, recv_data, my_proc);
	}
	else {
	    printf("%d: Comm_Do returned error code %d\n", my_proc, flag);
	}

if (out_level > 1) printf("%d: About to call comm_do_reverse\n", my_proc);
	i = (true_answer.sizes != NULL && plan->indices_to != NULL);
	MPI_Allreduce(&i, &j, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
	if (j == 0) flag = Zoltan_Comm_Do_Reverse(plan, 2, (char *) recv_data,
	    nbytes, true_answer.sizes, (char *) reverse_data);
	else {
	    if (my_proc == 0)
		printf(">> Non-blocked, variable-sized recvs not supported\n");
	    flag = ZOLTAN_FATAL;
	}

	if (flag == ZOLTAN_OK) {
if (out_level > 1) printf("%d: About to call check_answer_reverse\n", my_proc);
	    /* Check answers */
	    check_comm_answer_reverse(&my_send_data, reverse_data, my_proc);
	}
	else {
if (out_level > 1) printf("%d: Comm_Do_Reverse returned error code %d\n", my_proc, flag);
	}


	/* Free up data structures */
	ZOLTAN_FREE(&reverse_data);
	ZOLTAN_FREE(&recv_data);

	free_comm_data(&data, &my_send_data, &true_answer);

	Zoltan_Comm_Destroy(&plan);

	/* Read some problem descriptors from a file */
	more_problems = read_comm_problem(in_file, &params, &out_level, my_proc);

    }

    Zoltan_Memory_Stats();
    MPI_Finalize();

    return(0);
}
コード例 #15
0
int Zoltan_Order_Get_GID_Order(
    struct Zoltan_Struct       *zz,
    ZOLTAN_ID_PTR               global_ids,
    ZOLTAN_ID_PTR               order_ids
)
{
    int * proctab;
    ZOLTAN_ID_PTR sendgidtab;
    int * sendtab, *recvtab;
    int i;
    int nrecv;
    struct Zoltan_Comm_Obj *plan;
    int ierr = ZOLTAN_OK;
    int *vtxdist;
    int tag = 24061986;
    int offset;

    zz->Order.gidrank = order_ids;

    if (!strcmp(zz->Order.order_type, "LOCAL")) {
        for (i = 0 ; i < zz->Order.nbr_objects ; ++i) {
            order_ids[i] = global_ids[zz->Order.rank[i] - zz->Order.start_index];
        }
        return (ZOLTAN_OK);
    }

    ierr = Zoltan_Get_Distribution(zz, &vtxdist);
    if (ierr != ZOLTAN_OK) return (ierr);

    proctab = (int*) ZOLTAN_MALLOC(3*zz->Order.nbr_objects*sizeof(int));
    sendgidtab = ZOLTAN_MALLOC_GID_ARRAY(zz, 2*zz->Order.nbr_objects);
    if (proctab == NULL)  {
        ZOLTAN_FREE(&vtxdist);
        return (ZOLTAN_MEMERR);
    }
    if (sendgidtab == NULL) {
        ZOLTAN_FREE(&proctab);
        ZOLTAN_FREE(&vtxdist);
        return (ZOLTAN_MEMERR);
    }
    sendtab = proctab + zz->Order.nbr_objects;
    recvtab = sendtab + zz->Order.nbr_objects;

    for (i = 0 ; i < zz->Order.nbr_objects ; ++i) {
        proctab[i] = Zoltan_Get_Processor_Graph(vtxdist, zz->Num_Proc, zz->Order.rank[i] - zz->Order.start_index);
        sendtab[i] = zz->Order.rank[i] - zz->Order.start_index;
    }

    ierr = Zoltan_Comm_Create(&plan, zz->Order.nbr_objects, proctab, zz->Communicator, tag++,
                              &nrecv);
    if (ierr != ZOLTAN_OK) {
        ZOLTAN_FREE(&sendgidtab);
        ZOLTAN_FREE(&proctab);
        ZOLTAN_FREE(&vtxdist);
        return (ierr);
    }

    ierr = Zoltan_Comm_Do(plan, tag++, (char *) sendtab, sizeof(int), (char *) recvtab);

    if (ierr != ZOLTAN_OK) {
        ZOLTAN_FREE(&sendgidtab);
        ZOLTAN_FREE(&proctab);
        ZOLTAN_FREE(&vtxdist);
        return (ierr);
    }

    offset = vtxdist[zz->Proc];
    for (i = 0 ; i < zz->Order.nbr_objects ; ++i) {
        int position;
        position = recvtab[i]-offset;
        ZOLTAN_SET_GID(zz, &sendgidtab[i], &global_ids[position]);
    }

    ierr = Zoltan_Comm_Do_Reverse(plan, tag++, (char *) sendgidtab, zz->Num_GID*sizeof(int), NULL,
                                  (char *) order_ids);

    Zoltan_Comm_Destroy(&plan);
    ZOLTAN_FREE(&sendgidtab);
    ZOLTAN_FREE(&proctab);
    ZOLTAN_FREE(&vtxdist);

    return (ierr);
}
コード例 #16
0
/* if !copy, inmat is not usable after this call */
int
Zoltan_Matrix2d_Distribute (ZZ* zz, Zoltan_matrix inmat, /* Cannot be const as we can share it inside outmat */
			    Zoltan_matrix_2d *outmat, int copy)
{
  static char *yo = "Zoltan_Matrix2d_Distribute";
  int ierr = ZOLTAN_OK;
  int nProc_x, nProc_y;
  int myProc_x, myProc_y;
  int i, j, cnt;
  int *proclist = NULL;
  Zoltan_Arc *nonzeros= NULL, *sendbuf= NULL;
  ZOLTAN_GNO_TYPE *perm_y = NULL;
  float *wgtarray = NULL;
  float *tmpwgtarray = NULL;
  int msg_tag = 1021982;
  ZOLTAN_COMM_OBJ *plan;
  MPI_Comm communicator = MPI_COMM_NULL;
  int nProc;
  ZOLTAN_GNO_TYPE *yGNO = NULL;
  ZOLTAN_GNO_TYPE *pinGNO = NULL;
  ZOLTAN_GNO_TYPE tmp_gno;
  void *partdata = NULL;
  MPI_Datatype zoltan_gno_mpi_type;

  ZOLTAN_TRACE_ENTER(zz, yo);

  zoltan_gno_mpi_type = Zoltan_mpi_gno_type();

  memcpy(&outmat->mtx, &inmat, sizeof(Zoltan_matrix));
  if(copy) {
    /* TODO: We need to copy the arrays also */
    Zoltan_Matrix_Reset (&outmat->mtx);
    /* Copy also directories */
    outmat->mtx.ddX = Zoltan_DD_Copy (inmat.ddX);
    if (inmat.ddY == inmat.ddX)
      outmat->mtx.ddY = outmat->mtx.ddX;
    else
      outmat->mtx.ddY = Zoltan_DD_Copy (inmat.ddY);
  }

  communicator = outmat->comm->Communicator;
  nProc = outmat->comm->nProc;

  nProc_x = outmat->comm->nProc_x;
  nProc_y = outmat->comm->nProc_y;
  myProc_x = outmat->comm->myProc_x;
  myProc_y = outmat->comm->myProc_y;

KDDKDDKDD(zz->Proc, "    Zoltan_Matrix_Remove_Duplicates");
  ierr = Zoltan_Matrix_Remove_Duplicates(zz, outmat->mtx, &outmat->mtx);

/* KDDKDDKDD  FIX INDENTATION OF THIS BLOCK */
if (inmat.opts.speed != MATRIX_NO_REDIST) {
  if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin) {
    /* I need to know the original distribution */
    if (outmat->mtx.ddX != outmat->mtx.ddY) { /* No initial distribution */
      outmat->hashDistFct = (distFnct *)&Zoltan_Distribute_Linear;
    }
    else {
      int *cmember = NULL;

      cmember = (int*)ZOLTAN_MALLOC(outmat->mtx.nY*sizeof(int));
      if (outmat->mtx.nY > 0 && cmember == NULL) MEMORY_ERROR;
      Zoltan_DD_Find (outmat->mtx.ddY, (ZOLTAN_ID_PTR)outmat->mtx.yGNO, NULL, (char *)cmember, NULL,
		      outmat->mtx.nY, NULL);
KDDKDDKDD(zz->Proc, "    Zoltan_Distribute_Partition_Register");
      partdata = Zoltan_Distribute_Partition_Register(zz, outmat->mtx.nY, outmat->mtx.yGNO,
						      cmember, zz->Num_Proc, zz->Num_Proc);
      ZOLTAN_FREE(&cmember);
      Zoltan_Distribute_Set(outmat, (distFnct *)&Zoltan_Distribute_Origin, partdata);
    }
  }

  /*
   * Build comm plan for sending non-zeros to their target processors in
   * 2D data distribution.
   */
  /* TRICK: create fake arc (edgeno, -1) for empty Y. Upper bound for size might be nPins + nY */
  proclist = (int *)ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY) *sizeof(int));
  sendbuf = (Zoltan_Arc*) ZOLTAN_MALLOC((outmat->mtx.nPins +outmat->mtx.nY)* sizeof(Zoltan_Arc));

  if ((outmat->mtx.nPins + outmat->mtx.nY >0) && (proclist == NULL || sendbuf == NULL)) MEMORY_ERROR;

  wgtarray = (float*) ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY)*outmat->mtx.pinwgtdim*sizeof(float));

  if (outmat->mtx.nPins*outmat->mtx.pinwgtdim && !wgtarray) MEMORY_ERROR;

  yGNO = outmat->mtx.yGNO;
  pinGNO = outmat->mtx.pinGNO;

KDDKDDKDD(zz->Proc, "    CommPlan Hash");
  cnt = 0;
  for (i = 0; i < outmat->mtx.nY; i++) {
    ZOLTAN_GNO_TYPE edge_gno=-1;
    /* processor row for the edge */
    edge_gno = yGNO[i];

    for (j = outmat->mtx.ystart[i]; j < outmat->mtx.yend[i]; j++) {
      ZOLTAN_GNO_TYPE vtx_gno=-1;
      /* processor column for the vertex */
      vtx_gno = pinGNO[j];

      proclist[cnt] = (*outmat->hashDistFct)(edge_gno, vtx_gno, outmat->hashDistData,
					  &sendbuf[cnt].part_y);
      if (proclist[cnt] < 0) /* Discard this nnz */
        continue;
      sendbuf[cnt].GNO[0] = edge_gno;
      sendbuf[cnt].GNO[1] = vtx_gno;
      memcpy(wgtarray+cnt*outmat->mtx.pinwgtdim, outmat->mtx.pinwgt+j*outmat->mtx.pinwgtdim,
               outmat->mtx.pinwgtdim*sizeof(float));
      cnt++;
    }
    if(outmat->mtx.ystart[i] == outmat->mtx.yend[i]) {
      proclist[cnt] = (*outmat->hashDistFct)(edge_gno, -1, outmat->hashDistData,
					  &sendbuf[cnt].part_y);
      if (proclist[cnt] < 0) /* Discard this nnz */
        continue;
      sendbuf[cnt].GNO[0] = edge_gno;
      sendbuf[cnt].GNO[1] = -1;
      memset(wgtarray+cnt*outmat->mtx.pinwgtdim, 0,outmat->mtx.pinwgtdim*sizeof(float));
      cnt++;
    }
  }

  if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin)
    Zoltan_Distribute_Partition_Free(&outmat->hashDistData);

  if (outmat->mtx.yend != outmat->mtx.ystart + 1)
    ZOLTAN_FREE(&outmat->mtx.yend);
  outmat->mtx.yend = NULL;
  ZOLTAN_FREE(&outmat->mtx.ystart);
  ZOLTAN_FREE(&outmat->mtx.yGNO);
  ZOLTAN_FREE(&outmat->mtx.pinGNO);
  ZOLTAN_FREE(&outmat->mtx.pinwgt);
  ZOLTAN_FREE(&outmat->mtx.yGID);


  /*
   * Send pins to their target processors.
   * They become non-zeros in the 2D data distribution.
   */

KDDKDDKDD(zz->Proc, "    CommPlan Create");
  msg_tag--;
  ierr = Zoltan_Comm_Create(&plan, cnt, proclist, communicator, msg_tag, &outmat->mtx.nPins);
  ZOLTAN_FREE(&proclist);

  nonzeros = (Zoltan_Arc *) ZOLTAN_MALLOC((outmat->mtx.nPins) * sizeof(Zoltan_Arc));
  if (outmat->mtx.nPins && nonzeros == NULL) MEMORY_ERROR;

  msg_tag--;
  Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, sizeof(Zoltan_Arc), (char *) nonzeros);
  ZOLTAN_FREE(&sendbuf);

  if (outmat->mtx.pinwgtdim) { /* We have to take care about weights */
    tmpwgtarray = (float*) ZOLTAN_MALLOC(outmat->mtx.nPins*outmat->mtx.pinwgtdim*sizeof(float));
    if (outmat->mtx.nPins && tmpwgtarray == NULL) MEMORY_ERROR;

    msg_tag--;
    Zoltan_Comm_Do(plan, msg_tag, (char *) wgtarray, outmat->mtx.pinwgtdim*sizeof(float),
		   (char *) tmpwgtarray);
    ZOLTAN_FREE(&wgtarray);
  }
  Zoltan_Comm_Destroy(&plan);

  /* Unpack the non-zeros received. */

KDDKDDKDD(zz->Proc, "    Zoltan_Matrix_Remove_DupArcs");
  /* TODO: do take care about singletons */
  Zoltan_Matrix_Remove_DupArcs(zz, outmat->mtx.nPins, (Zoltan_Arc*)nonzeros, tmpwgtarray,
			       &outmat->mtx);
}

  /* Now we just have to change numbering */
  outmat->dist_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_y+1), sizeof(ZOLTAN_GNO_TYPE));
  outmat->dist_x = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_x+1), sizeof(ZOLTAN_GNO_TYPE));
  if (outmat->dist_y == NULL || outmat->dist_x == NULL) MEMORY_ERROR;

  /* FIXME: Work only in 1D */
  tmp_gno = (ZOLTAN_GNO_TYPE)outmat->mtx.nY;
  MPI_Allgather(&tmp_gno, 1, zoltan_gno_mpi_type, outmat->dist_y+1, 1, zoltan_gno_mpi_type, communicator);
  for (i = 1 ; i <= nProc_y ; i ++) {
    outmat->dist_y[i] += outmat->dist_y[i-1];
  }
  outmat->dist_x[1] = outmat->mtx.globalX;

  perm_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(outmat->mtx.nY * sizeof(ZOLTAN_GNO_TYPE));
  if (outmat->mtx.nY > 0 && perm_y == NULL) MEMORY_ERROR;
  for (i = 0 ; i < outmat->mtx.nY ; ++i){
    perm_y[i] = i + outmat->dist_y[myProc_y];
  }

KDDKDDKDD(zz->Proc, "    Zoltan_Matrix_Permute");
  Zoltan_Matrix_Permute(zz, &outmat->mtx, perm_y);

KDDKDDKDD(zz->Proc, "    Zoltan_Matrix_Permute done");
 End:
  ZOLTAN_FREE(&perm_y);
  ZOLTAN_FREE(&proclist);
  ZOLTAN_FREE(&sendbuf);
  ZOLTAN_FREE(&nonzeros);
  ZOLTAN_FREE(&tmpwgtarray);


  ZOLTAN_TRACE_EXIT(zz, yo);

  return (ierr);
}
コード例 #17
0
ファイル: DD_Find.c プロジェクト: xunzhang/ESMF_Regridding
int Zoltan_DD_Find (
 Zoltan_DD_Directory *dd, /* contains directory state information          */
 ZOLTAN_ID_PTR gid,       /* Incoming list of GIDs to get owners proc      */
 ZOLTAN_ID_PTR lid,       /* Outgoing corresponding list of LIDs           */
 ZOLTAN_ID_PTR data,      /* Outgoing optional corresponding user data     */
 int *partition,          /* Outgoing optional partition information       */
 int  count,              /* Count of GIDs in above list (in)              */
 int *owner)              /* Outgoing corresponding list of data locations */
   {
   ZOLTAN_COMM_OBJ *plan  = NULL ;  /* efficient MPI communication     */
   char            *rbuff = NULL ;  /* receive buffer                  */
   char            *sbuff = NULL ;  /* send buffer                     */
   int             *procs = NULL ;  /* list of processors to contact   */
   DD_Find_Msg     *ptr   = NULL ;
   int              i ;
   int              nrec ;          /* number of messages to receive   */
   int              err ;           /* return error condition          */
   int              errcount ;      /* count of GIDs not found         */

   char            *yo = "Zoltan_DD_Find" ;

   if (dd != NULL && dd->debug_level > 1)
      ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL);

   /* input sanity check */
   if (dd == NULL || count < 0 || ((owner == NULL || gid == NULL) && count > 0))
      {
      ZOLTAN_PRINT_ERROR ((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc),
       yo, "Invalid input argument.") ;
      if (dd != NULL && dd->debug_level > 1)
         ZOLTAN_TRACE_OUT((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc),
          yo, NULL);
      return ZOLTAN_DD_INPUT_ERROR ;
      }

   /* allocate memory for processors to contact for directory info */
   if (count > 0)
      {
      procs = (int *) ZOLTAN_MALLOC (sizeof (int) * count) ;
      if (procs == NULL)
        {
        ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list.") ;
        if (dd->debug_level > 1)
           ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
        return ZOLTAN_DD_MEMORY_ERROR ;
        }
      }

   /* allocate memory for DD_Find_Msg send buffer */
   if (count > 0)
      {
      sbuff = (char *) ZOLTAN_MALLOC (dd->find_msg_size * count) ;
      if (sbuff == NULL)
         {
         ZOLTAN_FREE (&procs) ;
         ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer.") ;
         if (dd->debug_level > 1)
            ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
         return ZOLTAN_DD_MEMORY_ERROR ;
         }
      }

   if (dd->debug_level > 2)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs.");

   /* for each GID, fill DD_Find_Msg buffer and contact list */
   for (i = 0 ; i < count ; i++)
      {
      procs[i] = dd->hash (gid + i*dd->gid_length, dd->gid_length, dd->nproc) ;
      ptr      = (DD_Find_Msg *) (sbuff + i * dd->find_msg_size) ;

      ptr->index = i ;
      ptr->proc  = procs[i] ;
      ZOLTAN_SET_ID (dd->gid_length, ptr->id, gid + i*dd->gid_length) ;
      }

   if (dd->debug_level > 2)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill.");

   /* create efficient communication plan */
   err = Zoltan_Comm_Create (&plan, count, procs, dd->comm,
    ZOLTAN_DD_FIND_MSG_TAG, &nrec) ;
   if (dd->debug_level > 2)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create.");
   if (err != ZOLTAN_OK)
      goto fini ;

   /* allocate receive buffer */
   if (nrec > 0)
      {
      rbuff = (char *) ZOLTAN_MALLOC (nrec * dd->find_msg_size) ;
      if (rbuff == NULL)
         {
         err = ZOLTAN_DD_MEMORY_ERROR ;
         goto fini ;
         }
      }

   /* send out find messages across entire system */
   err = Zoltan_Comm_Do (plan, ZOLTAN_DD_FIND_MSG_TAG+1, sbuff,
    dd->find_msg_size, rbuff) ;
   if (err != ZOLTAN_OK)
      goto fini ;

   if (dd->debug_level > 2)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do.");

   /* get find messages directed to me, fill in return information */
   errcount = 0 ;
   for (i = 0 ; i < nrec ; i++)
      {
      ptr = (DD_Find_Msg *) (rbuff + i*dd->find_msg_size) ;

      err = DD_Find_Local (dd, ptr->id, ptr->id, ptr->id + dd->max_id_length,
       (partition) ? (&ptr->partition) : NULL, &ptr->proc) ;
      if (err == ZOLTAN_DD_GID_NOT_FOUND_ERROR)
          errcount++ ;
      }

   if (dd->debug_level > 2)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill in return info.");

   /* send return information back to requester */
   err = Zoltan_Comm_Do_Reverse(plan, ZOLTAN_DD_FIND_MSG_TAG+2, rbuff,
    dd->find_msg_size, NULL, sbuff) ;
   if (err != ZOLTAN_OK)
      goto fini ;

   if (dd->debug_level > 2)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Reverse.");

   /* fill in user supplied lists with returned information */
   for (i = 0 ; i < count ; i++)
      {
      ptr = (DD_Find_Msg *) (sbuff + i*dd->find_msg_size) ;

      owner[ptr->index] = ptr->proc ;

      if (partition != NULL)
         partition[ptr->index] = ptr->partition ;

      if (lid != NULL)
         ZOLTAN_SET_ID (dd->lid_length, lid + ptr->index * dd->lid_length,
           ptr->id) ;

      if (data != NULL)
         ZOLTAN_SET_ID (dd->user_data_length, data + ptr->index
          * dd->user_data_length, ptr->id + dd->max_id_length) ;
      }

   /* if at least one GID was not found, notify caller of error */
   err = (errcount == 0) ? ZOLTAN_DD_NORMAL_RETURN
                         : ZOLTAN_DD_GID_NOT_FOUND_ERROR ;

   if (dd->debug_level > 2)
      ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill return lists.");

fini:
   ZOLTAN_FREE (&sbuff) ;
   ZOLTAN_FREE (&rbuff) ;
   ZOLTAN_FREE (&procs) ;
   Zoltan_Comm_Destroy (&plan) ;

   if (err != ZOLTAN_DD_NORMAL_RETURN)
      ZOLTAN_PRINT_WARN (dd->my_proc, yo, "Return is not normal.") ;

   if (dd->debug_level > 0)
      {
      char str[100] ;      /* diagnostic message string */
      sprintf (str, "Processed %d GIDs, GIDs not found: %d", count, errcount) ;
      ZOLTAN_PRINT_INFO (dd->my_proc, yo, str) ;
      }

   if (dd->debug_level > 1)
      ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);

   return err ;
   }
コード例 #18
0
ファイル: verify_graph.c プロジェクト: 00liujj/trilinos
int Zoltan_Verify_Graph(MPI_Comm comm, indextype *vtxdist, indextype *xadj, 
       indextype *adjncy, weighttype *vwgt, weighttype *adjwgt, 
       int vwgt_dim, int ewgt_dim, 
       int graph_type, int check_graph, int output_level)
{
  int ierr, flag, cross_edges = 0, mesg_size, sum;
  int nprocs, proc, *proclist, errors, global_errors;
  int *perm=NULL; 
  int free_adjncy_sort=0;
  ZOLTAN_COMM_OBJ *comm_plan;
  static char *yo = "Zoltan_Verify_Graph";
  char msg[256];
  ZOLTAN_GNO_TYPE num_obj;
  int nrecv;
  indextype *ptr, *ptr1, *ptr2;
  indextype global_i, global_j;
  indextype *sendgno=NULL, *recvgno=NULL, *adjncy_sort=NULL;
  weighttype *sendwgt, *recvwgt;
  ZOLTAN_GNO_TYPE num_duplicates, num_singletons;
  ZOLTAN_GNO_TYPE num_selfs, nedges, global_sum, num_zeros;
  ZOLTAN_GNO_TYPE i, j, ii, k;
  MPI_Datatype zoltan_gno_mpi_type;

  ierr = ZOLTAN_OK;
  zoltan_gno_mpi_type = Zoltan_mpi_gno_type();

  /* Make sure all procs have same value of check_graph. */
  MPI_Allreduce(&check_graph, &i, 1, MPI_INT, MPI_MAX, comm);
  check_graph = i;

  if (check_graph == 0) /* perform no error checking at all */
     return ierr;

  /* Get number of procs and my rank */
  MPI_Comm_size(comm, &nprocs);
  MPI_Comm_rank(comm, &proc);

  /* Check number of vertices (objects) */
  num_obj = (ZOLTAN_GNO_TYPE)(vtxdist[proc+1] - vtxdist[proc]);
  MPI_Reduce(&num_obj, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
  if ((proc==0) && (global_sum==0)){
    if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
    if (output_level>0)
      ZOLTAN_PRINT_WARN(proc, yo, "No vertices in graph.");
  }

  /* Verify that vertex weights are non-negative */
  num_zeros = 0;
  if (vwgt_dim){
    for (i=0; i<num_obj; i++){
       sum = 0;
       for (k=0; k<vwgt_dim; k++){
         if (vwgt[i*vwgt_dim+k] < 0) {
            sprintf(msg, "Negative object weight of " TPL_WGT_SPEC " for object " ZOLTAN_GNO_SPEC ".", 
                    vwgt[i*vwgt_dim+k], i);
            ZOLTAN_PRINT_ERROR(proc, yo, msg);
            ierr = ZOLTAN_FATAL;
         }
         sum += vwgt[i*vwgt_dim+k];
       }
       if (sum == 0){
          num_zeros++;
          if (output_level>1) {
            sprintf(msg, "Zero vertex (object) weights for object " ZOLTAN_GNO_SPEC ".", i);
            ZOLTAN_PRINT_WARN(proc, yo, msg);
          }
          if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
       }
    }
    MPI_Reduce(&num_zeros, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
    if ((proc==0) && (global_sum>0)){
      if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
      if (output_level>0){
        sprintf(msg,  ZOLTAN_GNO_SPEC " objects have zero weights.", global_sum);
        ZOLTAN_PRINT_WARN(proc, yo, msg);
      }
    }
  }

  /* Check number of edges */
  nedges = (ZOLTAN_GNO_TYPE)xadj[num_obj];
  MPI_Reduce(&nedges, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
  if ((proc==0) && (global_sum==0)){
    if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
    if (output_level>0)
      ZOLTAN_PRINT_WARN(proc, yo, "No edges in graph.");
  }

  /* Verify that edge weights are non-negative */
  num_zeros = 0;
  if (ewgt_dim){
    for (j=0; j<nedges; j++){
      sum = 0;
      for (k=0; k<ewgt_dim; k++){
        if (adjwgt[j*ewgt_dim+k] < 0) {
          sprintf(msg, "Negative edge weight of " TPL_WGT_SPEC " in edge " ZOLTAN_GNO_SPEC ".", 
                  adjwgt[j*ewgt_dim+k], j);
          ZOLTAN_PRINT_ERROR(proc, yo, msg);
          ierr = ZOLTAN_FATAL;
        }
        sum += adjwgt[j*ewgt_dim+k];
      }
      if (sum == 0){
        num_zeros++;
        if (output_level>1) {
          sprintf(msg, "Zero edge (communication) weights for edge " ZOLTAN_GNO_SPEC ".", j);
          ZOLTAN_PRINT_WARN(proc, yo, msg);
        }
        if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
      }
    }

    MPI_Reduce(&num_zeros, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
    if ((proc==0) && (global_sum>0)){
      if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
      if (output_level>0){
        sprintf(msg,  ZOLTAN_GNO_SPEC " edges have zero weights.", global_sum);
        ZOLTAN_PRINT_WARN(proc, yo, msg);
      }
    }
  }

  /* Verify that the graph is symmetric (edge weights, too) */
  /* Also check for self-edges and multi-edges */

  /* Pre-processing: Check if edge lists are sorted. If not, */
  /* make a copy and sort so we can save time in the lookups. */
  flag = 0; /* Assume sorted. */
  for (i=0; (i<num_obj) && (flag==0); i++){
    for (ii=xadj[i]; ii<xadj[i+1]-1; ii++){
      if (adjncy[ii] > adjncy[ii+1]){
        flag = 1; /* Not sorted. */
        break; 
      }
    }
  }
  if (flag){ /* Need to sort. */
    adjncy_sort = (indextype *) ZOLTAN_MALLOC(nedges*sizeof(indextype));
    perm = (int *) ZOLTAN_MALLOC(nedges*sizeof(int));
    free_adjncy_sort = 1;
    if (nedges && (!adjncy_sort || !perm)){
      /* Out of memory. */
      ZOLTAN_PRINT_ERROR(proc, yo, "Out of memory.");
      ierr = ZOLTAN_MEMERR;
    }
    for (k=0; k<nedges; k++){
      adjncy_sort[k] = adjncy[k];
      perm[k] = k;
    }
    if (sizeof(indextype) == sizeof(short)){
      for (i=0; i<num_obj; i++) 
        Zoltan_quicksort_list_inc_short((short *)adjncy_sort, perm, (int)xadj[i], (int)xadj[i+1]-1);
    }
    else if (sizeof(indextype) == sizeof(int)){
      for (i=0; i<num_obj; i++) 
        Zoltan_quicksort_list_inc_int((int *)adjncy_sort, perm, (int)xadj[i], (int)xadj[i+1]-1);
    }
    else if (sizeof(indextype) == sizeof(long)){
      for (i=0; i<num_obj; i++) 
        Zoltan_quicksort_list_inc_long((long *)adjncy_sort, perm, (int)xadj[i], (int)xadj[i+1]-1);
    }
    else if (sizeof(indextype) == sizeof(int64_t)){
      for (i=0; i<num_obj; i++) 
        Zoltan_quicksort_list_inc_long_long((int64_t*)adjncy_sort, perm, (int)xadj[i], (int)xadj[i+1]-1);
    }
    else{
      ZOLTAN_PRINT_ERROR(proc, yo, "Error in third party library data type support.");
      ierr = ZOLTAN_MEMERR;
    }
  }
  else { /* Already sorted. */
    adjncy_sort = adjncy;
  }

  /* First pass: Check on-processor edges and count # off-proc edges */
  cross_edges = 0;
  num_selfs = 0;
  num_duplicates = 0;
  num_singletons = 0;
  for (i=0; i<num_obj; i++){
    if (IS_GLOBAL_GRAPH(graph_type)){
      global_i = vtxdist[proc]+i;
    }
    else{ /* graph_type == LOCAL_GRAPH */
      global_i = i; /* A bit confusingly, global_i = i for local graphs */
    }
    /* Singleton? */
    if (xadj[i] == xadj[i+1]){
      num_singletons++;
      if (output_level>1){
        sprintf(msg, "Vertex " TPL_IDX_SPEC " has no edges.", global_i);
        ZOLTAN_PRINT_WARN(proc, yo, msg);
      }
    }
    for (ii=xadj[i]; ii<xadj[i+1]; ii++){
      global_j = adjncy_sort[ii];
      /* Valid vertex number? */
      if ((IS_GLOBAL_GRAPH(graph_type) &&
           ((global_j < vtxdist[0]) || (global_j >= vtxdist[nprocs])))
          || (IS_LOCAL_GRAPH(graph_type) && 
           ((global_j < 0) || (global_j >= num_obj)))){
        sprintf(msg, "Edge to invalid vertex " TPL_IDX_SPEC " detected.", global_j);
        ZOLTAN_PRINT_ERROR(proc, yo, msg);
        ierr = ZOLTAN_FATAL;
      }
      /* Self edge? */
      if (global_j == global_i){
        num_selfs++;
        if (output_level>1){
          sprintf(msg, "Self edge for vertex " TPL_IDX_SPEC " detected.", global_i);
          ZOLTAN_PRINT_WARN(proc, yo, msg);
        }
      }
      /* Duplicate edge? */
      if ((ii+1<xadj[i+1]) && (adjncy_sort[ii]==adjncy_sort[ii+1])){
        num_duplicates++;
        if (output_level>1){
          sprintf(msg, "Duplicate edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ") detected.", global_i, global_j);
          ZOLTAN_PRINT_WARN(proc, yo, msg);
        }
      }
      /* Is global_j a local vertex? */
      if (IS_LOCAL_GRAPH(graph_type) || (IS_GLOBAL_GRAPH(graph_type) &&
          (global_j >= vtxdist[proc]) && (global_j < vtxdist[proc+1]))){
        /* Check if (global_j, global_i) is an edge */
        if (IS_GLOBAL_GRAPH(graph_type))
          j = global_j - vtxdist[proc];
        else /* graph_type == LOCAL_GRAPH */
          j = global_j;
        /* Binary search for edge (global_j, global_i) */
        ptr = (indextype *)bsearch(&global_i, &adjncy_sort[xadj[j]], (int)(xadj[j+1]-xadj[j]),
              sizeof(indextype), Zoltan_Compare_Indextypes);
        if (ptr){
          /* OK, found edge (global_j, global_i) */
          if ((adjncy_sort==adjncy) && ewgt_dim){
            /* Compare weights */
            /* EBEB For now, don't compare weights if we sorted edge lists. */
            flag = 0;
            for (k=0; k<ewgt_dim; k++){
              if (adjwgt[(ptr-adjncy_sort)*ewgt_dim+k] != 
                                           adjwgt[ii*ewgt_dim+k]){
                 /* Numerically nonsymmetric */
                 flag = -1;
                 if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
              }
            }
            if (flag<0 && output_level>0){
              sprintf(msg, "Graph is numerically nonsymmetric "
                "in edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ")", global_i, global_j);
              ZOLTAN_PRINT_WARN(proc, yo, msg);
            }
          }
        }
        else { /* bsearch failed */
          sprintf(msg, "Graph is not symmetric. "
                  "Edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ") exists, but no edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ").", 
                  global_i, global_j, global_j, global_i);
          ZOLTAN_PRINT_ERROR(proc, yo, msg);
          ierr = ZOLTAN_FATAL;
        }
      }
      else {
        cross_edges++;
      }
    }
  }

  /* Sum up warnings so far. */
  MPI_Reduce(&num_selfs, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
  if ((proc==0) && (global_sum>0)){
    ierr = ZOLTAN_WARN;
    if (output_level>0){
      sprintf(msg,  ZOLTAN_GNO_SPEC " self-edges in graph.", global_sum);
      ZOLTAN_PRINT_WARN(proc, yo, msg);
    }
  }
  MPI_Reduce(&num_duplicates, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
  if ((proc==0) && (global_sum>0)){
    ierr = ZOLTAN_WARN;
    if (output_level>0){
      sprintf(msg,  ZOLTAN_GNO_SPEC " duplicate edges in graph.", global_sum);
      ZOLTAN_PRINT_WARN(proc, yo, msg);
    }
  }
  MPI_Reduce(&num_singletons, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
  if ((proc==0) && (global_sum>0)){
    ierr = ZOLTAN_WARN;
    if (output_level>0){
      sprintf(msg,  ZOLTAN_GNO_SPEC " vertices in the graph are singletons (have no edges).", global_sum);
      ZOLTAN_PRINT_WARN(proc, yo, msg);
    }
  }
  
  
  /* Check if any processor has encountered an error so far */
  errors = 0;
  if (ierr == ZOLTAN_WARN)
    errors |= 1;
  if (ierr == ZOLTAN_MEMERR)
    errors |= 2;
  if (ierr == ZOLTAN_FATAL)
    errors |= 4;

  MPI_Allreduce(&errors, &global_errors, 1, MPI_INT, MPI_BOR, comm);

  if (global_errors & 4){
    /* Fatal error: return now */
    if (free_adjncy_sort) ZOLTAN_FREE(&adjncy_sort);
    if (free_adjncy_sort) ZOLTAN_FREE(&perm);
    return ZOLTAN_FATAL;
  }

  if ((IS_GLOBAL_GRAPH(graph_type)) && (check_graph >= 2)) {
    /* Test for consistency across processors. */

    /* Allocate space for off-proc data */
    mesg_size = (2*sizeof(indextype)) + (ewgt_dim * sizeof(weighttype));
    sendgno = (indextype *) ZOLTAN_MALLOC(cross_edges*mesg_size);
    recvgno = (indextype  *) ZOLTAN_MALLOC(cross_edges*mesg_size);
    proclist = (int *) ZOLTAN_MALLOC(cross_edges*sizeof(int));

    if (cross_edges && !(sendgno && recvgno && proclist)){
       ZOLTAN_PRINT_ERROR(proc, yo, "Out of memory.");
       ierr = ZOLTAN_MEMERR;
    }

    /* Second pass: Copy data to send buffer */
    nedges = 0;
    ptr1 = (indextype *) sendgno;
    for (i=0; i<num_obj; i++){
      global_i = vtxdist[proc]+i;
      for (ii=xadj[i]; ii<xadj[i+1]; ii++){
        global_j = adjncy[ii];
        /* Is global_j off-proc? */
        if ((global_j < vtxdist[proc]) || (global_j >= vtxdist[proc+1])){
           /* Add to list */
           k=0; 
           while (global_j >= vtxdist[k+1]) k++;
           proclist[nedges++] = k;
           /* Copy (global_i, global_j) and corresponding weights to sendgno */
           *ptr1++ = global_i;
           *ptr1++ = global_j;
           for (k=0; k<ewgt_dim; k++){
             *ptr1++ = adjwgt[ii*ewgt_dim+k];
           }
        }
      }
    }

    /* Do the irregular communication */
    ierr = Zoltan_Comm_Create(&comm_plan, cross_edges, proclist, comm, TAG1, &nrecv);
    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
      sprintf(msg, "Error %s returned from Zoltan_Comm_Create.", 
              (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
      ZOLTAN_PRINT_ERROR(proc, yo, msg);
    }
    else {
      if (nrecv != cross_edges){
        sprintf(msg, "Incorrect number of edges to/from proc %d.", proc);
        ZOLTAN_PRINT_ERROR(proc, yo, msg);
        ierr = ZOLTAN_FATAL;
      }

      ierr = Zoltan_Comm_Do(comm_plan, TAG2, (char *)sendgno, mesg_size, (char *)recvgno);
      Zoltan_Comm_Destroy(&comm_plan);
      if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
        sprintf(msg, "Error %s returned from Zoltan_Comm_Do.",
                (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
        ZOLTAN_PRINT_ERROR(proc, yo, msg);
      }
      else {

        /* Third pass: Compare on-proc data to the off-proc data we received */
        /* sendgno and recvgno should contain the same data except (i,j) is  */
        /* (j,i)                                                             */
        for (i=0, ptr1=sendgno; i<cross_edges; i++){
          flag = 0;
          sendwgt = (weighttype *)(ptr1 + 2);
          for (j=0, ptr2=recvgno; j<cross_edges; j++){
            recvwgt = (weighttype *)(ptr2 + 2);
            if ((ptr2[0] == ptr1[1]) && (ptr2[1] == ptr1[0])){
              /* Found matching edge */
              flag = 1;
              /* Check weights */
              for (k=0; k<ewgt_dim; k++){
                if (sendwgt[k] != recvwgt[k]){
                  flag = -1;
                  ierr = ZOLTAN_WARN;
                }
              }
              if (flag<0 && output_level>0){
                  sprintf(msg, "Edge weight (" TPL_IDX_SPEC "," TPL_IDX_SPEC ") is not symmetric",
                          ptr1[0], ptr1[1]);
                  ZOLTAN_PRINT_WARN(proc, yo, msg);
              }
            }
            ptr2 = (indextype *)(recvwgt + ewgt_dim);
          }
          if (!flag){
            sprintf(msg, "Graph is not symmetric.  "
                    "Edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ") exists, but not (" TPL_IDX_SPEC "," TPL_IDX_SPEC ").", 
                    ptr1[0], ptr1[1], ptr1[1], ptr1[0]);
            ZOLTAN_PRINT_ERROR(proc, yo, msg);
            ierr = ZOLTAN_FATAL;
          }
          ptr1 = (indextype *)(sendwgt + ewgt_dim);
        }
      }
    }

    /* Free memory */
    ZOLTAN_FREE(&sendgno);
    ZOLTAN_FREE(&recvgno);
    ZOLTAN_FREE(&proclist);
  }

  if (free_adjncy_sort) ZOLTAN_FREE(&adjncy_sort);
  if (free_adjncy_sort) ZOLTAN_FREE(&perm);

  /* Compute global error code */
  errors = 0;
  if (ierr == ZOLTAN_WARN)
    errors |= 1;
  if (ierr == ZOLTAN_MEMERR)
    errors |= 2;
  if (ierr == ZOLTAN_FATAL)
    errors |= 4;

  MPI_Allreduce(&errors, &global_errors, 1, MPI_INT, MPI_BOR, comm);

  if (global_errors & 4){
    return ZOLTAN_FATAL;
  }
  else if (global_errors & 2){
    return ZOLTAN_MEMERR;
  }
  else if (global_errors & 1){
    return ZOLTAN_WARN;
  }
  else {
    if (proc==0 && output_level>0){
      printf("ZOLTAN %s: The graph is valid with check_graph = %1d\n", 
             yo, check_graph);
    }
    return ZOLTAN_OK;
  }

}
コード例 #19
0
ファイル: phg_Vcycle.c プロジェクト: xunzhang/ESMF_Regridding
/*  Main partitioning function for hypergraph partitioning. */
int Zoltan_PHG_Partition (
  ZZ *zz,               /* Zoltan data structure */
  HGraph *hg,           /* Input hypergraph to be partitioned */
  int p,                /* Input:  number partitions to be generated */
  float *part_sizes,    /* Input:  array of length p containing percentages
                           of work to be assigned to each partition */
  Partition parts,      /* Input:  initial partition #s; aligned with vtx 
                           arrays. 
                           Output:  computed partition #s */
  PHGPartParams *hgp,   /* Input:  parameters for hgraph partitioning. */
  int level)
{

  PHGComm *hgc = hg->comm;
  VCycle  *vcycle=NULL, *del=NULL;
  int  i, err = ZOLTAN_OK;
  int  prevVcnt     = 2*hg->dist_x[hgc->nProc_x];
  int  prevVedgecnt = 2*hg->dist_y[hgc->nProc_y];
  char *yo = "Zoltan_PHG_Partition";
  static int timer_match = -1,    /* Timers for various stages */
             timer_coarse = -1,   /* Declared static so we can accumulate */
             timer_refine = -1,   /* times over calls to Zoltan_PHG_Partition */
             timer_coarsepart = -1,
             timer_project = -1,
             timer_vcycle = -1;   /* times everything in Vcycle not included
                                     in above timers */
  int do_timing = (hgp->use_timers > 1);
  int vcycle_timing = (hgp->use_timers > 4);

  ZOLTAN_TRACE_ENTER(zz, yo);
    
  if (do_timing) {
    if (timer_vcycle < 0) 
      timer_vcycle = Zoltan_Timer_Init(zz->ZTime, 0, "Vcycle");
    if (timer_match < 0) 
      timer_match = Zoltan_Timer_Init(zz->ZTime, 1, "Matching");
    if (timer_coarse < 0) 
      timer_coarse = Zoltan_Timer_Init(zz->ZTime, 1, "Coarsening");
    if (timer_coarsepart < 0)
      timer_coarsepart = Zoltan_Timer_Init(zz->ZTime, 1,
                                           "Coarse_Partition");
    if (timer_refine < 0) 
      timer_refine = Zoltan_Timer_Init(zz->ZTime, 1, "Refinement");
    if (timer_project < 0) 
      timer_project = Zoltan_Timer_Init(zz->ZTime, 1, "Project_Up");

    ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
  }

  if (!(vcycle = newVCycle(zz, hg, parts, NULL, vcycle_timing))) {
    ZOLTAN_PRINT_ERROR (zz->Proc, yo, "VCycle is NULL.");
    return ZOLTAN_MEMERR;
  }

  /****** Coarsening ******/    
#define COARSEN_FRACTION_LIMIT 0.9  /* Stop if we don't make much progress */
  while ((hg->redl>0) && (hg->dist_x[hgc->nProc_x] > hg->redl)
    && ((hg->dist_x[hgc->nProc_x] < (int) (COARSEN_FRACTION_LIMIT * prevVcnt + 0.5))
     || (hg->dist_y[hgc->nProc_y] < (int) (COARSEN_FRACTION_LIMIT * prevVedgecnt + 0.5)))
    && hg->dist_y[hgc->nProc_y] && hgp->matching) {
      int *match = NULL;
      VCycle *coarser=NULL;
        
      prevVcnt     = hg->dist_x[hgc->nProc_x];
      prevVedgecnt = hg->dist_y[hgc->nProc_y];

#ifdef _DEBUG      
      /* UVC: load balance stats */
      Zoltan_PHG_LoadBalStat(zz, hg);
#endif
      
      if (hgp->output_level >= PHG_DEBUG_LIST) {
          uprintf(hgc,
                  "START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
                  hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl, 
                  hgp->redm_str,
                  hgp->coarsepartition_str, hgp->refinement_str, p);
          if (hgp->output_level > PHG_DEBUG_LIST) {
              err = Zoltan_HG_Info(zz, hg);
              if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
                  goto End;
          }
      }
      if (hgp->output_level >= PHG_DEBUG_PLOT)
        Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
         "coarsening plot");

      if (do_timing) {
        ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
        ZOLTAN_TIMER_START(zz->ZTime, timer_match, hgc->Communicator);
      }
      if (vcycle_timing) {
        if (vcycle->timer_match < 0) {
          char str[80];
          sprintf(str, "VC Matching %d", hg->info);
          vcycle->timer_match = Zoltan_Timer_Init(vcycle->timer, 0, str);
        }
        ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_match,
                           hgc->Communicator);
      }

      /* Allocate and initialize Matching Array */
      if (hg->nVtx && !(match = (int*) ZOLTAN_MALLOC (hg->nVtx*sizeof(int)))) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: Matching array");
        return ZOLTAN_MEMERR;
      }
      for (i = 0; i < hg->nVtx; i++)
        match[i] = i;
        
      /* Calculate matching (packing or grouping) */
      err = Zoltan_PHG_Matching (zz, hg, match, hgp);
      if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
        ZOLTAN_FREE ((void**) &match);
        goto End;
      }
      if (vcycle_timing)
        ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_match,
                          hgc->Communicator);

      if (do_timing) {
        ZOLTAN_TIMER_STOP(zz->ZTime, timer_match, hgc->Communicator);
        ZOLTAN_TIMER_START(zz->ZTime, timer_coarse, hgc->Communicator);
      }

      if (vcycle_timing) {
        if (vcycle->timer_coarse < 0) {
          char str[80];
          sprintf(str, "VC Coarsening %d", hg->info);
          vcycle->timer_coarse = Zoltan_Timer_Init(vcycle->timer, 0, str);
        }
        ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_coarse,
                           hgc->Communicator);
      }
            
      if (!(coarser = newVCycle(zz, NULL, NULL, vcycle, vcycle_timing))) {
        ZOLTAN_FREE ((void**) &match);
        ZOLTAN_PRINT_ERROR (zz->Proc, yo, "coarser is NULL.");
        goto End;
      }

      /* Construct coarse hypergraph and LevelMap */
      err = Zoltan_PHG_Coarsening (zz, hg, match, coarser->hg, vcycle->LevelMap,
       &vcycle->LevelCnt, &vcycle->LevelSndCnt, &vcycle->LevelData, 
       &vcycle->comm_plan, hgp);
      if (err != ZOLTAN_OK && err != ZOLTAN_WARN) 
        goto End;

      if (vcycle_timing)
        ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_coarse,
                          hgc->Communicator);
        
      if (do_timing) {
        ZOLTAN_TIMER_STOP(zz->ZTime, timer_coarse, hgc->Communicator);
        ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
      }

      ZOLTAN_FREE ((void**) &match);

      if ((err=allocVCycle(coarser))!= ZOLTAN_OK)
        goto End;
      vcycle = coarser;
      hg = vcycle->hg;
  }

  if (hgp->output_level >= PHG_DEBUG_LIST) {
    uprintf(hgc, "START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
     hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl, 
     hgp->redm_str, hgp->coarsepartition_str, hgp->refinement_str, p);
    if (hgp->output_level > PHG_DEBUG_LIST) {
      err = Zoltan_HG_Info(zz, hg);
      if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
        goto End;
    }
  }
  if (hgp->output_level >= PHG_DEBUG_PLOT)
    Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
     "coarsening plot");

  /* free array that may have been allocated in matching */
  if (hgp->vtx_scal) ZOLTAN_FREE(&(hgp->vtx_scal));

  if (do_timing) {
    ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
    ZOLTAN_TIMER_START(zz->ZTime, timer_coarsepart, hgc->Communicator);
  }

  /****** Coarse Partitioning ******/
  err = Zoltan_PHG_CoarsePartition (zz, hg, p, part_sizes, vcycle->Part, hgp);
  if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
    goto End;

  if (do_timing) {
    ZOLTAN_TIMER_STOP(zz->ZTime, timer_coarsepart, hgc->Communicator);
    ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
  }

  del = vcycle;
  /****** Uncoarsening/Refinement ******/
  while (vcycle) {
    VCycle *finer = vcycle->finer;
    hg = vcycle->hg;

    if (do_timing) {
      ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
      ZOLTAN_TIMER_START(zz->ZTime, timer_refine, hgc->Communicator);
    }
    if (vcycle_timing) {
      if (vcycle->timer_refine < 0) {
        char str[80];
        sprintf(str, "VC Refinement %d", hg->info);
        vcycle->timer_refine = Zoltan_Timer_Init(vcycle->timer, 0, str);
      }
      ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_refine,
                         hgc->Communicator);
    }

    err = Zoltan_PHG_Refinement (zz, hg, p, part_sizes, vcycle->Part, hgp);
        
    if (do_timing) {
      ZOLTAN_TIMER_STOP(zz->ZTime, timer_refine, hgc->Communicator);
      ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
    }
    if (vcycle_timing)
      ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_refine,
                        hgc->Communicator);

                          
    if (hgp->output_level >= PHG_DEBUG_LIST)     
      uprintf(hgc, 
              "FINAL %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d bal=%.2f cutl=%.2f\n",
              hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl, 
              hgp->redm_str,
              hgp->coarsepartition_str, hgp->refinement_str, p,
              Zoltan_PHG_Compute_Balance(zz, hg, part_sizes, p, vcycle->Part),
              Zoltan_PHG_Compute_ConCut(hgc, hg, vcycle->Part, p, &err));

    if (hgp->output_level >= PHG_DEBUG_PLOT)
      Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, vcycle->Part,
       "partitioned plot");
        
    if (do_timing) {
      ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
      ZOLTAN_TIMER_START(zz->ZTime, timer_project, hgc->Communicator);
    }
    if (vcycle_timing) {
      if (vcycle->timer_project < 0) {
        char str[80];
        sprintf(str, "VC Project Up %d", hg->info);
        vcycle->timer_project = Zoltan_Timer_Init(vcycle->timer, 0, str);
      }
      ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_project,
                         hgc->Communicator);
    }

    /* Project coarse partition to fine partition */
    if (finer)  { 
      int *rbuffer;
            
      /* easy to undo internal matches */
      for (i = 0; i < finer->hg->nVtx; i++)
        if (finer->LevelMap[i] >= 0)
          finer->Part[i] = vcycle->Part[finer->LevelMap[i]];
          
      /* fill sendbuffer with part data for external matches I owned */    
      for (i = 0; i < finer->LevelCnt; i++)  {
        ++i;          /* skip return lno */
        finer->LevelData[i] = finer->Part[finer->LevelData[i]]; 
      }
            
      /* allocate rec buffer */
      rbuffer = NULL;
      if (finer->LevelSndCnt > 0)  {
        rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->LevelSndCnt * sizeof(int));
        if (!rbuffer)    {
          ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory.");
          return ZOLTAN_MEMERR;
        }
      }       
      
      /* get partition assignments from owners of externally matchted vtxs */  
      Zoltan_Comm_Resize (finer->comm_plan, NULL, COMM_TAG, &i);
      Zoltan_Comm_Do_Reverse (finer->comm_plan, COMM_TAG+1, 
       (char*) finer->LevelData, 2 * sizeof(int), NULL, (char*) rbuffer);

      /* process data to undo external matches */
      for (i = 0; i < 2 * finer->LevelSndCnt;)  {
        int lno, partition;
        lno       = rbuffer[i++];
        partition = rbuffer[i++];      
        finer->Part[lno] = partition;         
      }

      ZOLTAN_FREE (&rbuffer);                  
      Zoltan_Comm_Destroy (&finer->comm_plan);                   
    }
    if (do_timing) {
      ZOLTAN_TIMER_STOP(zz->ZTime, timer_project, hgc->Communicator);
      ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
    }
    if (vcycle_timing)
      ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_project,
                        hgc->Communicator);

    vcycle = finer;
  }       /* while (vcycle) */
    
End:
  vcycle = del;
  while (vcycle) {
    if (vcycle_timing) {
      Zoltan_Timer_PrintAll(vcycle->timer, 0, hgc->Communicator, stdout);
      Zoltan_Timer_Destroy(&vcycle->timer);
    }
    if (vcycle->finer) {   /* cleanup by level */
      Zoltan_HG_HGraph_Free (vcycle->hg);
      Zoltan_Multifree (__FILE__, __LINE__, 4, &vcycle->Part, &vcycle->LevelMap,
                        &vcycle->LevelData, &vcycle->hg);
    }
    else                   /* cleanup top level */
      Zoltan_Multifree (__FILE__, __LINE__, 2, &vcycle->LevelMap,
                        &vcycle->LevelData);
    del = vcycle;
    vcycle = vcycle->finer;
    ZOLTAN_FREE(&del);
  }

  if (do_timing)
    ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
  ZOLTAN_TRACE_EXIT(zz, yo) ;
  return err;
}
コード例 #20
0
ファイル: lb_invert.c プロジェクト: askhl/octopus-dfrt2
int Zoltan_Invert_Lists(
  ZZ *zz,                       /* Zoltan structure.                  */
  int num_in,                   /* Number of objects in the input lists. */ 
  ZOLTAN_ID_PTR in_global_ids,  /* Array of input global IDs. */
  ZOLTAN_ID_PTR in_local_ids,   /* Array of input local IDs. */
  int *in_procs,                /* Array of processor IDs of processors owning
                                   the input objects. */
  int *in_to_part,              /* Optional:  Array of partition numbers to 
                                   which input objects should be assigned. */
  int *num_out,                 /* Returned value:  Number of output objs. */
  ZOLTAN_ID_PTR *out_global_ids,/* Returned value:  Array of global IDs of
                                   output objects. */
  ZOLTAN_ID_PTR *out_local_ids, /* Returned value:  Array of local IDs of
                                   output objects. */
  int **out_procs,              /* Returned value:  Array of processor IDs
                                   to which output objects are assigned. */
  int **out_to_part             /* Optional:  Returned value:  Array of 
                                   partition numbers to which output
                                   objects should be assigned. */
)
{
/*
 *  Routine to compute the inverse map.  Can be used in two ways:
 *  1.  Given, for each processor, a list of objects to be received by the 
 *  processor, compute the list of objects that the processor needs to send 
 *  to other processors to satisfy their needs.
 *  2.  Given, for each processor, a list of objects to be sent to other 
 *  processors, compute the list of objects that the processor needs to receive
 *  to satisfy its needs.
 */

char *yo = "Zoltan_Invert_Lists";
char msg[256];
ZOLTAN_COMM_OBJ *comm_plan;        /* Object returned communication routines  */
int msgtag, msgtag2;               /* Message tags for communication routines */
int num_gid_entries, num_lid_entries;  /* Length of global and local ids */
int include_parts;                 /* Flag indicating whether to compute
                                      inverse list for partitions. */
int ierr, ret_ierr = ZOLTAN_OK;

  ZOLTAN_TRACE_ENTER(zz, yo);
  /*
   *  Return if this processor is not in the Zoltan structure's
   *  communicator.
   */

  if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    return (ZOLTAN_OK);
  }

  /*
   *  Check that all procs use the same id types.
   */

  ierr = check_invert_input(zz, num_in, in_procs, in_to_part, 
                            &num_gid_entries, &num_lid_entries, &include_parts);
  if (ierr != ZOLTAN_OK) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    return ierr;
  }

  /* Initialize returned arrays. */
  *out_global_ids = NULL;
  *out_local_ids = NULL;
  *out_procs = NULL;
  if (include_parts) *out_to_part = NULL;


  /*
   *  Compute communication map and num_out, the number of objs this
   *  processor has to out to establish the new decomposition.
   */

  msgtag = 32767;
  ierr = Zoltan_Comm_Create(&comm_plan, num_in, in_procs, zz->Communicator, 
                        msgtag, num_out);
  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    sprintf(msg, "Error %s returned from Zoltan_Comm_Create.",
            (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
    ret_ierr = ierr;
    goto End;
  }
  

  ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm create");

  /*
   *  Allocate space for the object tags that need to be outed.  Communicate
   *  to get the list of objects to be outed.
   */

  if (*num_out > 0) {
    if (!Zoltan_Special_Malloc(zz,(void **)out_global_ids,*num_out,
                           ZOLTAN_SPECIAL_MALLOC_GID)) {
      ret_ierr = ZOLTAN_MEMERR;
      goto End;
    }
    if (!Zoltan_Special_Malloc(zz,(void **)out_local_ids,*num_out,
                           ZOLTAN_SPECIAL_MALLOC_LID)) {
      ret_ierr = ZOLTAN_MEMERR;
      goto End;
    }
    if (!Zoltan_Special_Malloc(zz,(void **)out_procs,*num_out,
                           ZOLTAN_SPECIAL_MALLOC_INT)) {
      ret_ierr = ZOLTAN_MEMERR;
      goto End;
    }
    if (include_parts) {
      if (!Zoltan_Special_Malloc(zz,(void **)out_to_part,*num_out,
                             ZOLTAN_SPECIAL_MALLOC_INT)) {
        ret_ierr = ZOLTAN_MEMERR;
        goto End;
      }
    }
  }

  /*
   *  Use the communication plan to send global IDs, local IDs, and processor
   *  numbers.  Do in separate communications to avoid a memory copy and to
   *  simplify implementation when a data type is added to the comm. package
   *  (to support heterogeneous computing).
   */

  msgtag2 = 32766;
  ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_global_ids, 
                    (int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)), 
                    (char *) *out_global_ids);
  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", 
            (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
    ret_ierr = ierr;
  }

  if (num_lid_entries) {
    msgtag2--;
    ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_local_ids, 
                      (int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries), 
                      (char *) *out_local_ids);
    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
      sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", 
              (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
      ret_ierr = ierr;
    }
  }

  Zoltan_Comm_Info(comm_plan, NULL, NULL, NULL, NULL, NULL, NULL,
                   NULL, NULL, NULL, NULL, NULL, *out_procs, NULL);
  
  if (include_parts) {
    msgtag2--;
    ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_to_part, 
                      (int) sizeof(int), (char *) *out_to_part);
    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
      sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", 
              (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
      ret_ierr = ierr;
    }
  }

  ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm_do");

End:

  Zoltan_Comm_Destroy(&comm_plan);

  if (ret_ierr == ZOLTAN_MEMERR) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
    Zoltan_Special_Free(zz,(void**)out_global_ids,ZOLTAN_SPECIAL_MALLOC_GID);
    Zoltan_Special_Free(zz,(void**)out_local_ids,ZOLTAN_SPECIAL_MALLOC_LID);
    Zoltan_Special_Free(zz,(void**)out_procs,ZOLTAN_SPECIAL_MALLOC_INT);
    if (include_parts)
      Zoltan_Special_Free(zz,(void**)out_to_part,ZOLTAN_SPECIAL_MALLOC_INT);
  }
  ZOLTAN_TRACE_EXIT(zz, yo);
  return (ret_ierr);
}
コード例 #21
0
ファイル: DD_Remove.c プロジェクト: agrippa/Trilinos
int Zoltan_DD_Remove (
 Zoltan_DD_Directory *dd,            /* directory state infomation      */
 ZOLTAN_ID_PTR gid,                  /* Incoming list of GIDs to remove */
 int count)                          /* Number of GIDs in removal list  */
{
   int             *procs = NULL;   /* list of processors to contact   */
   DD_Remove_Msg   *ptr   = NULL;
   ZOLTAN_COMM_OBJ *plan  = NULL;   /* efficient MPI communication     */
   char            *sbuff = NULL;   /* send buffer                     */
   char            *sbufftmp = NULL;/* pointer into send buffer        */
   char            *rbuff = NULL;   /* receive buffer                  */
   char            *rbufftmp = NULL;/* pointer into receive buffer     */

   int              nrec;           /* number of receives to expect    */
   int              i;
   int              err;            /* error condition to return       */
   int              errcount;       /* count of GIDs not found         */
   char             str[100];       /* string to build error messages  */
   char            *yo = "Zoltan_DD_Remove";



   /* input sanity checks */
   if (dd == NULL || count < 0 || (gid == NULL && count > 0)) {
      ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo,
       "Invalid input argument");
      return ZOLTAN_FATAL;
   }
   if (dd->debug_level > 4)
      ZOLTAN_TRACE_IN (dd->my_proc, yo, NULL);


   /* allocate memory for processor contact list */
   if (count)  {
      procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count);
      if (procs == NULL)  {
         ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list");
         if (dd->debug_level > 4)
            ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL);
         return ZOLTAN_MEMERR;
      }
   }

   /* allocate memory for DD_Remove_Msg send buffer */
   if (count) {
      sbuff = (char*)ZOLTAN_MALLOC((size_t)(dd->remove_msg_size)*(size_t)count);
      if (sbuff == NULL)  {
         ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer");
         err = ZOLTAN_MEMERR;
         goto fini;
      }
   }

   if (dd->debug_level > 6)
      ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After proc & sbuff mallocs");

   /* for each GID, fill in contact list and then message structure */
   sbufftmp = sbuff;
   for (i = 0; i < count; i++)  {
      procs[i] = dd->hash(gid + i*dd->gid_length, dd->gid_length, dd->nproc,
                          dd->hashdata, dd->hashfn);
      ptr = (DD_Remove_Msg*) sbufftmp;
      sbufftmp += dd->remove_msg_size;
      ptr->owner = dd->my_proc;
      ZOLTAN_SET_ID (dd->gid_length, ptr->gid, gid + i * dd->gid_length);
   }

   /* now create efficient communication plan */
   err = Zoltan_Comm_Create (&plan, count, procs, dd->comm,
    ZOLTAN_DD_REMOVE_MSG_TAG, &nrec);
   if (err != ZOLTAN_OK)  {
      ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Create error");
      goto fini;
   }
   if (dd->debug_level > 6)
      ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After Zoltan_Comm_Create");

   /* allocate receive buffer for nrec DD_Remove_Msg structures */
   if (nrec) {
      rbuff = (char*)ZOLTAN_MALLOC((size_t)nrec*(size_t)(dd->remove_msg_size));
      if (rbuff == NULL)  {
         ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Receive buffer malloc failed");
         err = ZOLTAN_MEMERR;
         goto fini;
      }
   }

   /* send my remove messages & receive removes directed to me */
   err = Zoltan_Comm_Do (plan, ZOLTAN_DD_UPDATE_MSG_TAG+1, sbuff,
    dd->remove_msg_size, rbuff);
   if (err != ZOLTAN_OK) {
      ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Do error");
      goto fini;
   }
   if (dd->debug_level > 6)
      ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After Zoltan_Comm_Do");

   /* for each message rec'd,  remove local directory info */
   errcount = 0;
   rbufftmp = rbuff;
   for (i = 0; i < nrec; i++)  {
      ptr = (DD_Remove_Msg*) rbufftmp;
      rbufftmp += dd->remove_msg_size;

      err = DD_Remove_Local (dd, ptr->gid);
      if (err == ZOLTAN_WARN)
         ++errcount;
   }

   err = ZOLTAN_OK;
   if (dd->debug_level)  {
      sprintf (str, "Processed %d GIDs (%d local), %d GIDs not found",
       count, nrec, errcount);
      ZOLTAN_PRINT_INFO (dd->my_proc, yo, str);
      err = (errcount) ? ZOLTAN_WARN : ZOLTAN_OK;
   }

   /* done, now free up things and return */
 fini:
   ZOLTAN_FREE (&procs);
   ZOLTAN_FREE (&sbuff);
   ZOLTAN_FREE (&rbuff);
   Zoltan_Comm_Destroy (&plan);

   if (dd->debug_level > 4)
      ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL);
   return err;
}
コード例 #22
0
ファイル: zcol.c プロジェクト: haripandey/trilinos
int main(int argc, char *argv[])
{
    int rc, i, ngids, maxcol, ncolors;
    float ver;
    struct Zoltan_Struct *zz=NULL;
#ifdef ZOLTANV31
    int numGidEntries, numLidEntries;
#else
    ZOLTAN_GRAPH_EVAL graph;
#endif
    int *color;
    ZOLTAN_ID_PTR gid_list;
    UZData guz, *uz=&guz;
    int msg_tag = 9999;
    int nlvtx, next, maxdeg=0;
    double times[9]={0.,0.,0.,0.,0.,0.,0.,0.}; /* Used for timing measurements */
    double gtimes[9]={0.,0.,0.,0.,0.,0.,0.,0.}; /* Used for timing measurements */
    
    /******************************************************************
     ** Initialize MPI and Zoltan
     ******************************************************************/

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &uz->myRank);
    MPI_Comm_size(MPI_COMM_WORLD, &uz->numProcs);

    MPI_Barrier(MPI_COMM_WORLD);
    times[0] = u_wseconds();
    
    rc = Zoltan_Initialize(argc, argv, &ver);
    if (rc != ZOLTAN_OK){
        fprintf(stderr, "Sorry Zoltan initialize failed...\n");
        goto End;
    }
    zz = Zoltan_Create(MPI_COMM_WORLD);

    if (argc<3 && !uz->myRank) {
        fprintf(stderr, "usage: %s [meshR] [meshC] [X-point stencil] [procR] [procC] [ws-beta] [<ZoltanParam>=<Val>] ...\n\n", argv[0]);
        fprintf(stderr, "ws-beta: is the probablity of adding an edge to a vertex to generate Watts-Strogatz graphs\n");
        fprintf(stderr, "Valid values for Stencil are 5, 7 and 9\n");
        fprintf(stderr, "Zoltan Coloring Parameters and values are\n");
        fprintf(stderr, "\tDISTANCE        : 1 or 2\n");
        fprintf(stderr, "\tSUPERSTEP_SIZE  : suggested >= 100\n"); 
        fprintf(stderr, "\tCOMM_PATTERN    : S or A\n");
        fprintf(stderr, "\tCOLOR_ORDER     : I, B, U\n");
        fprintf(stderr, "\tCOLORING_METHOD : F (for now)\n");
        fprintf(stderr, "\n");
    }

    uz->procR = uz->procC = 0;
    uz->meshR = uz->meshC = 1024;
    uz->stencil = 9;

    if (argc>1)
        uz->meshR = atoi(argv[1]);
    if (argc>2)
        uz->meshC = atoi(argv[2]);
    if (argc>3)
        uz->stencil = atoi(argv[3]);
    if (uz->stencil!=5 && uz->stencil!=7 && uz->stencil!=9) {
        fprintf(stderr, "\t invalid stencil value. Valid values are 5, 7 and 9. Assumed 9.\n");
        uz->stencil = 9;
    }
    --uz->stencil;

    if (argc>4)
        uz->procR = atoi(argv[4]);
    if (argc>5)
        uz->procC = atoi(argv[5]);
    if (uz->procR <= 0 || uz->procC <= 0)
        computeProcMesh(uz);
    
    if (uz->procR*uz->procC!=uz->numProcs) {
        fprintf(stderr, "#Procs=%d but requested %dx%d Proc Mesh Partitioning...\n", uz->numProcs, uz->procR, uz->procC);
        goto End;
    }

    if (argc>6)
        uz->beta = atof(argv[6]);
    else
        uz->beta = 0.0;
    
    /* compute which part of mesh I will compute */
    uz->myR = uz->myRank / uz->procC;
    uz->myC = uz->myRank % uz->procC;

    uz->_sr = uz->myR * (uz->meshR / uz->procR);
    uz->_er = (uz->myR+1) * (uz->meshR / uz->procR);
    if (uz->_er>uz->meshR)
        uz->_er = uz->meshR;
    uz->_sc = uz->myC * (uz->meshC / uz->procC);
    uz->_ec = (uz->myC+1) * (uz->meshC / uz->procC);
    if (uz->_ec>uz->meshC)
        uz->_ec = uz->meshC;


    if ( (uz->meshR % uz->procR) !=0 || (uz->meshC % uz->procC)!=0) {
        printf("Mesh dimensions are not divisible with proc mesh.\nRequested mesh is %dx%d and proc mesh is %d x %d\n", uz->meshR, uz->meshC, uz->procR, uz->procC);
        exit(1);
    }
    nlvtx= (uz->_er-uz->_sr) * (uz->_ec-uz->_sc);

    if (uz->myRank==0)
        printf("Running %s on %d x %d processor mesh, generating %d-point %d x %d mesh with beta=%.3lf\n", argv[0], uz->procR, uz->procC, uz->stencil+1, uz->meshR, uz->meshC, uz->beta);

    times[1] = u_wseconds();    
    uz->numredge = 0;
    uz->redgeto = NULL;
    if (uz->beta>0) { /* create random edges for WS graph */
        int ngvtx=uz->meshC*uz->meshR, trsh=(int) (uz->beta*100.0);
        int ierr=0;
        int *edges=NULL, *redges=NULL, *proclist=NULL, nedge;
        ZOLTAN_COMM_OBJ *plan;
            
        uz->redgeto = (int *) malloc(nlvtx*sizeof(int));
        for (i=0; i<nlvtx; ++i) {
            int rv = Zoltan_Rand_InRange(NULL, 100);
            if ( rv < trsh) {
                if ((uz->redgeto[i] = Zoltan_Rand_InRange(NULL,  ngvtx))==gIDfLID(i)) /* is it a self edge */
                    uz->redgeto[i] = -1;
                else 
                    ++uz->numredge;
            } else
                uz->redgeto[i] = -1;
        }

        edges = (int *) malloc(sizeof(int)*2*uz->numredge);
        proclist = (int *) malloc(sizeof(int)*uz->numredge);
        next = 0;
        for (i=0; i<nlvtx; ++i)
            if (uz->redgeto[i]>0) {
                edges[2*next] = uz->redgeto[i];
                edges[2*next+1] = gIDfLID(i);
                proclist[next] = pIDfGID(uz->redgeto[i]);
                ++next;
            }

        ierr = Zoltan_Comm_Create(&plan, uz->numredge, proclist, MPI_COMM_WORLD,
                                  msg_tag, &nedge);
        
        redges = (int *) malloc(sizeof(int)*2*nedge);
        
        --msg_tag;
        ierr |= Zoltan_Comm_Do(plan, msg_tag, (char *) edges, 2*sizeof(int),
                               (char *) redges);
        ierr |= Zoltan_Comm_Destroy(&plan);
        free(proclist);
        free(edges);
                
        if (ierr) {
            printf("error while communicating edges!\n");
            exit(1);
        }
        xadj = (int *) calloc(1+nlvtx, sizeof(int));
        adj = (int *) malloc(sizeof(int)*nedge);
        for (i=0; i<nedge; ++i)  {
            if (redges[2*i] < gID(uz->_sr, uz->_sc) || redges[2*i] >= gID(uz->_er, uz->_ec)) {
                printf("[%d/%d] received gid=%d doesn't blong to processor range [%d, %d) should go to proc %d\n", uz->myRank, uz->numProcs, redges[2*i],  gID(uz->_sr, uz->_sc), gID(uz->_er, uz->_ec), pIDfGID(redges[2*i]));
            }
                
            ++xadj[lIDfGID(redges[2*i])];
        }
        xadj[nlvtx] = nedge;
        maxdeg = xadj[0];
        for (i=1; i<nlvtx; ++i) {
            maxdeg = xadj[i]>maxdeg ? xadj[i] : maxdeg;
            xadj[i] += xadj[i-1];
        }
        
        for (i=0; i<nedge; ++i) {
            int u = lIDfGID(redges[2*i]);
            int v = redges[2*i+1];
            adj[--xadj[u]] = v;
        }
        free(redges);                    
    }
    maxdeg += uz->stencil+1;
    adjTemp = (int *) malloc(sizeof(int)*2*maxdeg);
    times[2] = u_wseconds();
    
    /*
      printf("My rank %d/%d at proc-mesh loc (%d, %d) generating [%d, %d) x [%d, %d) + %d random edges TotEdge=%d\n", uz->myRank, uz->numProcs, uz->myR, uz->myC, uz->_sr, uz->_er, uz->_sc, uz->_ec, uz->numredge, xadj[nlvtx]);  */
    printStats("Number of Vertices  ", nlvtx, uz->myRank, uz->numProcs);
    if (xadj)
        printStats("Number of Rand Edges", xadj[nlvtx], uz->myRank, uz->numProcs);

    
    /* General parameters */
#ifndef ZOLTANV31
    Zoltan_Set_Param(zz, "GRAPH_BUILD_TYPE", "FAST_NO_DUP");
#endif

    /* General parameters */
    Zoltan_Set_Param(zz, "DEBUG_LEVEL", "3");
    Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1"); 
    Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
    Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "0");


    /* coloring parameters */
    Zoltan_Set_Param(zz, "SUPERSTEP_SIZE", "500"); /* let's make S=500 default */
    for (i=7; i<argc; ++i) {
        char param[256], *eq;

        if (!uz->myRank)
            printf("processing argv[%d]='%s'\n", i, argv[i]);
        strncpy(param, argv[i], sizeof(param));
        eq = strchr(param, '=');
        if (!eq) {
            fprintf(stderr, "invalid argument '%s', Zoltan Paramters should be in the format <ZoltanParam>=<Val>\n", param);
            goto End;
        }
        *eq = 0;
        Zoltan_Set_Param(zz, param, eq+1);
    }


#if 0    
    /* Graph parameters */
    Zoltan_Set_Param(zz, "CHECK_GRAPH", "2");
#endif

    /* set call backs */
    Zoltan_Set_Num_Obj_Fn(zz, get_number_of_objects, uz);
    Zoltan_Set_Obj_List_Fn(zz, get_object_list, uz);
    Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, uz);
    Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, uz);

#if 0    
#ifndef ZOLTANV31
    Zoltan_LB_Eval_Graph(zz, 0, &graph);

    if (!uz->myRank) {
        printf("EdgeCut   Min=%8.0f  Max=%8.0f  Sum=%8.0f\n", graph.cuts[EVAL_GLOBAL_MIN], graph.cuts[EVAL_GLOBAL_MAX], graph.cuts[EVAL_GLOBAL_SUM]);
        printf("#Vertices Min=%8.0f  Max=%8.0f  Sum=%8.0f imbal=%.2f\n", graph.nobj[EVAL_GLOBAL_MIN], graph.nobj[EVAL_GLOBAL_MAX], graph.nobj[EVAL_GLOBAL_SUM], graph.obj_imbalance);        
    }
#endif
#endif

    /* now color */
    ngids = get_number_of_objects(uz, &rc);

    gid_list = (ZOLTAN_ID_PTR) malloc(sizeof(ZOLTAN_ID_TYPE) * ngids);
#ifndef ZOLTANV31
    next = 0;
    for (i=uz->_sr; i<uz->_er; ++i) {
        int j;
        for (j=uz->_sc; j<uz->_ec; ++j) {
            gid_list[next++] = i*uz->meshC + j;
        }
    }    
#endif
    color = (int *) malloc(sizeof(int) * ngids);    

    MPI_Barrier(MPI_COMM_WORLD);
    times[3] = u_wseconds();    
#ifdef ZOLTANV31
    rc = Zoltan_Color(zz, /* input (all remaining fields are output) */
                      &numGidEntries,  /* Number of integers used for a global ID */
                      &numLidEntries,  /* Number of integers used for a local ID */
                      ngids,           /* #objects to color in this proc */
                      gid_list,        /* global ids of colored vertices */
                      NULL,            /* we ignore local ids */
                      color);          /* result color */    
#else    
    rc = Zoltan_Color(zz, /* input (all remaining fields are output) */
                      1,  /* Number of integers used for a global ID */
                      ngids,           /* #objects to color in this proc */
                      gid_list,        /* global ids of colored vertices */
                      color);          /* result color */
#endif
    MPI_Barrier(MPI_COMM_WORLD);
    times[4] = u_wseconds();
    MPI_Reduce(times, gtimes, 5, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);


    if (rc != ZOLTAN_OK) 
        fprintf(stderr, "Zoltan_Color failed with return code %d...\n", rc);

    for (maxcol=i=0; i<ngids; ++i)
        if (color[i] > maxcol)
            maxcol = color[i];
    MPI_Reduce(&maxcol, &ncolors, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD);
    if (uz->myRank==0) {
        struct rusage usage;
                
        printf("%s setup             Proc-0: %8.2lf   Max: %8.2lf\n", argv[0], times[1]-times[0], gtimes[1]-gtimes[0]);
        printf("%s gen rand edges    Proc-0: %8.2lf   Max: %8.2lf\n", argv[0], times[2]-times[1], gtimes[2]-gtimes[1]);
        printf("%s set gids          Proc-0: %8.2lf   Max: %8.2lf\n", argv[0], times[3]-times[2], gtimes[3]-gtimes[2]);
        printf("%s Zoltan_Color call Proc-0: %8.2lf   Max: %8.2lf\n", argv[0], times[4]-times[3], gtimes[4]-gtimes[3]);
        printf("%s Coloring Time    : %.2lf   # Colors used : %d\n", argv[0], gtimes[4]-gtimes[0], ncolors);
        getrusage(RUSAGE_SELF, &usage);
        printf("%s maxrss=%ld minflt=%ld majflt=%ld nswap=%ld\n", argv[0], usage.ru_maxrss, usage.ru_minflt, usage.ru_majflt, usage.ru_nswap);                
    }
    
#ifdef _DEBUG
    saveColor(argv[0], uz, (int *) gid_list, color, ngids);
#endif

    /******************************************************************
     ** Clean up
     ******************************************************************/

    if (gid_list)
        free(gid_list);
    if (color)
        free(color);
    if (xadj)
        free(xadj);
    if (adj)
        free(adj);
    if (adjTemp)
        free(adjTemp);
    if (uz->redgeto)
        free(uz->redgeto);

End:    
    Zoltan_Destroy(&zz);
    MPI_Finalize();

    return 0;
}
コード例 #23
0
ファイル: migoct.c プロジェクト: haripandey/trilinos
int Zoltan_Oct_migrate_octants(ZZ *zz, int *newpids, pOctant *octs, int nocts, int *nrecocts) {
  int i,j = 0;
  int nsends = 0;
  int nreceives = 0;
  int *despid = NULL;
  OCTNEW_msg *snd_reply = NULL;
  OCTNEW_msg *rcv_reply = NULL;
  int ierr = ZOLTAN_OK;
  ZOLTAN_COMM_OBJ *comm_plan;        /* Object returned by communication routines */
  char *yo = "Zoltan_Oct_migrate_octants";
  pOctant *newocts = NULL;                          /* New foreign octant pointers */

  if((newocts = (pOctant *) ZOLTAN_MALLOC(sizeof(pOctant)*(nocts+10))) == NULL) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    return ZOLTAN_MEMERR;
  }

  for (i=0; i<nocts; i++)
    newocts[i]=NULL;


  /* count number of sends */

  nsends=0;
  for (i=0; i<nocts; i++)            
    if (newpids[i]!=zz->Proc) 
      nsends++;

  /* build message array */

/*   if(nsends > 0) { */
    if((snd_reply = (OCTNEW_msg *) ZOLTAN_MALLOC((nsends + 1) * sizeof(OCTNEW_msg))) == NULL) {
      ZOLTAN_TRACE_EXIT(zz, yo);
      return ZOLTAN_MEMERR;
    }
    if((despid = (int *) ZOLTAN_MALLOC((nsends+10) * sizeof(int))) == NULL) {
      ZOLTAN_TRACE_EXIT(zz, yo);
      ZOLTAN_FREE(&snd_reply);
      return ZOLTAN_MEMERR;
    }
/*   } */
/*   else { */
/*     snd_reply = NULL; */
/*     despid = NULL; */
/*   } */

  j = 0;
  for (i=0; i<nocts; i++)                    
    if (newpids[i]!=zz->Proc) {  
      snd_reply[j].num = i;
      despid[j++] = newpids[i];
    }
   
  /* send messages */

  ierr = Zoltan_Comm_Create(&comm_plan, nsends, despid, zz->Communicator, MigOctCommCreate, &nreceives);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&snd_reply);
    ZOLTAN_FREE(&despid);
    return (ierr);
  }

  if((rcv_reply = (OCTNEW_msg *) ZOLTAN_MALLOC((nreceives + 1) * sizeof(OCTNEW_msg))) == NULL) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&snd_reply);
    ZOLTAN_FREE(&despid);
    ZOLTAN_FREE(&rcv_reply);
    return ZOLTAN_MEMERR;
  }

  ierr = Zoltan_Comm_Do(comm_plan, MigOctCommDo, (char *) snd_reply,
		    sizeof(OCTNEW_msg), (char *) rcv_reply);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&snd_reply);
    ZOLTAN_FREE(&despid);
    ZOLTAN_FREE(&rcv_reply);
    return (ierr);
  }
    /* Reply to malloc requests and Receive malloc replies */
    
  for (i=0; i< nreceives; i++) {  
    rcv_reply[i].ptr = Zoltan_Oct_POct_new((OCT_Global_Info *) (zz->LB.Data_Structure)); 
  }
;
  ierr = Zoltan_Comm_Do_Reverse(comm_plan, MigOctCommReverse, (char *) rcv_reply,
			    sizeof(OCTNEW_msg), NULL, (char *) snd_reply);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&snd_reply);
    ZOLTAN_FREE(&despid);
    ZOLTAN_FREE(&rcv_reply);
    return (ierr);
  }
  


  /* store remote pointers locally for future migration */
  for (i=0; i<nsends; i++) {                  
    newocts[snd_reply[i].num] = snd_reply[i].ptr;
  }
    
  ierr = Zoltan_Comm_Destroy(&comm_plan);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&snd_reply);
    ZOLTAN_FREE(&despid);
    ZOLTAN_FREE(&rcv_reply);
    return (ierr);
  }


  ZOLTAN_FREE(&snd_reply);
  ZOLTAN_FREE(&despid);
  ZOLTAN_FREE(&rcv_reply);

  /* set return value */

  *nrecocts = nreceives;
  ierr = Zoltan_Oct_Update_Connections(zz, octs, newpids, newocts, nocts);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    abort();
    return (ierr);
  }
  ierr = Zoltan_Oct_Final_Migration(zz, octs,newpids,newocts,nocts, *nrecocts);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    abort();
    return (ierr);
  }
  
  Zoltan_Oct_Update_Map(zz);

  ZOLTAN_FREE(&newocts);
  return ierr;
}
コード例 #24
0
/*
 * void malloc_new_objects();
 *
 * gets the tags being imported into this processor, and sets up the
 * import_tags array, and the nrectags array.
 */
static int malloc_new_objects(ZZ *zz, int nsentags, pRegion exported_tags, 
			      ZOLTAN_ID_PTR exported_gids,
			      ZOLTAN_ID_PTR exported_lids, int *tag_pids,
			      int *nstags, pRegion *ex_tags,
			      pRegion prev_tags, ZOLTAN_ID_PTR prev_gids,
			      ZOLTAN_ID_PTR prev_lids, int npimtags,
			      float *c3)
{
  char *yo = "malloc_new_objects";
  int i;                                  /* index counter */
  int nreceives;                          /* number of messages received */
  pRegion t_b_exp;                        /* array of tags to be exported */
  pRegion tmp = NULL;
  ZOLTAN_ID_PTR tmp_gids = NULL;
  ZOLTAN_ID_PTR tmp_lids = NULL;
  int msgtag, msgtag2;
  int j;
  int ierr = ZOLTAN_OK;
  int num_gid_entries = zz->Num_GID;
  int num_lid_entries = zz->Num_LID;
  float im_load;
  ZOLTAN_COMM_OBJ *comm_plan;  /* Object returned by communication routines */

  im_load = 0;
  msgtag = 32767;

  ierr = Zoltan_Comm_Create(&comm_plan, nsentags, tag_pids, zz->Communicator,
				msgtag, &nreceives);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, 
		       "Error returned from Zoltan_Comm_Create.");
    ZOLTAN_TRACE_EXIT(zz, yo);
    return(ierr);
  }


  if (nreceives > 0) {
    tmp = (pRegion) ZOLTAN_MALLOC(nreceives * sizeof(Region));
    tmp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, nreceives);
    tmp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, nreceives);
    if(tmp == NULL || !tmp_gids || (num_lid_entries && !tmp_lids)) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
      ZOLTAN_FREE(&tmp);
      ZOLTAN_FREE(&tmp_gids);
      ZOLTAN_FREE(&tmp_lids);
      ZOLTAN_TRACE_EXIT(zz, yo);
      return ZOLTAN_MEMERR;
    }
  }
  
  msgtag2 = 32766;
  ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_tags,
			sizeof(Region), (char *) tmp);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&tmp);
    ZOLTAN_FREE(&tmp_gids);
    ZOLTAN_FREE(&tmp_lids);
    return(ierr);
  }

  msgtag2--;
  ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_gids,
			sizeof(ZOLTAN_ID_TYPE)*num_gid_entries, 
			(char *) tmp_gids);
  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
    fprintf(stderr, "OCT %s Error %s returned from Zoltan_Comm_Do\n", yo,
            (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
    ZOLTAN_FREE(&tmp);
    ZOLTAN_FREE(&tmp_gids);
    ZOLTAN_FREE(&tmp_lids);
    return(ierr);
  }

  if (num_lid_entries > 0) {
    msgtag2--;
    ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_lids,
			  sizeof(ZOLTAN_ID_TYPE)*num_lid_entries,
			  (char *) tmp_lids);
    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
      ZOLTAN_FREE(&tmp);
      ZOLTAN_FREE(&tmp_gids);
      ZOLTAN_FREE(&tmp_lids);
      return(ierr);
    }
  }

  ierr = Zoltan_Comm_Destroy(&comm_plan);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, 
		       "Error returned from Zoltan_Comm_Destroy.");
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&tmp);
    return(ierr);
  }

  /* get each message sent, and store region in import array */
  j=0;
  for (i=0; i<nreceives; i++) {
    im_load += tmp[i].Weight;
    if(tmp[i].newProc != zz->Proc) {
      j++;
    }
  }
  
  if((j + npimtags) != 0) {                   /* malloc import array */
    if((t_b_exp = (pRegion)ZOLTAN_MALLOC((j+npimtags)*sizeof(Region)))==NULL){
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
      ZOLTAN_TRACE_EXIT(zz, yo);
      ZOLTAN_FREE(&tmp);
      ZOLTAN_FREE(&tmp_gids);
      ZOLTAN_FREE(&tmp_lids);
      return ZOLTAN_MEMERR;
    }
  }
  else
    t_b_exp = NULL;

  /* setup return pointer */
  (*ex_tags) = t_b_exp;

  j=0;
  for (i=0; i<nreceives; i++) {
    if(tmp[i].newProc != zz->Proc) {
      t_b_exp[j] = tmp[i];
      t_b_exp[j].Global_ID = ZOLTAN_MALLOC_GID(zz);
      t_b_exp[j].Local_ID  = ZOLTAN_MALLOC_LID(zz);
      if (!(t_b_exp[j].Global_ID) ||
	  (num_lid_entries && !(t_b_exp[j].Local_ID))) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
        ZOLTAN_TRACE_EXIT(zz, yo);
        ZOLTAN_FREE(&tmp);
        ZOLTAN_FREE(&tmp_gids);
        ZOLTAN_FREE(&tmp_lids);
        return ZOLTAN_MEMERR;
      }
      ZOLTAN_SET_GID(zz, t_b_exp[j].Global_ID,
		     &(tmp_gids[i*num_gid_entries]));
      ZOLTAN_SET_LID(zz, t_b_exp[j].Local_ID, &(tmp_lids[i*num_lid_entries]));
      j++;
    }
  }
  
  if(npimtags > 0) {
    for(i=0; i<npimtags; i++) {
      t_b_exp[j] = prev_tags[i];
      t_b_exp[j].Global_ID = ZOLTAN_MALLOC_GID(zz);
      t_b_exp[j].Local_ID  = ZOLTAN_MALLOC_LID(zz);
      if (!(t_b_exp[j].Global_ID) ||
	  (num_lid_entries && !(t_b_exp[j].Local_ID))) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
        ZOLTAN_TRACE_EXIT(zz, yo);
        ZOLTAN_FREE(&tmp);
        ZOLTAN_FREE(&tmp_gids);
        ZOLTAN_FREE(&tmp_lids);
        return ZOLTAN_MEMERR;
      }
      ZOLTAN_SET_GID(zz, t_b_exp[j].Global_ID,
		     &(prev_gids[i*num_gid_entries]));
      ZOLTAN_SET_LID(zz, t_b_exp[j].Local_ID, 
		     &(prev_lids[i*num_lid_entries]));
      j++;
    }
  }
  *nstags = j;

  ZOLTAN_FREE(&tmp);
  ZOLTAN_FREE(&tmp_gids);
  ZOLTAN_FREE(&tmp_lids);

  if((*nstags == 0) && (*ex_tags != NULL)) {
    ZOLTAN_TRACE_DETAIL(zz, yo,
                 "Fatal error, import tags not empty but no tags received\n");
    return ZOLTAN_FATAL;
  }

  *c3 = im_load;
  return ierr;
}
コード例 #25
0
ファイル: perm.c プロジェクト: abhishek4747/pv-zoltan
int Zoltan_Inverse_Perm(
  ZZ  *zz,		/* Input: Zoltan struct */
  int *perm, 		/* Input: Permutation to invert. */
  int *inv_perm, 	/* Output: Inverse permutation of perm. */
  int *vtxdist, 	/* Input: Distribution of the vectors. */
  char *order_type, 	/* Input: Local or global ordering? */
  int start_index	/* Input: Do permutations start with 0 or 1? */
  )
{
  int i, ierr, num_obj, nrecv, offset;
  int *proclist, *sendlist, *recvlist;
  ZOLTAN_COMM_OBJ *comm_plan;
  char msg[256];
  char *yo = "Zoltan_Inverse_Perm";

  ierr = ZOLTAN_OK;
  proclist = sendlist = recvlist = NULL;
  comm_plan = NULL;

  /* num_obj = local number of objects (elements in the perm vectors) */
  num_obj = vtxdist[(zz->Proc)+1] - vtxdist[zz->Proc];

  /* Verify that input permutation is really a permutation */
  /* Also check that start_index is correct. */

  /* Convert permutation vector to 0-base if necessary */
  if (start_index>0){
    for (i=0; i<num_obj; i++)
      perm[i] -= start_index;
  }

  if (strcmp(order_type, "LOCAL")==0){
    /* Local inverse */
    for (i=0; i<num_obj; i++)
      inv_perm[perm[i]] = i;
  }
  else if (strcmp(order_type, "GLOBAL")==0){
    /* Global inverse; use Zoltan Comm package */
    proclist = (int *) ZOLTAN_MALLOC (5*num_obj*sizeof(int));
    sendlist = &proclist[num_obj];
    recvlist = &proclist[3*num_obj];
    /* Set up comm plan. We know where to send. */
    /* Send pairs of (i, perm[i]) to other procs */
    offset = vtxdist[zz->Proc];
    for (i=0; i<num_obj; i++){
      sendlist[2*i] = offset+i;
      sendlist[2*i+1] = perm[i];
      proclist[i] =  Zoltan_Get_Processor_Graph(vtxdist, zz->Num_Proc, perm[i]);
    }
    ierr = Zoltan_Comm_Create(&comm_plan, num_obj, proclist, 
             zz->Communicator, TAG1, &nrecv);
    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error in Zoltan_Comm_Create");
      goto error;
    }
    if (nrecv != num_obj){
      /* This should never happen. */
      sprintf(msg, "Internal error: nrecv (%3d) != num_obj (%3d). Invalid permutation.\n", nrecv, num_obj); 
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
      ierr = ZOLTAN_FATAL;
      goto error;
    }
    /* Do the communication. */
    ierr = Zoltan_Comm_Do(comm_plan, TAG2, (char *)sendlist, 2*sizeof(int), (char *) recvlist);
    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error in Zoltan_Comm_Do");
      goto error;
    }

    /* Permute data locally. */
    for (i=0; i<num_obj; i++){
      /* inv_perm[perm[i]] = i; */
      inv_perm[recvlist[2*i+1]-offset] = recvlist[2*i];
    }
  }
  else {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown order_type.");
    ierr = ZOLTAN_FATAL;
    goto error;
  }

  /* Convert permutation vectors back to their right start_index */
  if (start_index>0){
    for (i=0; i<num_obj; i++){
      perm[i] += start_index;
      inv_perm[i] += start_index;
    }
  }

error:
  /* Free the comm_plan, proclist, sendlist, recvlist. */
  if (comm_plan) Zoltan_Comm_Destroy( &comm_plan);
  if (proclist ) ZOLTAN_FREE(&proclist);

  return (ierr);
}
コード例 #26
0
ファイル: lb_migrate.c プロジェクト: haripandey/trilinos
int Zoltan_Migrate(
    ZZ *zz,                      /* Zoltan structure.                  */
    int num_import,              /* Number of non-local objects assigned to the
                                  processor in the new decomposition.        */
    ZOLTAN_ID_PTR import_global_ids, /* Array of global IDs for non-local objects
                                  assigned to this processor in the new
                                  decomposition; this field can be NULL if
                                  the application doesn't provide import IDs.*/
    ZOLTAN_ID_PTR import_local_ids,  /* Array of local IDs for non-local objects
                                  assigned to the processor in the new
                                  decomposition; this field can be NULL if the
                                  application does not provide import IDs.   */
    int *import_procs,           /* Array of processor IDs of processors owning
                                  the non-local objects that are assigned to
                                  this processor in the new decomposition; this
                                  field can be NULL if the application does
                                  not provide import IDs.                    */
    int *import_to_part,         /* Array of partition numbers to which imported
                                  objects should be assigned.                */
    int num_export,              /* Number of objs to be exported
                                  to other processors to establish the new
                                  decomposition.                             */
    ZOLTAN_ID_PTR export_global_ids, /* Array of global IDs of
                                  objects to be exported to other processors
                                  to establish the new decomposition.        */
    ZOLTAN_ID_PTR export_local_ids,  /* Array of local IDs of
                                  objects to be exported to other processors
                                  to establish the new decomposition.        */
    int *export_procs,           /* Array of processor IDs
                                  to which objects will be exported
                                  to establish the new decomposition.        */
    int *export_to_part          /* Array of partition numbers to which exported
                                  objects should be assigned.                */
)
{
    /*
     *  Routine to help perform migration.  If migration pre-processing routine
     *  (ZOLTAN_PRE_MIGRATE_FN) is specified, this routine first calls that fn.
     *  It then calls a function to obtain the size of the migrating objects
     *  (ZOLTAN_OBJ_SIZE_FN).  The routine next calls an application-specified
     *  object packing routine (ZOLTAN_PACK_OBJ_FN) for each object
     *  to be exported.  It develops the needed communication map to move the
     *  objects to other processors.  It performs the communication according
     *  to the map, and then calls an application-specified object unpacking
     *  routine (ZOLTAN_UNPACK_OBJ_FN) for each object imported.
     */

    char *yo = "Zoltan_Migrate";
    int num_gid_entries, num_lid_entries;  /* lengths of global & local ids */
    int *sizes = NULL;       /* sizes (in bytes) of the object data for export. */
    int id_size;             /* size (in bytes) of ZOLTAN_GID + padding for
                            alignment                                       */
    int tag_size;            /* size (in bytes) of ZOLTAN_GID + one int
                            (for message size) */
    char *export_buf = NULL; /* buffer for packing export data.                 */
    char *import_buf = NULL; /* buffer for receiving imported data.             */
    char *tmp;               /* temporary pointer into buffers.                 */
    int i;                   /* loop counter.                                   */
    int tmp_size;            /* size of a single object's data.                 */
    int *idx = NULL;         /* index used for multi-fn packs and unpacks.      */
    int idx_cnt = 0;         /* index counter for idx array.                    */
    ZOLTAN_ID_PTR tmp_id = NULL; /* pointer to storage for a global ID in comm
                                buf  */
    ZOLTAN_ID_PTR lid;       /* temporary pointer to a local ID; used to pass
                            NULL to query functions when NUM_LID_ENTRIES=0. */
    ZOLTAN_COMM_OBJ *imp_plan = NULL; /* Comm obj built from import lists. */
    ZOLTAN_COMM_OBJ *exp_plan = NULL; /* Comm obj built from export lists. */
    int msgtag, msgtag2;     /* Tags for communication routines                 */
    int total_send_size;     /* Total size of outcoming message (in #items)     */
    int total_recv_size;     /* Total size of incoming message (in #items)      */
    int aligned_int;         /* size of an int padded for alignment             */
    int dest;                /* temporary destination partition.                */
    int include_parts = 0;   /* flag indicating whether partition info is
                            provided */
    int ierr = ZOLTAN_OK;
    int actual_num_exp = 0;
    int actual_exp_allocated = 0;
    ZOLTAN_ID_PTR actual_exp_gids = NULL;    /* Arrays containing only objs to  */
    ZOLTAN_ID_PTR actual_exp_lids = NULL;    /* actually be packed.  Objs that  */
    int *actual_exp_procs = NULL;            /* are changing partition but not  */
    int *actual_exp_to_part = NULL;          /* processor may not be included.  */
    int actual_num_imp = 0;
    int actual_imp_allocated = 0;
    ZOLTAN_ID_PTR actual_imp_gids = NULL;    /* Arrays containing only objs to  */
    ZOLTAN_ID_PTR actual_imp_lids = NULL;    /* actually be imported. Objs that  */
    int *actual_imp_procs = NULL;            /* are changing partition but not  */
    int *actual_imp_to_part = NULL;          /* processor may not be included.  */

    ZOLTAN_TRACE_ENTER(zz, yo);

    /*
     *  Return if this processor is not in the Zoltan structure's
     *  communicator.
     */

    if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
        goto End;
    }

    /*
     *  Check that all procs use the same id types.
     */

    ierr = check_input(zz,
                       ((num_export >= 0 && export_to_part) ||
                        (num_import >= 0 && import_to_part)),
                       &include_parts);
    if (ierr != ZOLTAN_OK)
        goto End;

    num_gid_entries = zz->Num_GID;
    num_lid_entries = zz->Num_LID;

    /*
     *  Check that all necessary query functions are available.
     */

    if (zz->Get_Obj_Size == NULL && zz->Get_Obj_Size_Multi == NULL) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
                           "ZOLTAN_OBJ_SIZE_FN or ZOLTAN_OBJ_SIZE_MULTI_FN function "
                           "to use the migration-help tools.");
        ierr = ZOLTAN_FATAL;
        goto End;
    }

    if (zz->Pack_Obj == NULL && zz->Pack_Obj_Multi == NULL) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
                           "ZOLTAN_PACK_OBJ_FN or ZOLTAN_PACK_OBJ_MULTI_FN function "
                           "to use the migration-help tools.");
        ierr = ZOLTAN_FATAL;
        goto End;
    }

    if (zz->Unpack_Obj == NULL && zz->Unpack_Obj_Multi == NULL) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
                           "ZOLTAN_UNPACK_OBJ_FN or ZOLTAN_UNPACK_OBJ_MULTI_FN function "
                           "to use the migration-help tools.");
        ierr = ZOLTAN_FATAL;
        goto End;
    }


    if (num_export >= 0) {

        /* Build the actual export arrays */
        ierr = actual_arrays(zz, num_gid_entries, num_lid_entries,
                             num_export, export_global_ids, export_local_ids,
                             export_procs, export_to_part,
                             &actual_num_exp, &actual_exp_gids, &actual_exp_lids,
                             &actual_exp_procs, &actual_exp_to_part,
                             &actual_exp_allocated);
        if (ierr < 0)
            goto End;

        /* Compute communication map based on actual exports.  */

        msgtag = 32767;
        ierr = Zoltan_Comm_Create(&exp_plan, actual_num_exp, actual_exp_procs,
                                  zz->Communicator, msgtag, &actual_num_imp);
        if (ierr < 0) {
            ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create.");
            goto End;
        }
    }

    else if (num_import >= 0) {

        /* Build the actual import arrays */
        ierr = actual_arrays(zz, num_gid_entries, num_lid_entries,
                             num_import, import_global_ids, import_local_ids,
                             import_procs, import_to_part,
                             &actual_num_imp, &actual_imp_gids, &actual_imp_lids,
                             &actual_imp_procs, &actual_imp_to_part,
                             &actual_imp_allocated);
        if (ierr < 0)
            goto End;

        /* Compute communication map based on imports.  */
        msgtag = 32767;
        ierr = Zoltan_Comm_Create(&imp_plan, actual_num_imp, actual_imp_procs,
                                  zz->Communicator, msgtag, &actual_num_exp);
        if (ierr < 0) {
            ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create.");
            goto End;
        }

        /* Compute actual export lists for packing objects */
        if (actual_num_exp > 0) {
            actual_exp_allocated = 1;
            actual_exp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_exp);
            actual_exp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, actual_num_exp);
            actual_exp_procs = (int *) ZOLTAN_MALLOC(sizeof(int) * actual_num_exp);
            if (include_parts)
                actual_exp_to_part = (int *) ZOLTAN_MALLOC(sizeof(int)*actual_num_exp);
            if (actual_exp_gids == NULL ||
                    (num_lid_entries && actual_exp_lids == NULL) ||
                    actual_exp_procs == NULL ||
                    (import_to_part != NULL && actual_exp_to_part == NULL)) {
                Zoltan_Multifree(__FILE__, __LINE__, 4,
                                 &actual_exp_gids, &actual_exp_lids,
                                 &actual_exp_procs, &actual_exp_to_part);
                ierr = ZOLTAN_MEMERR;
                goto End;
            }
        }

        msgtag2 = 32766;
        ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_gids,
                              (int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)),
                              (char *) actual_exp_gids);
        if (ierr < 0) {
            ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
            goto End;
        }

        if (num_lid_entries) {
            msgtag2--;
            ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_lids,
                                  (int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries),
                                  (char *) actual_exp_lids);
            if (ierr < 0) {
                ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
                goto End;
            }
        }

        Zoltan_Comm_Info(imp_plan, NULL, NULL, NULL, NULL, NULL, NULL,
                         NULL, NULL, NULL, NULL, NULL, actual_exp_procs, NULL);

        if (include_parts) {
            msgtag2--;
            ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_to_part,
                                  (int) sizeof(int), (char *) actual_exp_to_part);
            if (ierr < 0) {
                ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
                goto End;
            }
        }

        /* Create inverse plan (i.e., plan based on exports) so can set
         * variable sizes.
         * (Zoltan_Comm_Do_Reverse(imp_plan, ...) allows sending variable
         * but does not tell how large to allocate receive buffer.
         */
        ierr = Zoltan_Comm_Invert_Plan(&imp_plan);
        exp_plan = imp_plan;
        imp_plan = NULL;
    }
    else {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Import or export lists needed.");
        ierr = ZOLTAN_FATAL;
        goto End;
    }

    if (zz->Migrate.Pre_Migrate_PP != NULL) {
        zz->Migrate.Pre_Migrate_PP(zz->Migrate.Pre_Migrate_PP_Data,
                                   num_gid_entries, num_lid_entries,
                                   num_import, import_global_ids,
                                   import_local_ids, import_procs, import_to_part,
                                   num_export, export_global_ids,
                                   export_local_ids, export_procs, export_to_part,
                                   &ierr);
        if (ierr < 0) {
            ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                               "ZOLTAN_PRE_MIGRATE_PP_FN function.");
            goto End;
        }
    }

    if (zz->Migrate.Pre_Migrate != NULL) {
        zz->Migrate.Pre_Migrate(zz->Migrate.Pre_Migrate_Data,
                                num_gid_entries, num_lid_entries,
                                num_import, import_global_ids,
                                import_local_ids, import_procs,
                                num_export, export_global_ids,
                                export_local_ids, export_procs,
                                &ierr);
        if (ierr < 0) {
            ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                               "ZOLTAN_PRE_MIGRATE_FN function.");
            goto End;
        }
    }

    ZOLTAN_TRACE_DETAIL(zz, yo, "Done pre-migration processing");

    id_size = Zoltan_Align(num_gid_entries * sizeof(ZOLTAN_ID_TYPE));
    /* Note that alignment is not strictly necessary
       when ZOLTAN_ID_TYPE is int or unsigned int. */
    aligned_int = Zoltan_Align(sizeof(int));
    tag_size = id_size + aligned_int;

    /*
     * For each object, allow space for its global ID and its data plus
     * one int (for the object data size).
     * Zoltan will pack the global IDs; the application must pack the data
     * through the pack routine.  Zoltan needs the global IDs for unpacking,
     * as the order of the data received during communication is not
     * necessarily the same order as import_global_ids[].
     * Zoltan also needs to communicate the sizes of the objects because
     * only the sender knows the size of each object.
     */
    if (actual_num_exp > 0) {
        sizes = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int));
        if (!sizes) {
            ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
            ierr = ZOLTAN_MEMERR;
            goto End;
        }

        if (zz->Get_Obj_Size_Multi != NULL) {
            zz->Get_Obj_Size_Multi(zz->Get_Obj_Size_Multi_Data,
                                   num_gid_entries, num_lid_entries, actual_num_exp,
                                   actual_exp_gids, actual_exp_lids, sizes, &ierr);
            if (ierr < 0) {
                ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                                   "ZOLTAN_OBJ_SIZE_MULTI function.");
                goto End;
            }
        }
        else {
            for (i = 0; i < actual_num_exp; i++) {
                lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL);
                sizes[i] = zz->Get_Obj_Size(zz->Get_Obj_Size_Data,
                                            num_gid_entries, num_lid_entries,
                                            &(actual_exp_gids[i*num_gid_entries]),
                                            lid, &ierr);
                if (ierr < 0) {
                    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                                       "ZOLTAN_OBJ_SIZE function.");
                    goto End;
                }
            }
        }

        total_send_size = 0;

        for (i = 0; i < actual_num_exp; i++) {
            sizes[i] = Zoltan_Align(sizes[i]);
            total_send_size += sizes[i] + tag_size;
        }
        export_buf = (char *) ZOLTAN_CALLOC(total_send_size, sizeof(char));
        if (!export_buf) {
            ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
            ierr = ZOLTAN_MEMERR;
            goto End;
        }

        if (zz->Pack_Obj_Multi != NULL) {
            /* Allocate an index array for ZOLTAN_PACK_OBJ_MULTI_FN. */
            idx = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int));
            if (!idx) {
                ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
                ierr = ZOLTAN_MEMERR;
                goto End;
            }
        }

        /*
         *  Pack the objects for export.
         */

        idx_cnt = 0;
        tmp = export_buf;
        for (i = 0; i < actual_num_exp; i++) {

            /* Pack the object's global ID */
            tmp_id = (ZOLTAN_ID_PTR) tmp;
            ZOLTAN_SET_GID(zz, tmp_id, &(actual_exp_gids[i*num_gid_entries]));
            tmp += id_size;

            /* Pack the object's size */
            *((int *)tmp) = sizes[i];
            tmp += aligned_int;

            /* If using ZOLTAN_PACK_OBJ_MULTI_FN, build the index array. */
            idx_cnt += tag_size;
            if (idx != NULL) {
                idx[i] = idx_cnt;
            }
            tmp += sizes[i];
            idx_cnt += sizes[i];
        }

        if (zz->Pack_Obj_Multi != NULL) {
            if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
                printf("[%1d] DEBUG in %s: Packing objects with multi-pack\n",
                       zz->Proc, yo);
            }
            zz->Pack_Obj_Multi(zz->Pack_Obj_Multi_Data,
                               num_gid_entries, num_lid_entries, actual_num_exp,
                               actual_exp_gids, actual_exp_lids,
                               (actual_exp_to_part!=NULL ? actual_exp_to_part
                                : actual_exp_procs),
                               sizes, idx, export_buf, &ierr);
            if (ierr < 0) {
                ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                                   "ZOLTAN_PACK_OBJ_MULTI function.");
                goto End;
            }
        }
        else {
            tmp = export_buf + tag_size;
            for (i = 0; i < actual_num_exp; i++) {
                if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
                    printf("[%1d] DEBUG in %s: Packing object with gid ", zz->Proc, yo);
                    ZOLTAN_PRINT_GID(zz, &(actual_exp_gids[i*num_gid_entries]));
                    printf("size = %d bytes\n", sizes[i]);
                }

                /* Pack the object's data */
                lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL);
                dest = (actual_exp_to_part != NULL ? actual_exp_to_part[i]
                        : actual_exp_procs[i]);
                zz->Pack_Obj(zz->Pack_Obj_Data,
                             num_gid_entries, num_lid_entries,
                             &(actual_exp_gids[i*num_gid_entries]), lid, dest,
                             sizes[i], tmp, &ierr);
                if (ierr < 0) {
                    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                                       "ZOLTAN_PACK_OBJ function.");
                    goto End;
                }
                tmp += sizes[i] + tag_size;
            }
        }
        ZOLTAN_FREE(&idx);
        tmp_id = NULL;
    }

    ZOLTAN_TRACE_DETAIL(zz, yo, "Done packing objects");


    /* Modify sizes[] to contain message sizes, not object sizes */
    for (i=0; i<actual_num_exp; i++) {
        sizes[i] += tag_size;
    }

    msgtag--;
    ierr = Zoltan_Comm_Resize(exp_plan, sizes, msgtag, &total_recv_size);
    if (ierr < 0) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Resize.");
        goto End;
    }

    if (actual_num_imp > 0) {
        import_buf = (char *) ZOLTAN_MALLOC(total_recv_size);
        if (!import_buf) {
            ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
            ierr = ZOLTAN_MEMERR;
            goto End;
        }
    }

    /*
     *  Send the export data using the communication plan.
     */

    msgtag2 = 32765;
    ierr = Zoltan_Comm_Do(exp_plan, msgtag2, export_buf, 1, import_buf);
    if (ierr < 0) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
        goto End;
    }

    /*
     *  Free whatever memory we can.
     */

    Zoltan_Comm_Destroy(&exp_plan);
    ZOLTAN_FREE(&export_buf);
    ZOLTAN_FREE(&sizes);

    ZOLTAN_TRACE_DETAIL(zz, yo, "Done communication");

    /*
     *  Perform application-specified processing before unpacking the data.
     */
    if (zz->Migrate.Mid_Migrate_PP != NULL) {
        zz->Migrate.Mid_Migrate_PP(zz->Migrate.Mid_Migrate_PP_Data,
                                   num_gid_entries, num_lid_entries,
                                   num_import, import_global_ids,
                                   import_local_ids, import_procs, import_to_part,
                                   num_export, export_global_ids,
                                   export_local_ids, export_procs, export_to_part,
                                   &ierr);
        if (ierr < 0) {
            ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                               "ZOLTAN_MID_MIGRATE_PP_FN function.");
            goto End;
        }
    }

    if (zz->Migrate.Mid_Migrate != NULL) {
        zz->Migrate.Mid_Migrate(zz->Migrate.Mid_Migrate_Data,
                                num_gid_entries, num_lid_entries,
                                num_import, import_global_ids,
                                import_local_ids, import_procs,
                                num_export, export_global_ids,
                                export_local_ids, export_procs,
                                &ierr);
        if (ierr < 0) {
            ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                               "ZOLTAN_MID_MIGRATE_FN function.");
            goto End;
        }
    }

    /*
     *  Unpack the object data.
     */

    if (actual_num_imp > 0) {

        if (zz->Unpack_Obj_Multi != NULL) {

            /* Allocate and fill input arrays for Unpack_Obj_Multi. */
            sizes = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int));
            tmp_id = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_imp);
            idx = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int));
            if (!sizes || !tmp_id || !idx) {
                ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
                ierr = ZOLTAN_MEMERR;
                goto End;
            }

            tmp = import_buf;
            idx_cnt = 0;
            for (i = 0; i < actual_num_imp; i++) {

                /* Unpack the object's global ID */
                ZOLTAN_SET_GID(zz, &(tmp_id[i*num_gid_entries]), (ZOLTAN_ID_PTR) tmp);
                tmp += id_size;

                /* Unpack the object's size */
                sizes[i] = *((int *)tmp);
                tmp += aligned_int;

                /* If using ZOLTAN_UNPACK_OBJ_MULTI_FN, build the index array. */
                idx_cnt += tag_size;
                if (idx != NULL) {
                    idx[i] = idx_cnt;
                }

                tmp += sizes[i];
                idx_cnt += sizes[i];
            }

            if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
                printf("[%1d] DEBUG in %s: Unpacking objects with multi-fn\n",
                       zz->Proc,yo);
            }
            zz->Unpack_Obj_Multi(zz->Unpack_Obj_Multi_Data, num_gid_entries,
                                 actual_num_imp, tmp_id, sizes, idx, import_buf, &ierr);
            ZOLTAN_FREE(&import_buf);
            ZOLTAN_FREE(&sizes);
            ZOLTAN_FREE(&tmp_id);
            ZOLTAN_FREE(&idx);
            if (ierr < 0) {
                ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                                   "ZOLTAN_UNPACK_OBJ_MULTI_FN.");
                goto End;
            }
        }
        else {
            tmp = import_buf;
            for (i = 0; i < actual_num_imp; i++) {
                tmp_size = *((int *)(tmp + id_size));
                if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
                    printf("[%1d] DEBUG in %s: Unpacking object with gid ", zz->Proc, yo);
                    ZOLTAN_PRINT_GID(zz, (ZOLTAN_ID_PTR)tmp);
                    printf("size = %d bytes\n", tmp_size);
                }

                /* Unpack the object's data */

                zz->Unpack_Obj(zz->Unpack_Obj_Data, num_gid_entries,
                               (ZOLTAN_ID_PTR) tmp, tmp_size,
                               tmp + tag_size, &ierr);
                if (ierr < 0) {
                    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                                       "ZOLTAN_UNPACK_OBJ_FN.");
                    goto End;
                }
                tmp += (tmp_size + tag_size);
            }
            ZOLTAN_FREE(&import_buf);
        }
    }

    ZOLTAN_TRACE_DETAIL(zz, yo, "Done unpacking objects");

    if (zz->Migrate.Post_Migrate_PP != NULL) {
        zz->Migrate.Post_Migrate_PP(zz->Migrate.Post_Migrate_PP_Data,
                                    num_gid_entries, num_lid_entries,
                                    num_import, import_global_ids,
                                    import_local_ids, import_procs, import_to_part,
                                    num_export, export_global_ids,
                                    export_local_ids, export_procs, export_to_part,
                                    &ierr);
        if (ierr < 0) {
            ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                               "ZOLTAN_POST_MIGRATE_PP_FN function.");
            goto End;
        }
    }

    if (zz->Migrate.Post_Migrate != NULL) {
        zz->Migrate.Post_Migrate(zz->Migrate.Post_Migrate_Data,
                                 num_gid_entries, num_lid_entries,
                                 num_import, import_global_ids,
                                 import_local_ids, import_procs,
                                 num_export, export_global_ids,
                                 export_local_ids, export_procs,
                                 &ierr);
        if (ierr < 0) {
            ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                               "ZOLTAN_POST_MIGRATE_FN function.");
            goto End;
        }
    }

End:

    if (actual_exp_allocated) {
        Zoltan_Multifree(__FILE__, __LINE__, 4,
                         &actual_exp_gids, &actual_exp_lids,
                         &actual_exp_procs, &actual_exp_to_part);
    }
    if (actual_imp_allocated) {
        Zoltan_Multifree(__FILE__, __LINE__, 4,
                         &actual_imp_gids, &actual_imp_lids,
                         &actual_imp_procs, &actual_imp_to_part);
    }

    if (ierr < 0) {
        if (exp_plan) Zoltan_Comm_Destroy(&exp_plan);
        Zoltan_Multifree(__FILE__, __LINE__, 5,
                         &import_buf, &tmp_id, &sizes, &idx, &export_buf);
    }
    ZOLTAN_TRACE_EXIT(zz, yo);
    return (ierr);
}
コード例 #27
0
ファイル: phg.c プロジェクト: askhl/octopus-dfrt2
static int Zoltan_PHG_Output_Parts (
  ZZ *zz,
  ZHG *zhg,
  Partition hg_parts   /* Output partitions relative to the 2D distribution
                          of zhg->HG */
)
{
/* Function to map the computed partition from the distribution in HGraph
 * to the input distribution 
 */

static char *yo = "Zoltan_PHG_Output_Parts";
int i;
int msg_tag = 31000;
int ierr = ZOLTAN_OK;
int nObj = zhg->nObj;
int *outparts = NULL;
int *sendbuf = NULL;  
HGraph *phg = &(zhg->HG);

  zhg->Output_Parts = outparts 
                     = (int*) ZOLTAN_MALLOC (nObj * sizeof(int));
  if (zhg->VtxPlan != NULL) {
    /* Get the partition information from the 2D decomposition back to the
     * original owning processor for each GID.  */
    sendbuf = (int*) ZOLTAN_MALLOC(zhg->nRecv_GNOs * sizeof(int));
    for (i = 0; i < zhg->nRecv_GNOs; i++)
      sendbuf[i] = hg_parts[VTX_GNO_TO_LNO(phg, zhg->Recv_GNOs[i])];
    ierr = Zoltan_Comm_Do_Reverse(zhg->VtxPlan, msg_tag, (char*) sendbuf,
                                  sizeof(int), NULL, (char *) outparts);
    if (ierr) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error from Zoltan_Comm_Do_Reverse");
      goto End;
    }

    ZOLTAN_FREE(&sendbuf);
    Zoltan_Comm_Destroy(&(zhg->VtxPlan));
  }
  else {
    for (i = 0; i < zhg->nRecv_GNOs; i++)
      outparts[i] = hg_parts[zhg->Recv_GNOs[i]];
  }

  if (zz->LB.Remap_Flag) {
    int new_map;
    int *newproc = (int *) ZOLTAN_MALLOC(nObj * sizeof(int));
    int num_gid_entries = zz->Num_GID;

    if (nObj && !newproc) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
      ierr = ZOLTAN_MEMERR;
      goto End;
    }
    for (i = 0; i < nObj; i++){
      newproc[i] = Zoltan_LB_Part_To_Proc(zz, outparts[i],
                                          &(zhg->GIDs[i*num_gid_entries]));
      if (newproc[i]<0){
        ZOLTAN_PRINT_ERROR(zz->Proc, yo,
         "Zoltan_LB_Part_To_Proc returned invalid processor number.");
        ierr = ZOLTAN_FATAL;
        ZOLTAN_FREE(&newproc);
        goto End;
      }
    }
    
    ierr = Zoltan_LB_Remap(zz, &new_map, nObj, newproc, zhg->Input_Parts,
                           outparts, 1);
    if (ierr < 0) 
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_LB_Remap");
    ZOLTAN_FREE(&newproc);
  }

End:
  if (zhg->Recv_GNOs) ZOLTAN_FREE(&(zhg->Recv_GNOs));
  zhg->nRecv_GNOs = 0;
  return ierr;
}
コード例 #28
0
ファイル: phg_distrib.c プロジェクト: askhl/octopus-dfrt2
static int Zoltan_PHG_Redistribute_Hypergraph(
    ZZ *zz, 
    PHGPartParams *hgp,     /* Input:  parameters; used only for UseFixedVtx */
    HGraph  *ohg,           /* Input:  Local part of distributed hypergraph */
    int     firstproc,      /* Input:  rank (in ocomm) of the first proc of 
                                       the ncomm*/
    int     *v2Col,         /* Input:  Vertex to processor Column Mapping */
    int     *n2Row,         /* Input:  Net to processor Row Mapping */
    PHGComm *ncomm,         /* Input:  communicators of new distribution */
    HGraph  *nhg,           /* Output: Newly redistributed hypergraph */
    int     **vmap,         /* Output: allocated with the size nhg->nVtx and
                               vertex map from nhg to ohg's local vertex number*/
    int     **vdest         /* Output: allocated with the size nhg->nVtx and
                               stores dest proc in ocomm */
    )
{
    char * yo = "Zoltan_PHG_Redistribute_Hypergraph";
    PHGComm *ocomm = ohg->comm;
    int ierr=ZOLTAN_OK;
    int i, v, n, nPins, nsend, elemsz, nVtx, nEdge;
    int msg_tag = 9999;
    int *proclist=NULL, *sendbuf=NULL;
    int *vno=NULL, *nno=NULL, *dist_x=NULL, *dist_y=NULL,
        *vsn=NULL, *nsn=NULL, *pins=NULL, *cnt=NULL;
    ZOLTAN_COMM_OBJ *plan;    
    
    Zoltan_HG_HGraph_Init (nhg);
    nhg->comm = ncomm;
    
    nhg->dist_x = (int *) ZOLTAN_CALLOC(ncomm->nProc_x+1, sizeof(int));
    nhg->dist_y = (int *) ZOLTAN_CALLOC(ncomm->nProc_y+1, sizeof(int));
    dist_x = (int *) ZOLTAN_CALLOC(ncomm->nProc_x+1, sizeof(int));
    dist_y = (int *) ZOLTAN_CALLOC(ncomm->nProc_y+1, sizeof(int));
    vsn = (int *) ZOLTAN_CALLOC(ncomm->nProc_x+1, sizeof(int));
    nsn = (int *) ZOLTAN_CALLOC(ncomm->nProc_y+1, sizeof(int));
    vno = (int *) ZOLTAN_MALLOC(ohg->nVtx * sizeof(int));
    nno = (int *) ZOLTAN_MALLOC(ohg->nEdge * sizeof(int));

    if (!nhg->dist_x || !nhg->dist_y || !dist_x || !dist_y ||
        !vsn || !nsn || (ohg->nVtx && !vno) || (ohg->nEdge && !nno) ) {
        uprintf(ocomm, " new comm nProcx=%d nProcy=%d nvtx=%d nedge=%d", ncomm->nProc_x, ncomm->nProc_y, ohg->nVtx, ohg->nEdge);
        MEMORY_ERROR;
    }
      
    for (v = 0; v < ohg->nVtx; ++v)
        ++dist_x[v2Col[v]];
    for (n = 0; n < ohg->nEdge; ++n)
        ++dist_y[n2Row[n]];

    /* UVCUVC: CHECK ASSUMPTION
       This code assumes that the objects in the receive buffer of
       Zoltan_Comm_Do function are
         1- in the increasing processor order,
         2- order of the items send by a processor is preserved.
     */
    

    /* compute prefix sum to find new vertex start numbers; for each processor */
    MPI_Scan(dist_x, vsn, ncomm->nProc_x, MPI_INT, MPI_SUM, ocomm->row_comm);
    /* All reduce to compute how many each processor will have */ 
    MPI_Allreduce(dist_x, &(nhg->dist_x[1]), ncomm->nProc_x, MPI_INT, MPI_SUM, 
                  ocomm->row_comm);
    nhg->dist_x[0] = 0;    
    for (i=1; i<=ncomm->nProc_x; ++i) 
        nhg->dist_x[i] += nhg->dist_x[i-1];
    
    MPI_Scan(dist_y, nsn, ncomm->nProc_y, MPI_INT, MPI_SUM, ocomm->col_comm);

    MPI_Allreduce(dist_y, &(nhg->dist_y[1]), ncomm->nProc_y, MPI_INT, MPI_SUM, ocomm->col_comm);
    nhg->dist_y[0] = 0;
    for (i=1; i<=ncomm->nProc_y; ++i)
        nhg->dist_y[i] += nhg->dist_y[i-1];

#ifdef _DEBUG1
    PrintArr(ocomm, "vsn", vsn, ncomm->nProc_x);
    PrintArr(ocomm, "nsn", nsn, ncomm->nProc_y);
#endif
    
    /* find mapping of current LOCAL vertex no (in my node)
       to "new" vertex no LOCAL to dest node*/
    for (v = ohg->nVtx-1; v>=0; --v)
        vno[v] = --vsn[v2Col[v]];
    for (n = ohg->nEdge-1; n>=0; --n)
        nno[n] = --nsn[n2Row[n]];

    nsend = MAX(MAX(ohg->nPins, ohg->nVtx), ohg->nEdge);
    elemsz = MAX(MAX(2, ohg->VtxWeightDim), ohg->EdgeWeightDim);
    elemsz = (sizeof(float)>sizeof(int)) ? sizeof(float)*elemsz : sizeof(int)*elemsz;

    proclist = (int *) ZOLTAN_MALLOC(nsend * sizeof(int));
    sendbuf = (int *) ZOLTAN_MALLOC(nsend * elemsz);

    /* first communicate pins */
    nPins = 0;
    for (v = 0; v < ohg->nVtx; ++v) { 
        for (i = ohg->vindex[v]; i < ohg->vindex[v+1]; ++i) {
#ifdef _DEBUG1
            if ((n2Row[ohg->vedge[i]] * ncomm->nProc_x + v2Col[v])<0 ||
                (n2Row[ohg->vedge[i]] * ncomm->nProc_x + v2Col[v])>=ocomm->nProc)
                errexit("vertex %d vedge[%d]=%d n2Row=%d #Proc_x=%d v2Col=%d", i, ohg->vedge[i], n2Row[ohg->vedge[i]], ncomm->nProc_x , v2Col[v]);
#endif
            proclist[nPins]   = firstproc + n2Row[ohg->vedge[i]] * ncomm->nProc_x + v2Col[v];
            sendbuf[2*nPins]  = vno[v];
            sendbuf[2*nPins+1]= nno[ohg->vedge[i]];
            ++nPins; 
        }
    }
#ifdef _DEBUG1
    if (nPins!=ohg->nPins) {
        uprintf(ocomm, "sanity check failed nPins(%d)!=hg->nPins(%d)\n", nPins, ohg->nPins);
        errexit("terminating");
    }
#endif

    --msg_tag;
    ierr |= Zoltan_Comm_Create(&plan, ohg->nPins, proclist, ocomm->Communicator,
                               msg_tag, &nPins);

#ifdef _DEBUG1
    if (ncomm->myProc==-1 && nPins>1) { /* this processor is not in new comm but receiving data?*/
        uprintf(ocomm, "Something wrong; why I'm receiving data nPins=%d\n", nPins);
        errexit("terminating");
    }
#endif
    
    if (nPins && (pins = (int *) ZOLTAN_MALLOC(nPins * 2 * sizeof(int)))==NULL) 
        MEMORY_ERROR;

    --msg_tag;
    Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, 2*sizeof(int),
                   (char *) pins);
    Zoltan_Comm_Destroy(&plan);

    /* now communicate vertex map */
    nsend = 0;
    if (!ocomm->myProc_y) { /* only first row sends to the first row of ncomm */
        for (v = 0; v < ohg->nVtx; ++v) { 
            proclist[nsend] = firstproc+v2Col[v];
            sendbuf[nsend++] = ohg->vmap[v];
        }
    }
        
    --msg_tag; 
    ierr |= Zoltan_Comm_Create(&plan, nsend, proclist, ocomm->Communicator,
                               msg_tag, &nVtx); 

#ifdef _DEBUG1
    if (ncomm->myProc==-1 && nVtx>1) { /* this processor is not in new comm but receiving data?*/ 
        uprintf(ocomm, "Something wrong; why I'm receiving data nVtx=%d\n", nVtx);
        errexit("terminating");
    }
#endif

    /* those are only needed in the first row of ncomm */
    *vmap = *vdest = NULL;  
    if (!ncomm->myProc_y && nVtx &&
        (!(*vmap = (int *) ZOLTAN_MALLOC(nVtx * sizeof(int))) ||
         !(*vdest = (int *) ZOLTAN_MALLOC(nVtx * sizeof(int)))))
        MEMORY_ERROR;
    
    --msg_tag;
    Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, sizeof(int),
                   (char *) *vmap);

    if (!ocomm->myProc_y) { /* only first row sends to the first row of ncomm */
        for (v = 0; v < ohg->nVtx; ++v) 
            sendbuf[v] = ocomm->myProc;
    }
    --msg_tag;
    Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, sizeof(int),
                   (char *) *vdest);
        
    if (ncomm->myProc!=-1) { /* I'm in the new comm */
        /* ncomm's first row now bcast to other rows */
        MPI_Bcast(&nVtx, 1, MPI_INT, 0, ncomm->col_comm);
#ifdef _DEBUG1
        if (nVtx!=(nhg->dist_x[ncomm->myProc_x+1] - nhg->dist_x[ncomm->myProc_x]))
            errexit("nVtx(%d)!= nhg->dist_x[ncomm->myProc_x+1] - nhg->dist_x[ncomm->myProc_x](%d)", nVtx, nhg->dist_x[ncomm->myProc_x+1] - nhg->dist_x[ncomm->myProc_x]);
#endif
        if (nVtx && (nhg->vmap = (int *) ZOLTAN_MALLOC(nVtx * sizeof(int)))==NULL) 
            MEMORY_ERROR;
        for (i=0; i<nVtx; ++i)
            nhg->vmap[i] = i;
    }


    /* now communicate vertex weights */
    if (ohg->VtxWeightDim) {
        if (nVtx)
            nhg->vwgt = (float*) ZOLTAN_MALLOC(nVtx*ohg->VtxWeightDim*sizeof(float));
    
        --msg_tag;
        Zoltan_Comm_Do(plan, msg_tag, (char *) ohg->vwgt,
                       ohg->VtxWeightDim*sizeof(float), (char *) nhg->vwgt);
        if (ncomm->myProc!=-1)  /* ncomm's first row now bcast to other rows */
            MPI_Bcast(nhg->vwgt, nVtx*ohg->VtxWeightDim, MPI_FLOAT, 0, ncomm->col_comm);
    }    

    /* communicate fixed vertices, if any */
    if (hgp->UseFixedVtx) {
        if (nVtx)
            nhg->fixed_part = (int *) ZOLTAN_MALLOC(nVtx*sizeof(int));
        --msg_tag;
        Zoltan_Comm_Do(plan, msg_tag, (char *) ohg->fixed_part,
                       sizeof(int), (char *) nhg->fixed_part);
        if (ncomm->myProc!=-1)  /* ncomm's first row now bcast to other rows */
            MPI_Bcast(nhg->fixed_part, nVtx, MPI_INT, 0, ncomm->col_comm);
    }    
    /* communicate pref parts, if any */
    if (hgp->UsePrefPart) {
        if (nVtx)
            nhg->pref_part = (int *) ZOLTAN_MALLOC(nVtx*sizeof(int));
        --msg_tag;
        Zoltan_Comm_Do(plan, msg_tag, (char *) ohg->pref_part,
                       sizeof(int), (char *) nhg->pref_part);
        if (ncomm->myProc!=-1)  /* ncomm's first row now bcast to other rows */
            MPI_Bcast(nhg->pref_part, nVtx, MPI_INT, 0, ncomm->col_comm);
    }    

    /* this comm plan is no longer needed. */
    Zoltan_Comm_Destroy(&plan);

    
    if (ohg->EdgeWeightDim) { /* now communicate edge weights */
        nsend = 0;
        if (!ocomm->myProc_x)  /* only first column sends to first column of ncomm */
            for (n = 0; n < ohg->nEdge; ++n) 
                proclist[nsend++] = firstproc + n2Row[n]*ncomm->nProc_x;
    
        --msg_tag;
        ierr |= Zoltan_Comm_Create(&plan, nsend, proclist, ocomm->Communicator,
                                   msg_tag, &nEdge);

#ifdef _DEBUG1
        if (ncomm->myProc==-1 && nEdge>1) { /* this processor is not in new comm but receiving data?*/
            uprintf(ocomm, "Something wrong; why I'm receiving data nEdge=%d\n", nEdge);
            errexit("terminating");
        }
#endif
        if (ncomm->myProc!=-1) { /* if we're in the new comm */
            /* ncomm's first column now bcast to other columns */
            MPI_Bcast(&nEdge, 1, MPI_INT, 0, ncomm->row_comm);
#ifdef _DEBUG1
            if (nEdge != (nhg->dist_y[ncomm->myProc_y+1] - nhg->dist_y[ncomm->myProc_y]))
            errexit("nEdge(%d)!=nhg->dist_y[ncomm->myProc_y+1] - nhg->dist_y[ncomm->myProc_y](%d)", nEdge, nhg->dist_y[ncomm->myProc_y+1] - nhg->dist_y[ncomm->myProc_y]);
#endif
        }
        
        if (nEdge)
            nhg->ewgt = (float*) ZOLTAN_MALLOC(nEdge*ohg->EdgeWeightDim*sizeof(float));
    
        --msg_tag;
        Zoltan_Comm_Do(plan, msg_tag, (char *) ohg->ewgt,
                       ohg->EdgeWeightDim*sizeof(float), (char *) nhg->ewgt);
        if (ncomm->myProc!=-1) { /* if we're in the new comm */
            /* ncomm's first column now bcast to other columns */
            if (nEdge) 
                MPI_Bcast(nhg->ewgt, nEdge*ohg->EdgeWeightDim, MPI_FLOAT, 0, 
                          ncomm->row_comm);
        }

        Zoltan_Comm_Destroy(&plan);
    } else 
        nEdge = (ncomm->myProc==-1) 
                ? 0 
                : nhg->dist_y[ncomm->myProc_y+1] - nhg->dist_y[ncomm->myProc_y];
    

    if (ncomm->myProc==-1) {
#ifdef _DEBUG1
        if (nPins || nVtx || nEdge)
            errexit("I should not have any data: hey nPins=%d  nVtx=%d  nEdge=%d\n", nPins, nVtx, nEdge);
#endif
        nhg->nEdge = nhg->nVtx = nhg->nPins = 0;
    } else {
        nhg->nEdge = nhg->dist_y[ncomm->myProc_y+1] - nhg->dist_y[ncomm->myProc_y];
        nhg->nVtx = nhg->dist_x[ncomm->myProc_x+1] - nhg->dist_x[ncomm->myProc_x];
        nhg->nPins = nPins;
    
        /* Unpack the pins received. */
        cnt = (int *) ZOLTAN_CALLOC(nhg->nVtx + 1, sizeof(int));
        nhg->vindex = (int *) ZOLTAN_CALLOC(nhg->nVtx + 1, sizeof(int));
        nhg->vedge = (int *) ZOLTAN_MALLOC(nhg->nPins * sizeof(int));
        
        if (!cnt || !nhg->vindex || (nPins && !nhg->vedge))
            MEMORY_ERROR;

        /* Count the number of pins per vertex */
        for (i = 0; i < nPins; ++i)
            ++cnt[pins[2*i]];
        
        /* Compute prefix sum to represent hindex correctly. */
        for (i = 0; i < nhg->nVtx; ++i)  {
            nhg->vindex[i+1] = nhg->vindex[i] + cnt[i];
            cnt[i] = nhg->vindex[i];
        }

        for (i = 0; i < nPins; ++i) 
            nhg->vedge[cnt[pins[2*i]]++] = pins[2*i+1];
        
        nhg->info               = ohg->info;
        nhg->VtxWeightDim       = ohg->VtxWeightDim;
        nhg->EdgeWeightDim      = ohg->EdgeWeightDim;

        ierr = Zoltan_HG_Create_Mirror(zz, nhg);
        if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN)
            MEMORY_ERROR;
    }

 End:
    Zoltan_Multifree(__FILE__, __LINE__, 10,
                     &proclist, &sendbuf, &pins, &cnt,
                     &vno, &nno, &dist_x, &dist_y, &vsn, &nsn
        );
    
    return ierr;
}
コード例 #29
0
ファイル: migoct.c プロジェクト: haripandey/trilinos
static int Zoltan_Oct_Update_Connections(
ZZ *zz,
pOctant *octs,      /* octs[nocts]    */
int *newpids,       /* newpids[nocts] */
pOctant *newocts,   /* newocts[nocts] */
int nocts)          /* number of octants leaving this processor */
{
  int i, j;
  int nsends;
  int nreceives;
  pOctant parent;
  pOctant child;
  int ppid;
  int cpid;
  int childnum;
  int *despid = NULL;
  Update_msg umsg;
  Update_msg *localumsg = NULL;
  Update_msg *remoteumsg = NULL;
  Update_msg *rcv_umsg = NULL;
  int localcount;
  int remotecount;
  int ierr = ZOLTAN_OK;


  ZOLTAN_COMM_OBJ *comm_plan;           /* Object returned by communication routines */
  char *yo = "Zoltan_Oct_Update_Connections";
  OCT_Global_Info *OCT_info = (OCT_Global_Info *) zz->LB.Data_Structure;
  localcount=0;
  remotecount=0;

  /* count number of sends */
  nsends = 0;
  for (i=0; i<nocts; i++)              
    if (newpids[i]!=zz->Proc)
      nsends++;

  if(nocts > 0) {
    if((remoteumsg = (Update_msg *) ZOLTAN_MALLOC((nocts+1) * sizeof(Update_msg)*9)) == NULL) {
      ZOLTAN_TRACE_EXIT(zz, yo);
      return ZOLTAN_MEMERR;
    }
    
    if((localumsg  = (Update_msg *) ZOLTAN_MALLOC((nocts+1) * sizeof(Update_msg)*9)) == NULL) {
      ZOLTAN_TRACE_EXIT(zz, yo);
      ZOLTAN_FREE(&remoteumsg);
      return ZOLTAN_MEMERR;
    }
    
    if((despid = (int *) ZOLTAN_MALLOC((nocts+1) * sizeof(int)*9)) == NULL) {
      ZOLTAN_TRACE_EXIT(zz, yo);
      ZOLTAN_FREE(&remoteumsg);
      ZOLTAN_FREE(&localumsg);
      return ZOLTAN_MEMERR;
    }
  }
  else {
    remoteumsg = NULL;
    localumsg = NULL;
    despid = NULL;
  }
  localcount = 0;
  remotecount = 0;

  for (i=0; i<nocts; i++)                       /* Send connection updates */
    if (newpids[i]!=zz->Proc) {
	parent = Zoltan_Oct_parent(octs[i]); 
        ppid   = Zoltan_Oct_Ppid(octs[i]); 
        childnum = Zoltan_Oct_childnum(octs[i]);
	if (parent) {      /* Let parent of oct[i] know that it's moving   */
	  if (ppid==zz->Proc) {
	    FILLUPDATEMSG(localumsg[localcount], parent, childnum, newocts[i], newpids[i]);
	    localcount++;
	  }
	  else {
	    FILLUPDATEMSG(remoteumsg[remotecount], parent, childnum, newocts[i], newpids[i]);
	    despid[remotecount++] = ppid;
	  }
	}
	for (j=0; j<8; j++) {
	  child = Zoltan_Oct_child(octs[i],j);
	  cpid = octs[i]->cpid[j];
	  /* Tell child of oct[i] that it is moving */
	  if (child) {
	    if (cpid==zz->Proc) {
	      /* NOTE: -1 signals PARENT   */
	      FILLUPDATEMSG(localumsg[localcount], child, -1, newocts[i], newpids[i]);
	      localcount++;
	    }
	    else {
	      /* NOTE: -1 signals PARENT   */
	      FILLUPDATEMSG(remoteumsg[remotecount], child, -1, newocts[i], newpids[i]); 
	      despid[remotecount++] = cpid;
	    }
	  }
	}
    }

  ierr = Zoltan_Comm_Create(&comm_plan, remotecount, despid, zz->Communicator,
			MigUpdCommCreate, &nreceives);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&remoteumsg);
    ZOLTAN_FREE(&localumsg);
    ZOLTAN_FREE(&despid);
    return (ierr);
  }


/*   if(nreceives > 0) { */
    if((rcv_umsg = (Update_msg *) ZOLTAN_MALLOC((nreceives +1) * sizeof(Update_msg)*9)) == NULL) {
      ZOLTAN_TRACE_EXIT(zz, yo);
      ZOLTAN_FREE(&remoteumsg);
      ZOLTAN_FREE(&localumsg);
      ZOLTAN_FREE(&despid);
      return ZOLTAN_MEMERR;
    }

    
    ierr = Zoltan_Comm_Do(comm_plan, MigUpdCommDo, (char *) remoteumsg,
		      sizeof(Update_msg), (char *) rcv_umsg);
    if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
      ZOLTAN_TRACE_EXIT(zz, yo);
      ZOLTAN_FREE(&remoteumsg);
      ZOLTAN_FREE(&localumsg);
      ZOLTAN_FREE(&despid);
      ZOLTAN_FREE(&rcv_umsg);
      return (ierr);
    }

/*   } */
/*   else { */
/*     rcv_umsg = NULL; */
/*   } */
  /* update new octants */
  for (i=0; i< (localcount+nreceives); i++)  {   
    if (i<localcount) 
      umsg=localumsg[i];
    else 
      umsg=rcv_umsg[i-localcount];
    if (umsg.childnum>=0) {
      Zoltan_Oct_setchild(umsg.oct,umsg.childnum,umsg.newptr);
      Zoltan_Oct_setCpid(umsg.oct,umsg.childnum,umsg.newpid);
    }
    else {
      if((Zoltan_Oct_data_newpid(umsg.oct) ==  OCT_info->OCT_localpid) ||
	 ((Zoltan_Oct_data_newpid(umsg.oct) !=  OCT_info->OCT_localpid) && (umsg.newpid == OCT_info->OCT_localpid)))
	Zoltan_Oct_POct_setparent(OCT_info, umsg.oct,umsg.newptr,umsg.newpid);
      else {
	umsg.oct->ppid = umsg.newpid;
	umsg.oct->parent = umsg.newptr;
      }
    }
  }

  ierr = Zoltan_Comm_Destroy(&comm_plan);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&remoteumsg);
    ZOLTAN_FREE(&localumsg);
    ZOLTAN_FREE(&despid);
    ZOLTAN_FREE(&rcv_umsg);
    return (ierr);
  }

  ZOLTAN_FREE(&remoteumsg);
  ZOLTAN_FREE(&localumsg);
  ZOLTAN_FREE(&rcv_umsg);
  ZOLTAN_FREE(&despid);
  return ierr;
}