Exemple #1
0
int
Zoltan_Matrix_Vertex_Info(ZZ* zz, const Zoltan_matrix * const m,
			  ZOLTAN_ID_PTR lid,
			  float *wwgt, int *input_part)
{
  static char *yo = "Zoltan_Matrix_Vertex_Info";
  int ierr = ZOLTAN_OK;
  int nX;
  ZOLTAN_ID_PTR l_gid = NULL;
  ZOLTAN_ID_PTR l_lid = NULL;
  float * l_xwgt = NULL;
  int *l_input_part = NULL;
  struct Zoltan_DD_Struct *dd = NULL;

  ZOLTAN_TRACE_ENTER(zz, yo);

  if (m->completed == 0) {
    ierr = ZOLTAN_FATAL;
    goto End;
  }

  ierr = Zoltan_Get_Obj_List(zz, &nX, &l_gid, &l_lid,
			     zz->Obj_Weight_Dim, &l_xwgt,
			     &l_input_part);

  ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, zz->Num_LID,
			   zz->Obj_Weight_Dim*sizeof(float)/sizeof(int),
			   nX, 0);
  CHECK_IERR;

    /* Make our new numbering public */
  Zoltan_DD_Update (dd, l_gid, l_lid, (ZOLTAN_ID_PTR) l_xwgt,l_input_part, nX);
  ZOLTAN_FREE(&l_gid);
  ZOLTAN_FREE(&l_lid);
  ZOLTAN_FREE(&l_xwgt);
  ZOLTAN_FREE(&l_input_part);

  ierr = Zoltan_DD_Find (dd, m->yGID, lid, (ZOLTAN_ID_PTR)wwgt, input_part,
		    m->nY, NULL);

 End:
  if (dd != NULL)
    Zoltan_DD_Destroy(&dd);
  ZOLTAN_FREE(&l_gid);
  ZOLTAN_FREE(&l_lid);
  ZOLTAN_FREE(&l_xwgt);
  ZOLTAN_FREE(&l_input_part);

  ZOLTAN_TRACE_EXIT(zz, yo);
  return (ierr);
}
Exemple #2
0
int Zoltan_Random(
  ZZ *zz,                       /* The Zoltan structure.                     */
  float *part_sizes,            /* Input:  Array of size zz->LB.Num_Global_Parts
                                   * zz->Obj_Weight_Dim
                                   containing the percentage of work to be
                                   assigned to each partition.               */
  int *num_import,              /* Return -1. Random uses only export lists. */
  ZOLTAN_ID_PTR *import_global_ids, /* Not used. */
  ZOLTAN_ID_PTR *import_local_ids,  /* Not used. */
  int **import_procs,           /* Not used. */
  int **import_to_part,         /* Not used. */
  int *num_export,              /* Output: Number of objects to export. */
  ZOLTAN_ID_PTR *export_global_ids, /* Output: GIDs to export. */
  ZOLTAN_ID_PTR *export_local_ids,  /* Output: LIDs to export. */
  int **export_procs,           /* Output: Processsors to export to. */
  int **export_to_part          /* Output: Partitions to export to. */
)
{
  int ierr = ZOLTAN_OK;
  int i, count, num_obj;
  int max_export;
  double rand_frac = 1.0;       /* Default is to move all objects. */
  ZOLTAN_ID_PTR global_ids = NULL;
  ZOLTAN_ID_PTR local_ids = NULL; 
  int *parts = NULL;
  float *dummy = NULL;
  static char *yo = "Zoltan_Random";
  static int first_time = 1;

  ZOLTAN_TRACE_ENTER(zz, yo);

  /* Synchronize the random number generator. 
   * This synchronization is needed only for sanity in our nightly testing. 
   * If some other operation (eg., Zoltan_LB_Eval) changes the status of 
   * the random number generator, the answers here will change.  They won't
   * be wrong, but they will be different from our accepted answers.
   */
  if (first_time) {
    Zoltan_Srand(zz->Seed, NULL);
    Zoltan_Rand(NULL);
    first_time=0;
  }

  /* No import lists computed. */
  *num_import = -1;

  /* Get parameter values. */
  Zoltan_Bind_Param(Random_params, "RANDOM_MOVE_FRACTION", (void *) &rand_frac);
  Zoltan_Assign_Param_Vals(zz->Params, Random_params, zz->Debug_Level, 
                           zz->Proc, zz->Debug_Proc);

  /* Get list of local objects. */
  ierr = Zoltan_Get_Obj_List(zz, &num_obj, &global_ids, &local_ids, 0,
                             &dummy, &parts);

  /* Bound number of objects to export. */
  max_export = 1.5*rand_frac*num_obj;

  /* Allocate export lists. */
  *export_global_ids = *export_local_ids = NULL;
  *export_procs = *export_to_part = NULL;
  if (max_export > 0) {
    if (!Zoltan_Special_Malloc(zz, (void **)export_global_ids, max_export,
                               ZOLTAN_SPECIAL_MALLOC_GID)
     || !Zoltan_Special_Malloc(zz, (void **)export_local_ids, max_export,
                               ZOLTAN_SPECIAL_MALLOC_LID)
     || !Zoltan_Special_Malloc(zz, (void **)export_procs, max_export,
                               ZOLTAN_SPECIAL_MALLOC_INT)
     || !Zoltan_Special_Malloc(zz, (void **)export_to_part, max_export,
                               ZOLTAN_SPECIAL_MALLOC_INT)) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
      ierr = ZOLTAN_MEMERR;
      goto End;
    }
  }

  /* Randomly assign ids to procs. */
  count=0;
  for (i=0; i<num_obj; i++){
    /* Randomly select some objects to move (export) */
    if ((count<max_export) && (Zoltan_Rand(NULL)<rand_frac*ZOLTAN_RAND_MAX)){
      /* export_global_ids[count] = global_ids[i]; */
      ZOLTAN_SET_GID(zz, &((*export_global_ids)[count*zz->Num_GID]),
                     &global_ids[i*zz->Num_GID]);
      if (local_ids)
        /* export_local_ids[count] = local_ids[i]; */
        ZOLTAN_SET_LID(zz, &((*export_local_ids)[count*zz->Num_LID]),
                       &local_ids[i*zz->Num_LID]);
      /* Randomly pick new partition number. */
      (*export_to_part)[count] = Zoltan_Rand_InRange(NULL, zz->LB.Num_Global_Parts);
      /* Processor number is derived from partition number. */
      (*export_procs)[count] = Zoltan_LB_Part_To_Proc(zz, 
                     (*export_to_part)[count], &global_ids[i*zz->Num_GID]);

      /* printf("Debug: Export gid %u to part %d and proc %d.\n", (*export_global_ids)[count], (*export_to_part)[count], (*export_procs)[count]); */

      ++count;
    }
  }
  (*num_export) = count;

End:
  /* Free local memory, but not export lists. */
  ZOLTAN_FREE(&global_ids);
  ZOLTAN_FREE(&local_ids);
  ZOLTAN_FREE(&parts);

  ZOLTAN_TRACE_EXIT(zz, yo);
  return ierr;
}
Exemple #3
0
int
Zoltan_Matrix_Build (ZZ* zz, Zoltan_matrix_options *opt, Zoltan_matrix* matrix)
{
  static char *yo = "Zoltan_Matrix_Build";
  int ierr = ZOLTAN_OK;
  int nX;
  int  *xGNO = NULL;
  ZOLTAN_ID_PTR xLID=NULL;
  ZOLTAN_ID_PTR xGID=NULL;
  ZOLTAN_ID_PTR yGID=NULL;
  ZOLTAN_ID_PTR pinID=NULL;
  float *xwgt = NULL;
  int * Input_Parts=NULL;
  struct Zoltan_DD_Struct *dd = NULL;
  int *proclist = NULL;
  int *xpid = NULL;
  int i;

  ZOLTAN_TRACE_ENTER(zz, yo);

  memset (matrix, 0, sizeof(Zoltan_matrix)); /* Set all fields to 0 */
  memcpy (&matrix->opts, opt, sizeof(Zoltan_matrix_options));

  /**************************************************/
  /* Obtain vertex information from the application */
  /**************************************************/

  ierr = Zoltan_Get_Obj_List(zz, &nX, &xGID, &xLID,
			     zz->Obj_Weight_Dim, &xwgt,
			     &Input_Parts);

  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error getting object data");
    goto End;
  }
  ZOLTAN_FREE(&Input_Parts);
  ZOLTAN_FREE(&xwgt);

  /*******************************************************************/
  /* Assign vertex consecutive numbers (gnos)                        */
  /*******************************************************************/

  if (matrix->opts.speed == MATRIX_FULL_DD) { /* Zoltan computes a translation */
    if (nX) {
      xGNO = (int*) ZOLTAN_MALLOC(nX*sizeof(int));
      if (xGNO == NULL)
	MEMORY_ERROR;
    }
    ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, xGNO, nX, matrix->opts.randomize, &matrix->globalX);

    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error assigning global numbers to vertices");
      goto End;
    }

    ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, 1,
			     0, nX, 0);
    CHECK_IERR;

    /* Make our new numbering public */
    Zoltan_DD_Update (dd, xGID, (ZOLTAN_ID_PTR) xGNO, NULL,  NULL, nX);
  }
  else { /* We don't want to use the DD */
    xGNO = (int *) xGID;
    MPI_Allreduce(&nX, &matrix->globalX, 1, MPI_INT, MPI_SUM, zz->Communicator);
  }

  /* I store : xGNO, xGID, xpid,  */
  ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, 1, zz->Num_GID,
			   1, matrix->globalX/zz->Num_Proc, 0);
  CHECK_IERR;

  /* Hope a linear assignment will help a little */
  Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc);
  /* Associate all the data with our xGNO */
  xpid = (int*)ZOLTAN_MALLOC(nX*sizeof(int));
  if (nX >0 && xpid == NULL) MEMORY_ERROR;
  for (i = 0 ; i < nX ; ++i)
    xpid[i] = zz->Proc;

  Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)xGNO, xGID, (ZOLTAN_ID_PTR) xpid, NULL, nX);
  ZOLTAN_FREE(&xpid);

  if (matrix->opts.pinwgt)
    matrix->pinwgtdim = zz->Edge_Weight_Dim;
  else
    matrix->pinwgtdim = 0;

  ierr = matrix_get_edges(zz, matrix, &yGID, &pinID, nX, &xGID, &xLID, &xGNO, &xwgt);
  CHECK_IERR;
  matrix->nY_ori = matrix->nY;

  if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)){
    goto End;
  }

  if (matrix->opts.enforceSquare && matrix->redist) {
    /* Convert yGID to yGNO using the same translation as x */
    /* Needed for graph : rowID = colID */
    /* y and x may have different distributions */
    matrix->yGNO = (int*)ZOLTAN_MALLOC(matrix->nY * sizeof(int));
    if (matrix->nY && matrix->yGNO == NULL)
      MEMORY_ERROR;
    ierr = Zoltan_DD_Find (dd, yGID, (ZOLTAN_ID_PTR)(matrix->yGNO), NULL, NULL,
		    matrix->nY, NULL);
    if (ierr != ZOLTAN_OK) {
      ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Hyperedge GIDs don't match.\n");
      ierr = ZOLTAN_FATAL;
      goto End;
    }
  }

  if (matrix->opts.local) { /* keep only local edges */
    proclist = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int));
    if (matrix->nPins && proclist == NULL) MEMORY_ERROR;
  }
  else
    proclist = NULL;

  /* Convert pinID to pinGNO using the same translation as x */
  if (matrix->opts.speed == MATRIX_FULL_DD) {
    matrix->pinGNO = (int*)ZOLTAN_MALLOC(matrix->nPins* sizeof(int));
    if ((matrix->nPins > 0) && (matrix->pinGNO == NULL)) MEMORY_ERROR;

    ierr = Zoltan_DD_Find (dd, pinID, (ZOLTAN_ID_PTR)(matrix->pinGNO), NULL, NULL,
			   matrix->nPins, proclist);
    if (ierr != ZOLTAN_OK) {
      ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Undefined GID found.\n");
      ierr = ZOLTAN_FATAL;
      goto End;
    }
    ZOLTAN_FREE(&pinID);
    Zoltan_DD_Destroy(&dd);
    dd = NULL;
  }
  else {
    matrix->pinGNO = (int *) pinID;
    pinID = NULL;
  }

/*   if (matrix->opts.local) {  /\* keep only local edges *\/ */
/*     int *nnz_list; /\* nnz offset to delete *\/ */
/*     int nnz;       /\* number of nnz to delete *\/ */
/*     int i; */

/*     nnz_list = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int)); */
/*     if (matrix->nPins && nnz_list == NULL) MEMORY_ERROR; */
/*     for (i = 0, nnz=0 ; i < matrix->nPins ; ++i) { */
/*       if (proclist[i] == zz->Proc) continue; */
/*       nnz_list[nnz++] = i; */
/*     } */
/*     ZOLTAN_FREE(&proclist); */
/*     Zoltan_Matrix_Delete_nnz(zz, matrix, nnz, nnz_list); */
/*   } */

  if (!matrix->opts.enforceSquare) {
    /* Hyperedges name translation is different from the one of vertices */
    matrix->yGNO = (int*)ZOLTAN_CALLOC(matrix->nY, sizeof(int));
    if (matrix->nY && matrix->yGNO == NULL) MEMORY_ERROR;

    /*     int nGlobalEdges = 0; */
    ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, matrix->yGNO, matrix->nY,
					     matrix->opts.randomize, &matrix->globalY);
    CHECK_IERR;

/*     /\**************************************************************************************** */
/*      * If it is desired to remove dense edges, divide the list of edges into */
/*      * two lists.  The ZHG structure will contain the removed edges (if final_output is true), */
/*      * and the kept edges will be returned. */
/*      ****************************************************************************************\/ */
/*     totalNumEdges = zhg->globalHedges; */

/*     ierr = remove_dense_edges_matrix(zz, zhg, edgeSizeThreshold, final_output, */
/*				     &nLocalEdges, &nGlobalEdges, &nPins, */
/*				     &edgeGNO, &edgeSize, &edgeWeight, &pinGNO, &pinProcs); */

/*     if (nGlobalEdges < totalNumEdges){ */
/*       /\* re-assign edge global numbers if any edges were removed *\/ */
/*       ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, edgeGNO, nLocalEdges, */
/*					       randomizeInitDist, &totalNumEdges); */
/*       if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { */
/*	ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error reassigning global numbers to edges"); */
/*	goto End; */
/*       } */
/*     } */

      /* We have to define ddY : yGNO, yGID, ywgt */
      ierr = Zoltan_DD_Create (&matrix->ddY, zz->Communicator, 1, zz->Num_GID,
			       0, matrix->globalY/zz->Num_Proc, 0);
      /* Hope a linear assignment will help a little */
      Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddY, matrix->globalY/zz->Num_Proc);
      /* Associate all the data with our yGNO */
      Zoltan_DD_Update (matrix->ddY, (ZOLTAN_ID_PTR)matrix->yGNO,
			yGID, NULL, NULL, matrix->nY);
  }

 End:
  ZOLTAN_FREE(&xpid);
  ZOLTAN_FREE(&xLID);
  ZOLTAN_FREE(&xGNO);
  ZOLTAN_FREE(&xGID);
  ZOLTAN_FREE(&xwgt);
  ZOLTAN_FREE(&Input_Parts);
  ZOLTAN_FREE(&proclist);
  if (dd != NULL)
    Zoltan_DD_Destroy(&dd);
  /* Already stored in the DD */
  ZOLTAN_FREE(&yGID);

  ZOLTAN_TRACE_EXIT(zz, yo);

  return (ierr);
}
Exemple #4
0
int Zoltan_Block(
  ZZ *zz,                       /* The Zoltan structure.                     */
  float *part_sizes,            /* Input:  Array of size zz->LB.Num_Global_Parts
                                   containing the percentage of work to be
                                   assigned to each partition.               */
  int *num_import,              /* Return -1. We use only export lists. */
  ZOLTAN_ID_PTR *import_global_ids, /* Not used. */
  ZOLTAN_ID_PTR *import_local_ids,  /* Not used. */
  int **import_procs,           /* Not used. */
  int **import_to_part,         /* Not used. */
  int *num_export,              /* Output: Number of objects to export. */
  ZOLTAN_ID_PTR *export_global_ids, /* Output: GIDs to export. */
  ZOLTAN_ID_PTR *export_local_ids,  /* Output: LIDs to export. */
  int **export_procs,           /* Output: Processsors to export to. */
  int **export_to_part          /* Output: Partitions to export to. */
)
{
  int ierr = ZOLTAN_OK;
  int i, count, num_obj;
  int wtflag = 0;
  ZOLTAN_ID_PTR global_ids = NULL;
  ZOLTAN_ID_PTR local_ids = NULL; 
  int *parts = NULL;
  int *newparts = NULL;
  float *wgts = NULL;
  static char *yo = "Zoltan_Block";

  ZOLTAN_TRACE_ENTER(zz, yo);

  /* No import lists computed. */
  *num_import = -1;
  *export_global_ids = *export_local_ids = NULL;
  *export_procs = *export_to_part = NULL;

  /* Get list of local objects. */
  if (zz->Obj_Weight_Dim > 1) {
    ierr = ZOLTAN_FATAL;
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, 
                      "OBJ_WEIGHT_DIM > 1 not supported by LB_METHOD BLOCK.");
    goto End;
  }
  wtflag = (zz->Obj_Weight_Dim>0 ? 1 : 0);
  ierr = Zoltan_Get_Obj_List(zz, &num_obj, &global_ids, &local_ids, wtflag,
                             &wgts, &parts);

  /* Compute the new partition numbers. */
  newparts = (int *) ZOLTAN_MALLOC(num_obj * sizeof(int));
  if (num_obj && (!newparts)){
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
    ierr = ZOLTAN_MEMERR;
    goto End;
  }
  block_part(zz, num_obj, wtflag, wgts, part_sizes, newparts);

  /* Check how many partition numbers changed. */
  count=0;
  for (i=0; i<num_obj; i++){
    if (newparts[i] != parts[i])
      ++count;
  }
  (*num_export) = count;

  /* Allocate export lists. */
  if ((*num_export) > 0) {
    if (!Zoltan_Special_Malloc(zz, (void **)export_global_ids, (*num_export),
                               ZOLTAN_SPECIAL_MALLOC_GID)
     || !Zoltan_Special_Malloc(zz, (void **)export_local_ids, (*num_export),
                               ZOLTAN_SPECIAL_MALLOC_LID)
     || !Zoltan_Special_Malloc(zz, (void **)export_procs, (*num_export),
                               ZOLTAN_SPECIAL_MALLOC_INT)
     || !Zoltan_Special_Malloc(zz, (void **)export_to_part, (*num_export),
                               ZOLTAN_SPECIAL_MALLOC_INT)) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
      ierr = ZOLTAN_MEMERR;
      goto End;
    }
  }

  /* Loop over objects and fill export lists. */
  count=0;
  for (i=0; i<num_obj; i++){
    if (newparts[i] != parts[i]){
      /* export_global_ids[count] = global_ids[i]; */
      ZOLTAN_SET_GID(zz, &((*export_global_ids)[count*zz->Num_GID]),
                     &global_ids[i*zz->Num_GID]);
      if (local_ids)
        /* export_local_ids[count] = local_ids[i]; */
        ZOLTAN_SET_LID(zz, &((*export_local_ids)[count*zz->Num_LID]),
                       &local_ids[i*zz->Num_LID]);
      /* Set new partition number. */
      (*export_to_part)[count] = newparts[i];
      /* Processor is derived from partition number. */
      (*export_procs)[count] = Zoltan_LB_Part_To_Proc(zz, 
                     (*export_to_part)[count], &global_ids[i*zz->Num_GID]);

      ++count;
    }
  }

End:
  /* Free local memory, but not export lists. */
  ZOLTAN_FREE(&global_ids);
  ZOLTAN_FREE(&local_ids);
  ZOLTAN_FREE(&parts);
  ZOLTAN_FREE(&newparts);
  if (wtflag) ZOLTAN_FREE(&wgts);

  ZOLTAN_TRACE_EXIT(zz, yo);
  return ierr;
}
Exemple #5
0
static void Zoltan_Oct_get_bounds(ZZ *zz, pRegion *ptr1, int *num_objs, 
		   COORD min, COORD max, int wgtflag, float *c0) 
{
  char *yo = "Zoltan_Oct_get_bounds";
  ZOLTAN_ID_PTR obj_global_ids = NULL; 
  ZOLTAN_ID_PTR obj_local_ids = NULL;
  int *parts = NULL;   /* Input partition assignments; currently unused. */
  float *obj_wgts = NULL;
  double *geom_vec = NULL;
  float objwgt;        /* Temporary value of an object weight; used to pass
                          0. to initialize_regions when wgtflag == 0. */
  ZOLTAN_ID_PTR lid;   /* Temporary pointer to a local ID; used to pass NULL 
                          to initialize_regions when NUM_LID_ENTRIES == 0. */
  int num_dim;
  int i;
  pRegion tmp=NULL, ptr;
  COORD global_min, global_max;
  double PADDING = 0.0000001;
  int ierr = 0;
  int num_gid_entries = zz->Num_GID;
  int num_lid_entries = zz->Num_LID;

  /* Initialization */
  max[0] = max[1] = max[2] = -DBL_MAX;
  min[0] = min[1] = min[2] =  DBL_MAX;

  ierr = Zoltan_Get_Obj_List(zz, num_objs, &obj_global_ids, &obj_local_ids,
                             wgtflag, &obj_wgts, &parts);
  if (ierr) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo,
                   "Error returned from user function Zoltan_Get_Obj_List.");
    exit (-1);
  }

  ierr = Zoltan_Get_Coordinates(zz, *num_objs, obj_global_ids, obj_local_ids,
                                &num_dim, &geom_vec);

  for (i = 0; i < (*num_objs); i++) {
    lid = (num_lid_entries ? obj_local_ids + i*num_lid_entries : NULL);
    objwgt = (wgtflag ? obj_wgts[i] : 0.);
    initialize_region(zz, &(ptr), obj_global_ids + i*num_gid_entries,
                      lid, wgtflag, objwgt,
                      num_dim, geom_vec + i*num_dim);
    if (i == 0) {
      tmp = ptr;
      *c0 = (float)tmp->Weight;
      vector_set(min, tmp->Coord);
      vector_set(max, tmp->Coord);
      *ptr1 = tmp;
    }
    else {
      *c0 += (float)ptr->Weight;
      /* the following is really a hack, since it has no real basis 
         in vector mathematics.... */
      if(ptr->Coord[0] < min[0])
        min[0] = ptr->Coord[0];
      if(ptr->Coord[1] < min[1])
        min[1] = ptr->Coord[1];
      if(ptr->Coord[2] < min[2])
        min[2] = ptr->Coord[2];
      if(ptr->Coord[0] > max[0])
        max[0] = ptr->Coord[0];
      if(ptr->Coord[1] > max[1])
        max[1] = ptr->Coord[1];
      if(ptr->Coord[2] > max[2])
        max[2] = ptr->Coord[2];
      tmp->next = ptr;
      tmp = tmp->next;
    }
  
    ptr = NULL;
  }
  ZOLTAN_FREE(&obj_global_ids);
  ZOLTAN_FREE(&obj_local_ids);
  ZOLTAN_FREE(&parts);
  ZOLTAN_FREE(&obj_wgts);
  ZOLTAN_FREE(&geom_vec);
  
  MPI_Allreduce(&(min[0]), &(global_min[0]), 3, 
		MPI_DOUBLE, MPI_MIN, zz->Communicator);
  MPI_Allreduce(&(max[0]), &(global_max[0]), 3,
		MPI_DOUBLE, MPI_MAX, zz->Communicator);

  max[0] = global_max[0];
  max[1] = global_max[1];
  max[2] = global_max[2];
  min[0] = global_min[0];
  min[1] = global_min[1];
  min[2] = global_min[2];
  
  /* hack used for sample program since working in 2D -- */
  /* causes problems for refining the octree */
  if(max[2] == min[2])
    max[2] = 1.0;

  for(i=0; i<3; i++) {
    /* min[i] -= PADDING; */
    max[i] += PADDING;
  }

  return;
}
Exemple #6
0
int
Zoltan_Matrix_Build (ZZ* zz, Zoltan_matrix_options *opt, Zoltan_matrix* matrix,
  int request_GNOs,                /* Input:  Flag indicating calling code 
                                              needs translation of extra GIDs
                                              to GNOs; partial 2D coloring
                                              needs this feature. */
  int num_requested,               /* Input:  Local # of GIDs needing 
                                              translation to GNOs. */
  ZOLTAN_ID_PTR requested_GIDs,    /* Input:  Calling code requests the 
                                              GNOs for these GIDs */
  ZOLTAN_GNO_TYPE *requested_GNOs  /* Output: Return GNOs of 
                                              the requested GIDs.  */
)  
{
  static char *yo = "Zoltan_Matrix_Build";
  int ierr = ZOLTAN_OK;
  int nX;
  ZOLTAN_GNO_TYPE tmp;
  ZOLTAN_GNO_TYPE *xGNO = NULL;
  ZOLTAN_ID_PTR xLID=NULL;
  ZOLTAN_ID_PTR xGID=NULL;
  ZOLTAN_ID_PTR yGID=NULL;
  ZOLTAN_ID_PTR pinID=NULL;
  float *xwgt = NULL;
  int * Input_Parts=NULL;
  struct Zoltan_DD_Struct *dd = NULL;
  int *proclist = NULL;
  int *xpid = NULL;
  int i;
  int gno_size_for_dd;
  MPI_Datatype zoltan_gno_mpi_type;
  int use_full_dd = (opt->speed == MATRIX_FULL_DD);
  int fast_build_base = opt->fast_build_base;
  matrix->opts.speed = opt->speed;  
  matrix->opts.fast_build_base = opt->fast_build_base;

  ZOLTAN_TRACE_ENTER(zz, yo);

  if (num_requested && (!requested_GIDs || !requested_GNOs)) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, 
                       "Error in requested input; needed arrays are NULL.\n");
  }

  /* ZOLTAN_GNO_TYPE is >= ZOLTAN_ID_TYPE */
  gno_size_for_dd = sizeof(ZOLTAN_GNO_TYPE) / sizeof(ZOLTAN_ID_TYPE);

  zoltan_gno_mpi_type = Zoltan_mpi_gno_type();

  memset (matrix, 0, sizeof(Zoltan_matrix)); /* Set all fields to 0 */
  memcpy (&matrix->opts, opt, sizeof(Zoltan_matrix_options));

  /**************************************************/
  /* Obtain vertex information from the application */
  /**************************************************/

  ierr = Zoltan_Get_Obj_List(zz, &nX, &xGID, &xLID,
			     zz->Obj_Weight_Dim, &xwgt,
			     &Input_Parts);

  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error getting object data");
    goto End;
  }
  ZOLTAN_FREE(&Input_Parts);
  ZOLTAN_FREE(&xwgt);

  /*******************************************************************/
  /* Assign vertex consecutive numbers (gnos)                        */
  /*******************************************************************/

  if (use_full_dd) {
    /* Zoltan computes a translation */
    /* Have to use Data Directory if request_GNOs is true. */
    if (nX) {
      xGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nX*sizeof(ZOLTAN_GNO_TYPE));
      if (xGNO == NULL)
	MEMORY_ERROR;
    }
    ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, xGNO, nX, matrix->opts.randomize, &matrix->globalX);

    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error assigning global numbers to vertices");
      goto End;
    }

    ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, gno_size_for_dd, 0, nX, 0);
    CHECK_IERR;

    /* Make our new numbering public */
    Zoltan_DD_Update (dd, xGID, (ZOLTAN_ID_PTR) xGNO, NULL,  NULL, nX);
    if (request_GNOs) {
      Zoltan_DD_Find(dd, requested_GIDs, (ZOLTAN_ID_PTR) requested_GNOs,
                     NULL, NULL, num_requested, NULL);
    }
  }
  else { /* We don't want to use the DD */
     /*
     * KDDKDD 2/10/11  This code cannot work when NUM_GID_ENTRIES>1.
     * KDDKDD 2/10/11  The assumption is that, if a user sets the
     * KDDKDD 2/10/11  appropriate parameter to enable this code, the user
     * KDDKDD 2/10/11  knows that his GIDs are compatible with integers.
     */
    if (sizeof(ZOLTAN_GNO_TYPE) != sizeof(ZOLTAN_ID_TYPE)){
      xGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nX*sizeof(ZOLTAN_GNO_TYPE));
      if (nX && xGNO == NULL)
        MEMORY_ERROR;
      for (i=0; i < nX; i++)
        xGNO[i] = (ZOLTAN_GNO_TYPE)xGID[i] - fast_build_base;
    }
    else {
      xGNO = (ZOLTAN_GNO_TYPE *)xGID;
      if (fast_build_base)
        for (i = 0; i < nX; i++)
          xGNO[i] -= fast_build_base;
    }

    for (i = 0; i < num_requested; i++)
      requested_GNOs[i] = (ZOLTAN_GNO_TYPE)requested_GIDs[i]
                        - fast_build_base;
     
    tmp = (ZOLTAN_GNO_TYPE)nX; 
    MPI_Allreduce(&tmp, &matrix->globalX, 1, zoltan_gno_mpi_type, MPI_SUM, zz->Communicator);
  }

  /* I store : xGNO, xGID, xpid,  */

  ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, gno_size_for_dd, zz->Num_GID,
			   sizeof(int), matrix->globalX/zz->Num_Proc, 0);
  CHECK_IERR;

  /* Hope a linear assignment will help a little */
  if (matrix->globalX/zz->Num_Proc)
    Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc);
  /* Associate all the data with our xGNO */
  xpid = (int*)ZOLTAN_MALLOC(nX*sizeof(int));
  if (nX >0 && xpid == NULL) MEMORY_ERROR;
  for (i = 0 ; i < nX ; ++i)
    xpid[i] = zz->Proc;

  Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)xGNO, xGID, (char *)xpid, NULL, nX);
  ZOLTAN_FREE(&xpid);

  if (matrix->opts.pinwgt)
    matrix->pinwgtdim = zz->Edge_Weight_Dim;
  else
    matrix->pinwgtdim = 0;

  ierr = matrix_get_edges(zz, matrix, &yGID, &pinID, nX, &xGID, &xLID, &xGNO, &xwgt, use_full_dd);
  CHECK_IERR;
  matrix->nY_ori = matrix->nY;

  if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)){
    goto End;
  }

  if (matrix->opts.enforceSquare && matrix->redist) {
    /* Convert yGID to yGNO using the same translation as x */
    /* Needed for graph : rowID = colID */
    /* y and x may have different distributions */
    matrix->yGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_MALLOC(matrix->nY * sizeof(ZOLTAN_GNO_TYPE));
    if (matrix->nY && matrix->yGNO == NULL) {
      ZOLTAN_FREE(&pinID);
      MEMORY_ERROR;
    }
    ierr = Zoltan_DD_Find (dd, yGID, (ZOLTAN_ID_PTR)(matrix->yGNO), NULL, NULL,
		    matrix->nY, NULL);
    if (ierr != ZOLTAN_OK) {
      ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Hyperedge GIDs don't match.\n");
      ierr = ZOLTAN_FATAL;
      ZOLTAN_FREE(&pinID);
      goto End;
    }
  }

  if (matrix->opts.local) { /* keep only local edges */
    proclist = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int));
    if (matrix->nPins && proclist == NULL) { 
      ZOLTAN_FREE(&pinID);
      MEMORY_ERROR;
    }
  }
  else
    proclist = NULL;

  /* Convert pinID to pinGNO using the same translation as x */
  if (use_full_dd) {
    matrix->pinGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_MALLOC(matrix->nPins* sizeof(ZOLTAN_GNO_TYPE));
    if ((matrix->nPins > 0) && (matrix->pinGNO == NULL)) {
        ZOLTAN_FREE(&pinID);
        MEMORY_ERROR;
    }

    ierr = Zoltan_DD_Find (dd, pinID, (ZOLTAN_ID_PTR)(matrix->pinGNO), NULL, NULL,
			   matrix->nPins, proclist);
    if (ierr != ZOLTAN_OK) {
      ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Undefined GID found.\n");
      ierr = ZOLTAN_FATAL;
      goto End;
    }
    ZOLTAN_FREE(&pinID);
    Zoltan_DD_Destroy(&dd);
    dd = NULL;
  }
  else {
    if (sizeof(ZOLTAN_GNO_TYPE) != sizeof(ZOLTAN_ID_TYPE)){
      matrix->pinGNO = (ZOLTAN_GNO_TYPE *)ZOLTAN_MALLOC(matrix->nPins * sizeof(ZOLTAN_GNO_TYPE));
      if (matrix->nPins && !matrix->pinGNO){
        ZOLTAN_FREE(&pinID);
        MEMORY_ERROR;
      }
      for (i=0; i < matrix->nPins; i++)
        matrix->pinGNO[i] = (ZOLTAN_GNO_TYPE)pinID[i] - fast_build_base;
      
      ZOLTAN_FREE(&pinID);
    }
    else{
      matrix->pinGNO = (ZOLTAN_GNO_TYPE *) pinID;
      if (fast_build_base)
        for (i=0; i < matrix->nPins; i++)
          matrix->pinGNO[i] -= fast_build_base;
      pinID = NULL;
    }
  }

/*   if (matrix->opts.local) {  /\* keep only local edges *\/ */
/*     int *nnz_list; /\* nnz offset to delete *\/ */
/*     int nnz;       /\* number of nnz to delete *\/ */
/*     int i; */

/*     nnz_list = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int)); */
/*     if (matrix->nPins && nnz_list == NULL) MEMORY_ERROR; */
/*     for (i = 0, nnz=0 ; i < matrix->nPins ; ++i) { */
/*       if (proclist[i] == zz->Proc) continue; */
/*       nnz_list[nnz++] = i; */
/*     } */
/*     ZOLTAN_FREE(&proclist); */
/*     Zoltan_Matrix_Delete_nnz(zz, matrix, nnz, nnz_list); */
/*   } */

  if (!matrix->opts.enforceSquare) {
    /* Hyperedges name translation is different from the one of vertices */
    matrix->yGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_CALLOC(matrix->nY, sizeof(ZOLTAN_GNO_TYPE));
    if (matrix->nY && matrix->yGNO == NULL) MEMORY_ERROR;

    /*     int nGlobalEdges = 0; */
    ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, matrix->yGNO, matrix->nY,
					     matrix->opts.randomize, &matrix->globalY);
    CHECK_IERR;

/*     /\**************************************************************************************** */
/*      * If it is desired to remove dense edges, divide the list of edges into */
/*      * two lists.  The ZHG structure will contain the removed edges (if final_output is true), */
/*      * and the kept edges will be returned. */
/*      ****************************************************************************************\/ */
/*     totalNumEdges = zhg->globalHedges; */

/*     ierr = remove_dense_edges_matrix(zz, zhg, edgeSizeThreshold, final_output, */
/*				     &nLocalEdges, &nGlobalEdges, &nPins, */
/*				     &edgeGNO, &edgeSize, &edgeWeight, &pinGNO, &pinProcs); */

/*     if (nGlobalEdges < totalNumEdges){ */
/*       /\* re-assign edge global numbers if any edges were removed *\/ */
/*       ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, edgeGNO, nLocalEdges, */
/*					       randomizeInitDist, &totalNumEdges); */
/*       if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { */
/*	ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error reassigning global numbers to edges"); */
/*	goto End; */
/*       } */
/*     } */

      /* We have to define ddY : yGNO, yGID, ywgt */
      ierr = Zoltan_DD_Create (&matrix->ddY, zz->Communicator, gno_size_for_dd, zz->Num_GID,
			       0, matrix->globalY/zz->Num_Proc, 0);
      /* Hope a linear assignment will help a little */
      if (matrix->globalY/zz->Num_Proc)
        Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddY, matrix->globalY/zz->Num_Proc);
      /* Associate all the data with our yGNO */
      Zoltan_DD_Update (matrix->ddY, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, NULL, NULL, matrix->nY);
  }

 End:
  ZOLTAN_FREE(&xpid);
  ZOLTAN_FREE(&xLID);
  ZOLTAN_FREE(&xGNO);
  ZOLTAN_FREE(&xGID);
  ZOLTAN_FREE(&xwgt);
  ZOLTAN_FREE(&Input_Parts);
  ZOLTAN_FREE(&proclist);
  if (dd != NULL)
    Zoltan_DD_Destroy(&dd);
  /* Already stored in the DD */
  ZOLTAN_FREE(&yGID);

  ZOLTAN_TRACE_EXIT(zz, yo);

  return (ierr);
}
Exemple #7
0
int Zoltan_Preprocess_Graph(
  ZZ *zz,                               /* Zoltan structure */
  ZOLTAN_ID_PTR *global_ids,
  ZOLTAN_ID_PTR *local_ids,
  ZOLTAN_Third_Graph *gr,              /* Graph for third part libs */
  ZOLTAN_Third_Geom  *geo,
  ZOLTAN_Third_Part  *prt,
  ZOLTAN_Third_Vsize *vsp
)
{
  static char *yo = "Zoltan_Preprocess_Graph";
  int ierr;
  float *float_vwgt, *float_ewgts;
  char msg[256];


  char add_obj_weight[MAX_PARAM_STRING_LEN+1];

  ZOLTAN_TRACE_ENTER(zz, yo);

  /* Initialize all local pointers to NULL. This is necessary
   * because we free all non-NULL pointers upon errors.
   */
  gr->vtxdist = gr->xadj = gr->adjncy = NULL;
  gr->vwgt = gr->ewgts = NULL;
  float_vwgt = float_ewgts = NULL;

  if (gr->obj_wgt_dim >= 0) {
    /* Check weight dimensions */
    if (zz->Obj_Weight_Dim<0){
      sprintf(msg, "Object weight dimension is %d, "
	      "but should be >= 0. Using Obj_Weight_Dim = 0.",
	      zz->Obj_Weight_Dim);
      ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
      gr->obj_wgt_dim = 0;
    }
    else {
      gr->obj_wgt_dim = zz->Obj_Weight_Dim;
    }
  }
  else
    gr->obj_wgt_dim = 0;
  if (gr->edge_wgt_dim >= 0) {
    if (zz->Edge_Weight_Dim<0){
      sprintf(msg, "Edge weight dimension is %d, "
	      "but should be >= 0. Using Edge_Weight_Dim = 0.",
	      zz->Edge_Weight_Dim);
      ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
      gr->edge_wgt_dim = 0;
    }
    else if (zz->Edge_Weight_Dim>1){
      ZOLTAN_PRINT_WARN(zz->Proc, yo, "This method does not support "
			"multidimensional edge weights. Using Edge_Weight_Dim = 1.");
      gr->edge_wgt_dim = 1;
    }
    else {
      gr->edge_wgt_dim = zz->Edge_Weight_Dim;
    }
  }
  else
      gr->edge_wgt_dim = 0;

  if (gr->graph_type >= 0)
    /* Default graph type is GLOBAL. */
    gr->graph_type = GLOBAL_GRAPH;
  else
    gr->graph_type = - gr->graph_type;

  /* Get parameter options shared by ParMetis and Jostle */
  gr->check_graph = 1;          /* default */
  gr->scatter = 1;              /* default */
  gr->final_output = 0;
  strcpy(add_obj_weight, "NONE");  /* default */
  Zoltan_Bind_Param(Graph_params, "CHECK_GRAPH", (void *) &gr->check_graph);
  Zoltan_Bind_Param(Graph_params, "SCATTER_GRAPH", (void *) &gr->scatter);
  Zoltan_Bind_Param(Graph_params, "FINAL_OUTPUT", (void *) &gr->final_output);
  Zoltan_Bind_Param(Graph_params, "ADD_OBJ_WEIGHT", (void *) add_obj_weight);
  Zoltan_Assign_Param_Vals(zz->Params, Graph_params, zz->Debug_Level, zz->Proc,
			   zz->Debug_Proc);

  /* If reorder is true, we already have the id lists. Ignore weights. */
  if ((*global_ids == NULL) || (!gr->id_known)){
    int * input_part;
    ierr = Zoltan_Get_Obj_List(zz, &gr->num_obj, global_ids, local_ids,
			       gr->obj_wgt_dim, &float_vwgt, &input_part);
    if (prt) {
      prt->input_part = input_part;
    }
    else { /* Ordering, dont need part */
      ZOLTAN_FREE(&input_part);
    }
    if (ierr){
      /* Return error */
      ZOLTAN_PARMETIS_ERROR(ierr, "Get_Obj_List returned error.");
    }
  }


  /* Build Graph for third party library data structures, or just get vtxdist. */
  ierr = Zoltan_Build_Graph(zz, gr->graph_type, gr->check_graph, gr->num_obj,
			    *global_ids, *local_ids, gr->obj_wgt_dim, gr->edge_wgt_dim,
			    &gr->vtxdist, &gr->xadj, &gr->adjncy, &float_ewgts, &gr->adjproc);
  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){
      ZOLTAN_PARMETIS_ERROR(ierr, "Zoltan_Build_Graph returned error.");
  }

  if (prt) {
    prt->part_sizes = prt->input_part_sizes;
    if (gr->num_obj >0) {
      prt->part = (indextype *)ZOLTAN_MALLOC((gr->num_obj+1) * sizeof(indextype));
      if (!prt->part){
	/* Not enough memory */
	ZOLTAN_PARMETIS_ERROR(ZOLTAN_MEMERR, "Out of memory.");
      }
      memcpy (prt->part, prt->input_part, (gr->num_obj) * sizeof(indextype));
    }
    else {
      prt->input_part = prt->part = NULL;
    }
  }

  /* Convert from float. */

  /* Get vertex weights if needed */
  if (gr->obj_wgt_dim){
    ierr = Zoltan_Preprocess_Scale_Weights (gr, float_vwgt, &gr->vwgt,
					    gr->num_obj, gr->obj_wgt_dim, 1, zz,
					    "vertex", gr->vtxdist[zz->Proc]);
    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){
      /* Return error code */
      ZOLTAN_PARMETIS_ERROR(ierr, "Error in scaling of weights.");
    }
    ZOLTAN_FREE(&float_vwgt);
  }

  if (strcasecmp(add_obj_weight, "NONE")){
    if (Zoltan_Preprocess_Add_Weight(zz, gr, prt, add_obj_weight) != ZOLTAN_OK)
      ZOLTAN_PARMETIS_ERROR(ierr, "Error in adding  vertex weights.");
  }

  /* Get edge weights if needed */
  if (gr->get_data)
    gr->num_edges = gr->xadj[gr->num_obj];
  else {
    gr->num_edges = 0;
    gr->edge_wgt_dim = 0;
  }

  if (gr->edge_wgt_dim){
    ierr = Zoltan_Preprocess_Scale_Weights (gr, float_ewgts, &gr->ewgts,
					    gr->num_edges, gr->edge_wgt_dim, 1, zz,
					    "edge", 0);
    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){
      /* Return error code */
      ZOLTAN_PARMETIS_ERROR(ierr, "Error in scaling of weights.");
    }
    if (!gr->final_output)
      ZOLTAN_FREE(&float_ewgts);
    else
      gr->float_ewgts = float_ewgts;
  }
  else
    ZOLTAN_FREE(&float_ewgts);

  if (geo){
    ierr = Zoltan_Preprocess_Extract_Geom (zz, global_ids, local_ids, gr, geo);
    if (ierr) {
      ZOLTAN_PARMETIS_ERROR(ZOLTAN_FATAL,
			    "Error returned from Zoltan_Preprocess_Extract_Geom");
    }
  }


  if (vsp) {
    ierr = Zoltan_Preprocess_Extract_Vsize (zz, global_ids, local_ids, gr, vsp);
    if (ierr) {
      ZOLTAN_PARMETIS_ERROR(ZOLTAN_FATAL,
			  "Error returned from Zoltan_Preprocess_Extract_Vsize");
    }
  }

  /* Scatter graph?
   * If data distribution is highly imbalanced, it is better to
   * redistribute the graph data structure before calling ParMetis.
   * After partitioning, the results must be mapped back.
   */
  if (gr->scatter < gr->scatter_min) gr->scatter = gr->scatter_min;

  if (gr->scatter>0) {
    ierr = Zoltan_Preprocess_Scatter_Graph (zz, gr, prt, geo, vsp);
    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
      ZOLTAN_PARMETIS_ERROR(ZOLTAN_FATAL,
			    "Error returned from Zoltan_Preprocess_Scatter_Graph");
    }
  }


  /* Verify that graph is correct */
  if (gr->get_data){
    int flag;

    if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
      flag = 2; /* Maximum output level */
    else
      flag = 1; /* Medium output level */
    ierr = Zoltan_Verify_Graph(zz->Communicator, gr->vtxdist, gr->xadj, gr->adjncy, gr->vwgt,
	      gr->ewgts, gr->obj_wgt_dim, gr->edge_wgt_dim, gr->graph_type, gr->check_graph, flag);

  }

 End:

  ZOLTAN_TRACE_EXIT(zz, yo);

  return (ierr);
}
Exemple #8
0
int Zoltan_LocalHSFC_Order(
			   ZZ *zz,               /* Zoltan structure */
			   int num_obj,          /* Number of (local) objects to order. */
			   ZOLTAN_ID_PTR gids,   /* List of global ids (local to this proc) */
			                         /* The application must allocate enough space */
			   ZOLTAN_ID_PTR lids,   /* List of local ids (local to this proc) */
			                         /* The application must allocate enough space */
			   int *rank,            /* rank[i] is the rank of gids[i] */
			   int *iperm,
			   ZOOS *order_opt       /* Ordering options, parsed by Zoltan_Order */
                           )
{

  static char *yo = "Zoltan_LocalHSFC_Order";

  int n, ierr=ZOLTAN_OK;

  double (*fhsfc)(ZZ*, double*);  /* space filling curve function */

  int wgt_dim=0; 
  float *obj_wgts=0;
  int *parts=0;

  int numGeomDims=0;
  double *geomArray=0;

  /* Variables for bounding box */
  double *minValInDim;
  double *maxValInDim;
  double *widthDim;


  double *hsfcKey=0;
  int *coordIndx=0;

  /* Counters */
  int objNum;
  int dimNum;

  int offset=0;

  int myrank;
  MPI_Comm_rank(MPI_COMM_WORLD,&myrank);

  ZOLTAN_TRACE_ENTER(zz, yo);

  /******************************************************************/
  /* If for some reason order_opt is NULL, allocate a new ZOOS here. */
  /* This should really never happen. */
  /******************************************************************/
  if (!order_opt)
  {
    order_opt = (ZOOS *) ZOLTAN_MALLOC(sizeof(ZOOS));
    strcpy(order_opt->method,"LOCAL_HSFC");
  }
  /******************************************************************/

  /* local HSFC only computes the rank vector */
  order_opt->return_args = RETURN_RANK; 


  /******************************************************************/
  /* Check that num_obj equals the number of objects on this proc. */
  /* This constraint may be removed in the future. */
  /******************************************************************/
  n = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
  if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN))
  {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Num_Obj returned error.");
    return(ZOLTAN_FATAL);
  }
  if (n != num_obj)
  {
    /* Currently this is a fatal error. */
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input num_obj does not equal the number of objects.");
    return(ZOLTAN_FATAL);
  }
  /******************************************************************/

  /******************************************************************/
  /* Get lists of objects                                           */
  /******************************************************************/
  ierr = Zoltan_Get_Obj_List(zz, &n, &gids, &lids, wgt_dim, &obj_wgts, &parts);
  if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN))
  {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Obj_List returned error.");
      return(ZOLTAN_FATAL);
  }
  /******************************************************************/

  /******************************************************************/
  /* Get geometry for objects*/
  /******************************************************************/
  ierr = Zoltan_Get_Coordinates(zz, n, gids, lids, &numGeomDims,
			       &geomArray);
  if (ierr != 0)
  {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Zoltan_Get_Coordinates returned error.");
      return(ZOLTAN_FATAL);
  }
  /******************************************************************/

  /******************************************************************/
  /* Place coords in bounding box                                   */
  /******************************************************************/
  minValInDim =  (double *) malloc(numGeomDims * sizeof (double));
  maxValInDim =  (double *) malloc(numGeomDims * sizeof (double));
  widthDim =  (double *) malloc(numGeomDims * sizeof (double));

  for(dimNum=0; dimNum<numGeomDims; dimNum++)
  {
    minValInDim[dimNum] = HUGE_VAL;
    maxValInDim[dimNum] = -HUGE_VAL;
  }

  /*************************************************************/
  /* Determine min, max, and width for each dimension          */
  /*************************************************************/
  for (objNum=0; objNum<n; objNum++)
  {
    for(dimNum=0; dimNum<numGeomDims; dimNum++)
    {
      if (geomArray[objNum * numGeomDims + dimNum] < minValInDim[dimNum])
      {
        minValInDim[dimNum] = geomArray[objNum * numGeomDims + dimNum];
      }
      if (geomArray[objNum * numGeomDims + dimNum] > maxValInDim[dimNum])
      {
        maxValInDim[dimNum] = geomArray[objNum * numGeomDims + dimNum];
      }
    }
  }

  for(dimNum=0; dimNum<numGeomDims; dimNum++)
  {
    widthDim[dimNum] = maxValInDim[dimNum] - minValInDim[dimNum]; 
  }
  /*************************************************************/

  /*************************************************************/
  /* Rescale values to fit in bounding box                     */
  /*************************************************************/
  for (objNum=0; objNum<n; objNum++)
  {
    for(dimNum=0; dimNum<numGeomDims; dimNum++)
    {
      geomArray[objNum * numGeomDims + dimNum] -= minValInDim[dimNum];
      geomArray[objNum * numGeomDims + dimNum] /= widthDim[dimNum];
    }
  }
  /*************************************************************/

  free(minValInDim); minValInDim=0;
  free(maxValInDim); maxValInDim=0;
  free(widthDim); widthDim=0;
  /******************************************************************/

  /******************************************************************/   
  /* Specify which HSFC function to use (based on dim) */
  /******************************************************************/
  if (numGeomDims==1)
  {
    fhsfc = Zoltan_HSFC_InvHilbert1d;
  }
  else if (numGeomDims==2)
  {
    fhsfc = Zoltan_HSFC_InvHilbert2d;
  }
  else if (numGeomDims==3)
  {
    fhsfc = Zoltan_HSFC_InvHilbert3d;
  }
  else /* this error should have been previously caught */
  {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Geometry should be of dimension 1, 2, or 3.");
      return(ZOLTAN_FATAL);
  }
  /******************************************************************/

  /******************************************************************/
  /* Generate hsfc keys and indices to be sorted                    */
  /******************************************************************/
  hsfcKey = (double *) malloc(n * sizeof (double));
  coordIndx = (int *) malloc(n *sizeof(int));
  for (objNum=0; objNum<n; objNum++)
  {
    hsfcKey[objNum] = fhsfc(zz, &(geomArray[objNum * numGeomDims]) );
    coordIndx[objNum] = objNum;
  }
  /******************************************************************/

  /******************************************************************/
  /* Sort indices based on keys                                     */
  /******************************************************************/
  Zoltan_quicksort_pointer_dec_double (coordIndx, hsfcKey, 0, n-1);
  /******************************************************************/


  /******************************************************************/
  /* get ranks                                                      */
  /******************************************************************/

  /******************************************************/
  /* Determine offsets                                  */
  /******************************************************/
  MPI_Scan(&n, &offset, 1, MPI_INT, MPI_SUM, zz->Communicator);
  offset -= n; /* MPI_Scan is inclusive, so subtract off local size */
  /******************************************************/

  for(objNum=0; objNum<n; objNum++)
  {
    /*MMW temporary hack to make Cedric's interface give me want I need */
    /*rank[coordIndx[objNum]] = objNum + offset; */
    rank[objNum] = coordIndx[objNum] + offset; 
  }

  /******************************************************************/

  /* iperm is to be deprecated so not calculated*/

  free(hsfcKey);
  free(coordIndx);

  ZOLTAN_TRACE_EXIT(zz, yo);

  return (ZOLTAN_OK);

}