Ejemplo n.º 1
0
int Zoltan_LB_Free_Data(
    ZOLTAN_ID_PTR *import_global_ids, /* Array of global IDs for non-local objects
                                    assigned to this processor in the new
                                    decomposition.                           */
    ZOLTAN_ID_PTR *import_local_ids,  /* Array of local IDs for non-local objects
                                    assigned to the processor in the new
                                    decomposition.                           */
    int **import_procs,           /* Array of processor IDs of processors owning
                                   the non-local objects that are assigned to
                                   this processor in the new decomposition.  */
    ZOLTAN_ID_PTR *export_global_ids, /* Array of global IDs of
                                   objects to be exported to other processors
                                   to establish the new decomposition.       */
    ZOLTAN_ID_PTR *export_local_ids,  /* Array of local IDs of
                                   objects to be exported to other processors
                                   to establish the new decomposition.       */
    int **export_procs            /* Array of processor IDs
                                   to which objects will be exported
                                   to establish the new decomposition.       */
)
{
    /*
     *  Routine to free the arrays returning the results of the load balancing.
     */

    Zoltan_LB_Free_Part(import_global_ids, import_local_ids, import_procs, NULL);
    Zoltan_LB_Free_Part(export_global_ids, export_local_ids, export_procs, NULL);

    return (ZOLTAN_OK);

}
Ejemplo n.º 2
0
int main(int argc, char *argv[])
{
  int rc, i; 
  ZOLTAN_GNO_TYPE numGlobalVertices;
  float ver;
  char dimstring[16];
  double min, max, avg, local;

  struct Zoltan_Struct *zz;
  int changes, numGidEntries, numLidEntries, numImport, numExport;
  ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids; 
  int *importProcs, *importToPart, *exportProcs, *exportToPart;

#ifdef HOST_LINUX
  signal(SIGSEGV, meminfo_signal_handler);
  signal(SIGINT, meminfo_signal_handler);
  signal(SIGTERM, meminfo_signal_handler);
  signal(SIGABRT, meminfo_signal_handler);
  signal(SIGFPE, meminfo_signal_handler);
#endif

  /******************************************************************
  ** Problem size
  ******************************************************************/

  numGlobalVertices = NUM_GLOBAL_VERTICES;
  vertexWeightDim = VERTEX_WEIGHT_DIMENSION;
  vertexDim = VERTEX_DIMENSION;

  if (argc > 1){
    sscanf(argv[1], "%zd", &numGlobalVertices);
    if (argc > 2){
      vertexWeightDim = atoi(argv[2]);
      if (argc > 3){
        vertexDim = atoi(argv[3]);
      }
    }
  }

  sprintf(dimstring,"%d",vertexWeightDim);

  /******************************************************************
  ** Initialize MPI and Zoltan
  ******************************************************************/

  MPI_Init(&argc, &argv);
  MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
  MPI_Comm_size(MPI_COMM_WORLD, &numProcs);

  rc = Zoltan_Initialize(argc, argv, &ver);

  if (rc != ZOLTAN_OK){
    printf("sorry...\n");
    MPI_Finalize();
    exit(1);
  }

  Zoltan_Memory_Debug(2);

  /******************************************************************
  ** Create vertices
  ******************************************************************/

  rc = create_vertices(numGlobalVertices, vertexDim, vertexWeightDim, numProcs, myRank);

  if (rc){
    fprintf(stderr,"Process rank %d: insufficient memory\n",myRank);
    MPI_Finalize();
    exit(1);
  }

  first_gid = vertex_gid[myRank];

  /******************************************************************
  ** Create a Zoltan library structure for this instance of load
  ** balancing.  Set the parameters and query functions that will
  ** govern the library's calculation.  See the Zoltan User's
  ** Guide for the definition of these and many other parameters.
  ******************************************************************/

  zz = Zoltan_Create(MPI_COMM_WORLD);

  /* General parameters */

  Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
  Zoltan_Set_Param(zz, "LB_METHOD", "RIB");
  Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1"); 
  Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
  Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", dimstring);
  Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");

  /* RIB parameters */

  Zoltan_Set_Param(zz, "RIB_OUTPUT_LEVEL", "0");

  /* Query functions, to provide geometry to Zoltan */

  Zoltan_Set_Num_Obj_Fn(zz, get_number_of_objects, NULL);
  Zoltan_Set_Obj_List_Fn(zz, get_object_list, NULL);
  Zoltan_Set_Num_Geom_Fn(zz, get_num_geometry, NULL);
  Zoltan_Set_Geom_Multi_Fn(zz, get_geometry_list, NULL);
  Zoltan_Set_Part_Multi_Fn(zz, get_partition_list, NULL);

  /******************************************************************
  ** Zoltan can now partition the vertices in the simple mesh.
  ** In this simple example, we assume the number of partitions is
  ** equal to the number of processes.  Process rank 0 will own
  ** partition 0, process rank 1 will own partition 1, and so on.
  ******************************************************************/

  if (myRank == 0){
    printf("Run Zoltan\n");
  }

  rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
        &changes,        /* 1 if partitioning was changed, 0 otherwise */ 
        &numGidEntries,  /* Number of integers used for a global ID */
        &numLidEntries,  /* Number of integers used for a local ID */
        &numImport,      /* Number of vertices to be sent to me */
        &importGlobalGids,  /* Global IDs of vertices to be sent to me */
        &importLocalGids,   /* Local IDs of vertices to be sent to me */
        &importProcs,    /* Process rank for source of each incoming vertex */
        &importToPart,   /* New partition for each incoming vertex */
        &numExport,      /* Number of vertices I must send to other processes*/
        &exportGlobalGids,  /* Global IDs of the vertices I must send */
        &exportLocalGids,   /* Local IDs of the vertices I must send */
        &exportProcs,    /* Process to which I send each of the vertices */
        &exportToPart);  /* Partition to which each vertex will belong */

  if (rc != ZOLTAN_OK){
    if (myRank == 0)printf("sorry...\n");
    MPI_Finalize();
    Zoltan_Destroy(&zz);
    exit(0);
  }

  /******************************************************************
  ** Check the balance of the partitions before running zoltan.
  ** The query function get_partition_list() will give the 
  ** partitions of the vertices before we called Zoltan.
  ******************************************************************/

  if (myRank == 0){
    printf("\nBALANCE before running Zoltan\n");
  }

  rc = Zoltan_LB_Eval_Balance(zz, 1, NULL);

  if (rc != ZOLTAN_OK){
    printf("sorry first LB_Eval_Balance...\n");
    MPI_Finalize();
    Zoltan_Destroy(&zz);
    exit(0);
  }

  /******************************************************************
  ** Print out the balance of the new partitions.
  ******************************************************************/
 
  vertex_part = (int *)malloc(sizeof(int) * numLocalVertices);

  if (!vertex_part){
    printf("sorry memory error...\n");
    MPI_Finalize();
    Zoltan_Destroy(&zz);
    exit(0);
  }

  for (i=0; i < numLocalVertices; i++){
    vertex_part[i] = myRank;
  }

  if (numExport > 0){
    for (i=0; i < numExport; i++){
      vertex_part[exportLocalGids[i]] = exportToPart[i];
    }
  }

  if (myRank == 0){
    printf("\nBALANCE after running Zoltan\n");
  }

  rc = Zoltan_LB_Eval_Balance(zz, 1, NULL);

  if (rc != ZOLTAN_OK){
    printf("sorry second LB_Eval_Balance...\n");
    MPI_Finalize();
    Zoltan_Destroy(&zz);
    exit(0);
  }

  /******************************************************************
  ** Free the arrays allocated by Zoltan_LB_Partition, and free
  ** the storage allocated for the Zoltan structure.
  ******************************************************************/

  if (myRank == 0){
    printf("Free structures\n");
  }

  Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, 
                      &importProcs, &importToPart);
  Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, 
                      &exportProcs, &exportToPart);

  Zoltan_Destroy(&zz);

  if (vertex_part) free(vertex_part);
  if (v_x) free(v_x);
  if (v_y) free(v_y);
  if (v_z) free(v_z);
  if (vertex_weight) free(vertex_weight);
  if (vertex_gid) free(vertex_gid);

  /**********************
  ** all done ***********
  **********************/

  local= (double)Zoltan_Memory_Usage(ZOLTAN_MEM_STAT_MAXIMUM)/(1024.0*1024);
  MPI_Reduce(&local, &avg, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
  avg /= (double)numProcs;
  MPI_Reduce(&local, &max, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
  MPI_Reduce(&local, &min, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);

  if (myRank == 0){
    printf("Total MBytes in use by test while Zoltan is running: %12.3lf\n",
             mbytes/(1024.0*1024));
    printf("Min/Avg/Max of maximum MBytes in use by Zoltan:    %12.3lf / %12.3lf / %12.3lf\n",
             min, avg, max);
  }

  MPI_Finalize();

  return 0;
}
Ejemplo n.º 3
0
int run_zoltan(struct Zoltan_Struct *zz, Matrix A)
{
  /* Variables returned by Zoltan */
  ZOLTAN_ID_PTR import_gids = NULL;  /* Global nums of objs to be imported   */
  ZOLTAN_ID_PTR import_lids = NULL;  /* Local indices to objs to be imported */
  int   *import_procs = NULL;        /* Proc IDs of procs owning objs to be
                                        imported.                            */
  int   *import_to_part = NULL;      /* Partition #s to which imported objs 
                                        should be assigned.                  */
  ZOLTAN_ID_PTR export_gids = NULL;  /* Global nums of objs to be exported   */
  ZOLTAN_ID_PTR export_lids = NULL;  /* local indices to objs to be exported */
  int   *export_procs = NULL;        /* Proc IDs of destination procs for objs
                                        to be exported.                      */
  int   *export_to_part = NULL;      /* Partition #s for objs to be exported.*/
  int num_imported;              /* Number of objs to be imported.          */
  int num_exported;              /* Number of objs to be exported.          */
  int new_decomp;                /* Flag indicating whether the decomposition
                                    has changed                              */
  int num_gid_entries;           /* Number of array entries in a global ID.  */
  int num_lid_entries;           /* Number of array entries in a local ID.   */

  int *ptr;
  int myrank;
  int k;

  /*
   * Call Zoltan to compute a new decomposition. 
   * The result is given as export and/or import lists.
   * If you need only the export lists, perform the 
   * following call before this function:
   *   Zoltan_Set_Param(zz, "RETURN_LISTS", "EXPORT");
   *
   * In this example, we ignore the import_to_part/export_to_part
   * arguments, as they are only useful if we distinguish between
   * partitions and processors.
   */
  if (Zoltan_LB_Partition(zz, &new_decomp, &num_gid_entries, &num_lid_entries,
               &num_imported, &import_gids,
               &import_lids, &import_procs, &import_to_part,
               &num_exported, &export_gids,
               &export_lids, &export_procs, &export_to_part) == ZOLTAN_FATAL){
    printf("fatal:  error returned from Zoltan_LB_Partition()\n");
    return 0;
  }

  /* Print results. */
  MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
  if (new_decomp){
    printf("[Proc %1d] My data to export are:\n", myrank);
    for (k=0; k<num_exported; k++){
      ptr = (int *) &export_gids[num_gid_entries*k];
      printf("[Proc %1d] Export (%d,%d) to proc %d\n", myrank, 
         *ptr, *(ptr+1), export_procs[k]);
    }
  }
  else
    printf("[Proc %1d] No changes over here.\n", myrank);

  /* Data migration goes here. */

  /* Clean up */
  Zoltan_LB_Free_Part(&import_gids, &import_lids,
                      &import_procs, &import_to_part);
  Zoltan_LB_Free_Part(&export_gids, &export_lids,
                      &export_procs, &export_to_part);

  return 1;
}
Ejemplo n.º 4
0
int main(int argc, char *argv[])
{
  int i, rc;
  float ver;
  struct Zoltan_Struct *zz;
  int changes, numGidEntries, numLidEntries, numImport, numExport;
  int myRank, numProcs;
  ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
  int *importProcs, *importToPart, *exportProcs, *exportToPart;
  int *parts;
  FILE *fp;
  GRAPH_DATA myGraph;

  /******************************************************************
  ** Initialize MPI and Zoltan
  ******************************************************************/

  MPI_Init(&argc, &argv);
  MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
  MPI_Comm_size(MPI_COMM_WORLD, &numProcs);

  rc = Zoltan_Initialize(argc, argv, &ver);

  if (rc != ZOLTAN_OK){
    printf("sorry...\n");
    MPI_Finalize();
    exit(0);
  }

  /******************************************************************
  ** Read graph from input file and distribute it 
  ******************************************************************/

  fp = fopen(fname, "r");
  if (!fp){
    if (myRank == 0) fprintf(stderr,"ERROR: Can not open %s\n",fname);
    MPI_Finalize();
    exit(1);
  }
  fclose(fp);

  read_input_file(myRank, numProcs, fname, &myGraph);

  /******************************************************************
  ** Create a Zoltan library structure for this instance of load
  ** balancing.  Set the parameters and query functions that will
  ** govern the library's calculation.  See the Zoltan User's
  ** Guide for the definition of these and many other parameters.
  ******************************************************************/

  zz = Zoltan_Create(MPI_COMM_WORLD);

  /* General parameters */

/*
  Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
  Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH");
  Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1"); 
  Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
  Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "0");
  Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");
*/

     Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1"); 
     Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
     Zoltan_Set_Param(zz,"LB_METHOD","GRAPH");
#ifdef HAVE_PARMETIS
     Zoltan_Set_Param(zz,"GRAPH_PACKAGE","PARMETIS");
#else
  #ifdef HAVE_SCOTCH
     Zoltan_Set_Param(zz,"GRAPH_PACKAGE","SCOTCH");
  #endif
#endif
     Zoltan_Set_Param(zz,"EDGE_WEIGHT_DIM","1");
     Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "0");
     Zoltan_Set_Param(zz,"LB_APPROACH","REPARTITION");
     Zoltan_Set_Param(zz,"GRAPH_SYMMETRIZE","TRANSPOSE");
     Zoltan_Set_Param(zz,"GRAPH_SYM_WEIGHT","ADD");

  /* Graph parameters */

  Zoltan_Set_Param(zz, "CHECK_GRAPH", "2"); 
  Zoltan_Set_Param(zz, "PHG_EDGE_SIZE_THRESHOLD", ".35");  /* 0-remove all, 1-remove none */

  /* Query functions - defined in simpleQueries.h */

  Zoltan_Set_Num_Obj_Fn(zz, get_number_of_vertices, &myGraph);
  Zoltan_Set_Obj_List_Fn(zz, get_vertex_list, &myGraph);
  Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, &myGraph);
  Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, &myGraph);

  /******************************************************************
  ** Zoltan can now partition the simple graph.
  ** In this simple example, we assume the number of partitions is
  ** equal to the number of processes.  Process rank 0 will own
  ** partition 0, process rank 1 will own partition 1, and so on.
  ******************************************************************/

  rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
        &changes,        /* 1 if partitioning was changed, 0 otherwise */ 
        &numGidEntries,  /* Number of integers used for a global ID */
        &numLidEntries,  /* Number of integers used for a local ID */
        &numImport,      /* Number of vertices to be sent to me */
        &importGlobalGids,  /* Global IDs of vertices to be sent to me */
        &importLocalGids,   /* Local IDs of vertices to be sent to me */
        &importProcs,    /* Process rank for source of each incoming vertex */
        &importToPart,   /* New partition for each incoming vertex */
        &numExport,      /* Number of vertices I must send to other processes*/
        &exportGlobalGids,  /* Global IDs of the vertices I must send */
        &exportLocalGids,   /* Local IDs of the vertices I must send */
        &exportProcs,    /* Process to which I send each of the vertices */
        &exportToPart);  /* Partition to which each vertex will belong */

  if (rc != ZOLTAN_OK){
    printf("sorry...\n");
    MPI_Finalize();
    Zoltan_Destroy(&zz);
    exit(0);
  }

  /******************************************************************
  ** Visualize the graph partitioning before and after calling Zoltan.
  ******************************************************************/

  parts = (int *)malloc(sizeof(int) * myGraph.numMyVertices);

  for (i=0; i < myGraph.numMyVertices; i++){
    parts[i] = myRank;
  }

  if (myRank== 0){
    printf("\nGraph partition before calling Zoltan\n");
  }

  showGraphPartitions(myRank, myGraph.numMyVertices, myGraph.vertexGID, parts, numProcs);

  for (i=0; i < numExport; i++){
    parts[exportLocalGids[i]] = exportToPart[i];
  }

  if (myRank == 0){
    printf("Graph partition after calling Zoltan\n");
  }

  showGraphPartitions(myRank, myGraph.numMyVertices, myGraph.vertexGID, parts, numProcs);

  free(parts);

  /******************************************************************
  ** Free the arrays allocated by Zoltan_LB_Partition, and free
  ** the storage allocated for the Zoltan structure.
  ******************************************************************/

  Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, 
                      &importProcs, &importToPart);
  Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, 
                      &exportProcs, &exportToPart);

  Zoltan_Destroy(&zz);

  /**********************
  ** all done ***********
  **********************/

  MPI_Finalize();

  if (myGraph.numMyVertices > 0){
    free(myGraph.vertexGID);
    free(myGraph.nborIndex);
    if (myGraph.numAllNbors > 0){
      free(myGraph.nborGID);
      free(myGraph.nborProc);
    }
  }

  return 0;
}
Ejemplo n.º 5
0
int main(int argc, char *argv[])
{
  int rc, i, myRank, numProcs;
  float ver;
  struct Zoltan_Struct *zz;
  int changes, numGidEntries, numLidEntries, numImport, numExport;
  ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids; 
  int *importProcs, *importToPart, *exportProcs, *exportToPart;
  int *parts;
  FILE *fp;
  MESH_DATA myMesh;

  /******************************************************************
  ** Initialize MPI and Zoltan
  ******************************************************************/

  MPI_Init(&argc, &argv);
  MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
  MPI_Comm_size(MPI_COMM_WORLD, &numProcs);

  rc = Zoltan_Initialize(argc, argv, &ver);

  if (rc != ZOLTAN_OK){
    printf("sorry...\n");
    MPI_Finalize();
    exit(0);
  }

  /******************************************************************
  ** Read geometry from input file and distribute it unevenly
  ******************************************************************/

  fp = fopen(fname, "r");
  if (!fp){
    if (myRank == 0) fprintf(stderr,"ERROR: Can not open %s\n",fname);
    MPI_Finalize();
    exit(1);
  }
  fclose(fp);

  read_input_objects(myRank, numProcs, fname, &myMesh);

  /******************************************************************
  ** Create a Zoltan library structure for this instance of load
  ** balancing.  Set the parameters and query functions that will
  ** govern the library's calculation.  See the Zoltan User's
  ** Guide for the definition of these and many other parameters.
  ******************************************************************/

  zz = Zoltan_Create(MPI_COMM_WORLD);

  /* General parameters */

  Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
  Zoltan_Set_Param(zz, "LB_METHOD", "RCB");
  Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1"); 
  Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
  Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "0");
  Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");

  /* RCB parameters */

  Zoltan_Set_Param(zz, "RCB_OUTPUT_LEVEL", "0");
  Zoltan_Set_Param(zz, "RCB_RECTILINEAR_BLOCKS", "1"); 
  /*Zoltan_Set_Param(zz, "RCB_RECTILINEAR_BLOCKS", "0"); */

  /* Query functions, to provide geometry to Zoltan */

  Zoltan_Set_Num_Obj_Fn(zz, get_number_of_objects, &myMesh);
  Zoltan_Set_Obj_List_Fn(zz, get_object_list, &myMesh);
  Zoltan_Set_Num_Geom_Fn(zz, get_num_geometry, &myMesh);
  Zoltan_Set_Geom_Multi_Fn(zz, get_geometry_list, &myMesh);

  /******************************************************************
  ** Zoltan can now partition the vertices in the simple mesh.
  ** In this simple example, we assume the number of partitions is
  ** equal to the number of processes.  Process rank 0 will own
  ** partition 0, process rank 1 will own partition 1, and so on.
  ******************************************************************/

  rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
        &changes,        /* 1 if partitioning was changed, 0 otherwise */ 
        &numGidEntries,  /* Number of integers used for a global ID */
        &numLidEntries,  /* Number of integers used for a local ID */
        &numImport,      /* Number of vertices to be sent to me */
        &importGlobalGids,  /* Global IDs of vertices to be sent to me */
        &importLocalGids,   /* Local IDs of vertices to be sent to me */
        &importProcs,    /* Process rank for source of each incoming vertex */
        &importToPart,   /* New partition for each incoming vertex */
        &numExport,      /* Number of vertices I must send to other processes*/
        &exportGlobalGids,  /* Global IDs of the vertices I must send */
        &exportLocalGids,   /* Local IDs of the vertices I must send */
        &exportProcs,    /* Process to which I send each of the vertices */
        &exportToPart);  /* Partition to which each vertex will belong */

  if (rc != ZOLTAN_OK){
    printf("sorry...\n");
    MPI_Finalize();
    Zoltan_Destroy(&zz);
    exit(0);
  }

  /******************************************************************
  ** Visualize the mesh partitioning before and after calling Zoltan.
  ******************************************************************/

  parts = (int *)malloc(sizeof(int) * myMesh.numMyPoints);

  for (i=0; i < myMesh.numMyPoints; i++){
    parts[i] = myRank;
  }

  if (myRank== 0){
    printf("\nMesh partition assignments before calling Zoltan\n");
  }

  showSimpleMeshPartitions(myRank, myMesh.numMyPoints, myMesh.myGlobalIDs, parts);

  for (i=0; i < numExport; i++){
    parts[exportLocalGids[i]] = exportToPart[i];
  }

  if (myRank == 0){
    printf("Mesh partition assignments after calling Zoltan\n");
  }

  showSimpleMeshPartitions(myRank, myMesh.numMyPoints, myMesh.myGlobalIDs, parts);

  free(parts);

  /******************************************************************
  ** Free the arrays allocated by Zoltan_LB_Partition, and free
  ** the storage allocated for the Zoltan structure.
  ******************************************************************/

  Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, 
                      &importProcs, &importToPart);
  Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, 
                      &exportProcs, &exportToPart);

  Zoltan_Destroy(&zz);

  /**********************
  ** all done ***********
  **********************/

  MPI_Finalize();

  if (myMesh.numMyPoints > 0){
    free(myMesh.myGlobalIDs);
    free(myMesh.x);
    free(myMesh.y);
  }

  return 0;
}
Ejemplo n.º 6
0
std::vector<int> zoltanGraphPartitionGridOnRoot(const CpGrid& cpgrid,
                                          const CollectiveCommunication<MPI_Comm>& cc,
                                          int root)
{
    int rc;
    float ver;
    struct Zoltan_Struct *zz;
    int changes, numGidEntries, numLidEntries, numImport, numExport;
    ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
    int *importProcs, *importToPart, *exportProcs, *exportToPart;
    int argc=0;
    char** argv;
    rc = Zoltan_Initialize(argc, argv, &ver);
    zz = Zoltan_Create(cc);
    if ( rc != ZOLTAN_OK )
    {
        OPM_THROW(std::runtime_error, "Could not initialize Zoltan!");
    }

    Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
    Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH");
    Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION");
    Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
    Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
    Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");
    Zoltan_Set_Param(zz, "DEBUG_LEVEL", "3");
    Zoltan_Set_Param(zz, "CHECK_GRAPH", "2");
    Zoltan_Set_Param(zz, "PHG_EDGE_SIZE_THRESHOLD", ".35");  /* 0-remove all, 1-remove none */

    bool pretendEmptyGrid = cc.rank()!=root;

    Dune::cpgrid::setCpGridZoltanGraphFunctions(zz, cpgrid, pretendEmptyGrid);

    rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
                             &changes,        /* 1 if partitioning was changed, 0 otherwise */
                             &numGidEntries,  /* Number of integers used for a global ID */
                             &numLidEntries,  /* Number of integers used for a local ID */
                             &numImport,      /* Number of vertices to be sent to me */
                             &importGlobalGids,  /* Global IDs of vertices to be sent to me */
                             &importLocalGids,   /* Local IDs of vertices to be sent to me */
                             &importProcs,    /* Process rank for source of each incoming vertex */
                             &importToPart,   /* New partition for each incoming vertex */
                             &numExport,      /* Number of vertices I must send to other processes*/
                             &exportGlobalGids,  /* Global IDs of the vertices I must send */
                             &exportLocalGids,   /* Local IDs of the vertices I must send */
                             &exportProcs,    /* Process to which I send each of the vertices */
                             &exportToPart);  /* Partition to which each vertex will belong */
    int size = cpgrid.numCells();
    int         rank  = cc.rank();
    std::vector<int> parts=std::vector<int>(size, rank);

    for ( int i=0; i < numExport; ++i )
    {
        parts[exportLocalGids[i]] = exportProcs[i];
    }
    cc.broadcast(&parts[0], parts.size(), root);
    Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, &exportProcs, &exportToPart);
    Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, &importProcs, &importToPart);
    Zoltan_Destroy(&zz);
    return parts;
}
Ejemplo n.º 7
0
int main(int argc, char *argv[])
{
  int i, rc;
  int myRank, numProcs;
  float ver;
  struct Zoltan_Struct *zz;
  int changes, numGidEntries, numLidEntries, numImport, numExport, start_gid, num_nbors;
  ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
  int *importProcs, *importToPart, *exportProcs, *exportToPart;
  int *parts=NULL;
  ZOLTAN_ID_PTR lids=NULL;
  FILE *fp;
  struct Zoltan_DD_Struct *dd;
  GRAPH_DATA myGraph;
  int gid_length = 1;   /* our global IDs consist of 1 integer */
  int lid_length = 1;   /* our local IDs consist of 1 integer */

  /******************************************************************
  ** Initialize MPI and Zoltan
  ******************************************************************/

  MPI_Init(&argc, &argv);
  MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
  MPI_Comm_size(MPI_COMM_WORLD, &numProcs);

  rc = Zoltan_Initialize(argc, argv, &ver);

  if (rc != ZOLTAN_OK){
    printf("sorry...\n");
    MPI_Finalize();
    exit(0);
  }

  /******************************************************************
  ** Read graph from input file and distribute it 
  ******************************************************************/

  fp = fopen(fname, "r");
  if (!fp){
    if (myRank == 0) fprintf(stderr,"ERROR: Can not open %s\n",fname);
    MPI_Finalize();
    exit(1);
  }
  fclose(fp);

  read_input_file(myRank, numProcs, fname, &myGraph);

  /*fprintf(stderr,"%d have %d objects\n",myRank,myGraph.numMyVertices);*/

  /******************************************************************
  ** Create a distributed data directory which maps vertex
  ** global IDs to their current partition number.  We'll use this
  ** after migrating vertices, to update the partition in which
  ** our vertices neighbors are.
  **
  ** Our local IDs (array "lids") are of type ZOLTAN_ID_TYPE because
  ** we are using Zoltan's distributed data directory.  It assumes
  ** that a global ID is a sequence of "gid_length" ZOLTAN_ID_TYPEs.
  ** It assumes that a local ID is a sequence of "lid_length"
  ** ZOLTAN_ID_TYPEs.
  ******************************************************************/

  rc = Zoltan_DD_Create(&dd, MPI_COMM_WORLD, 
                           gid_length,    /* length of a global ID */
                           lid_length,    /* length of a local ID */
                           0,             /* length of user data  */
                           myGraph.numMyVertices,  /* hash table size */
                           0);                     /* debug level */

  parts = malloc(myGraph.numMyVertices * sizeof(int));
  lids = malloc(myGraph.numMyVertices * sizeof(ZOLTAN_ID_TYPE));
  
  for (i=0; i < myGraph.numMyVertices; i++){
    parts[i] = myRank;   /* part number of this vertex */
    lids[i] = (ZOLTAN_ID_TYPE)i;         /* local ID on my process for this vertex */
  }
  
  rc = Zoltan_DD_Update(dd, 
                        myGraph.vertexGID, 
                        lids,
                        NULL,
                        parts,
                        myGraph.numMyVertices);
  

  myGraph.dd = dd;

  /******************************************************************
  ** Create a Zoltan library structure for this instance of load
  ** balancing.  Set the parameters and query functions that will
  ** govern the library's calculation.  See the Zoltan User's
  ** Guide for the definition of these and many other parameters.
  ******************************************************************/

  zz = Zoltan_Create(MPI_COMM_WORLD);

  /* General parameters */

  Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
  Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH");
  Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION");
  Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1"); 
  Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
  Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");

  /* Graph parameters */

  Zoltan_Set_Param(zz, "CHECK_GRAPH", "2"); 
  Zoltan_Set_Param(zz, "PHG_EDGE_SIZE_THRESHOLD", ".35");  /* 0-remove all, 1-remove none */

  /* Query functions, defined in this source file */

  Zoltan_Set_Num_Obj_Fn(zz, get_number_of_vertices, &myGraph);
  Zoltan_Set_Obj_List_Fn(zz, get_vertex_list, &myGraph);
  Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, &myGraph);
  Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, &myGraph);

  Zoltan_Set_Obj_Size_Multi_Fn(zz, get_message_sizes,&myGraph);
  Zoltan_Set_Pack_Obj_Multi_Fn(zz, pack_object_messages,&myGraph);
  Zoltan_Set_Unpack_Obj_Multi_Fn(zz, unpack_object_messages,&myGraph);
  Zoltan_Set_Mid_Migrate_PP_Fn(zz, mid_migrate,&myGraph);

  /******************************************************************
  ** Visualize the graph partitioning before calling Zoltan.
  ******************************************************************/

  if (myRank== 0){
    printf("\nGraph partition before calling Zoltan\n");
  }

  showGraphPartitions(myRank, myGraph.dd);

  /******************************************************************
  ** Zoltan can now partition the simple graph.
  ** In this simple example, we assume the number of partitions is
  ** equal to the number of processes.  Process rank 0 will own
  ** partition 0, process rank 1 will own partition 1, and so on.
  ******************************************************************/

  rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
        &changes,        /* 1 if partitioning was changed, 0 otherwise */ 
        &numGidEntries,  /* Number of integers used for a global ID */
        &numLidEntries,  /* Number of integers used for a local ID */
        &numImport,      /* Number of vertices to be sent to me */
        &importGlobalGids,  /* Global IDs of vertices to be sent to me */
        &importLocalGids,   /* Local IDs of vertices to be sent to me */
        &importProcs,    /* Process rank for source of each incoming vertex */
        &importToPart,   /* New partition for each incoming vertex */
        &numExport,      /* Number of vertices I must send to other processes*/
        &exportGlobalGids,  /* Global IDs of the vertices I must send */
        &exportLocalGids,   /* Local IDs of the vertices I must send */
        &exportProcs,    /* Process to which I send each of the vertices */
        &exportToPart);  /* Partition to which each vertex will belong */

  if (rc != ZOLTAN_OK){
    printf("sorry...\n");
    MPI_Finalize();
    Zoltan_Destroy(&zz);
    exit(0);
  }

  /*fprintf(stderr,"%d export %d import %d\n",myRank,numExport,numImport);*/

  /******************************************************************
  ** Update the data directory with the new partition numbers
  ******************************************************************/

  for (i=0; i < numExport; i++){
    parts[exportLocalGids[i]] = exportToPart[i];
  }

  rc = Zoltan_DD_Update(dd, 
                        myGraph.vertexGID, 
                        lids,
                        NULL,
                        parts,
                        myGraph.numMyVertices);

  /******************************************************************
  ** Migrate vertices to new partitions
  ******************************************************************/

  rc = Zoltan_Migrate(zz, 
                      numImport, importGlobalGids, importLocalGids,
                      importProcs, importToPart,
                      numExport, exportGlobalGids, exportLocalGids,
                      exportProcs, exportToPart);


  /******************************************************************
  ** Use the data dictionary to find neighbors' partitions
  ******************************************************************/

  start_gid = myGraph.numMyVertices - numImport;
  num_nbors = myGraph.nborIndex[myGraph.numMyVertices] - myGraph.nborIndex[start_gid];

  rc = Zoltan_DD_Find(dd,
             (ZOLTAN_ID_PTR)(myGraph.nborGID + start_gid), NULL, NULL, 
              myGraph.nborPart + start_gid, num_nbors, NULL);

  /******************************************************************
  ** Visualize the graph partitioning after calling Zoltan.
  ******************************************************************/

  if (myRank == 0){
    printf("Graph partition after calling Zoltan\n");
  }
  showGraphPartitions(myRank, myGraph.dd);

  /******************************************************************
  ** Free the arrays allocated by Zoltan_LB_Partition, and free
  ** the storage allocated for the Zoltan structure.
  ******************************************************************/

  Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, 
                      &importProcs, &importToPart);
  Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, 
                      &exportProcs, &exportToPart);

  Zoltan_Destroy(&zz);

  /**********************
  ** all done ***********
  **********************/

  MPI_Finalize();

  if (myGraph.vertex_capacity > 0){
    free(myGraph.vertexGID);
    free(myGraph.nborIndex);
    if (myGraph.nbor_capacity > 0){
      free(myGraph.nborGID);
      free(myGraph.nborPart);
    }
  }

  if (parts) free(parts);
  if (lids) free(lids);

  return 0;
}
Ejemplo n.º 8
0
int main(int argc, char *argv[])
{
  int rc, i;
  int myRank, numProcs;
  float ver;
  struct Zoltan_Struct *zz;
  int changes, numGidEntries, numLidEntries, numImport, numExport;
  ZOLTAN_ID_PTR importGlobalGids, importLocalGids;
  ZOLTAN_ID_PTR exportGlobalGids, exportLocalGids; 
  int *importProcs, *importToPart, *exportProcs, *exportToPart;
  int *parts = NULL;

  FILE *fp;
  OBJECT_DATA myData;

  /******************************************************************
  ** Initialize MPI and Zoltan
  ******************************************************************/

  MPI_Init(&argc, &argv);
  MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
  MPI_Comm_size(MPI_COMM_WORLD, &numProcs);

  rc = Zoltan_Initialize(argc, argv, &ver);

  if (rc != ZOLTAN_OK){
    printf("Error initializing Zoltan\n");
    MPI_Finalize();
    exit(0);
  }

  /******************************************************************
  ** Read objects from input file and distribute them unevenly
  ******************************************************************/

  fp = fopen(fname, "r");
  if (!fp){
    if (myRank == 0) fprintf(stderr,"ERROR: Can not open %s\n",fname);
    MPI_Finalize();
    exit(1);
  }
  fclose(fp);

  read_input_objects(myRank, numProcs, fname, &myData);

  /******************************************************************
  ** Create a Zoltan library structure for this instance of load
  ** balancing.  Set the parameters and query functions.
  ******************************************************************/

  zz = Zoltan_Create(MPI_COMM_WORLD);

  /* General parameters */

  Zoltan_Set_Param(zz, "LB_METHOD", "BLOCK");  /* Zoltan method: "BLOCK" */
  Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1"); /* global ID is 1 integer */
  Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1"); /* local ID is 1 integer */
  Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "0"); /* we omit object weights */

  /* Query functions */

  Zoltan_Set_Num_Obj_Fn(zz, get_number_of_objects, &myData);
  Zoltan_Set_Obj_List_Fn(zz, get_object_list, &myData);

  /******************************************************************
  ** Call Zoltan to partition the objects.
  ******************************************************************/

  rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
        &changes,        /* 1 if partitioning was changed, 0 otherwise */ 
        &numGidEntries,  /* Number of integers used for a global ID */
        &numLidEntries,  /* Number of integers used for a local ID */
        &numImport,      /* Number of objects to be sent to me */
        &importGlobalGids,  /* Global IDs of objects to be sent to me */
        &importLocalGids,   /* Local IDs of objects to be sent to me */
        &importProcs,    /* Process rank for source of each incoming object */
        &importToPart,   /* New partition for each incoming object */
        &numExport,      /* Number of objects I must send to other processes*/
        &exportGlobalGids,  /* Global IDs of the objects I must send */
        &exportLocalGids,   /* Local IDs of the objects I must send */
        &exportProcs,    /* Process to which I send each of the objects */
        &exportToPart);  /* Partition to which each object will belong */

  if (rc != ZOLTAN_OK){
    printf("Error in Zoltan library\n");
    MPI_Finalize();
    Zoltan_Destroy(&zz);
    exit(0);
  }

  /******************************************************************
  ** Visualize the object partitioning before and after calling Zoltan.
  **
  ** In this example, partition number equals process rank.
  ******************************************************************/

  parts = (int *)malloc(sizeof(int) * myData.numMyObjects);

  for (i=0; i < myData.numMyObjects; i++){
    parts[i] = myRank;
  }

  if (myRank== 0){
    printf("\nObject partition assignments before calling Zoltan\n");
  }

  showSimpleMeshPartitions(myRank, myData.numMyObjects, myData.myGlobalIDs, parts);

  for (i=0; i < numExport; i++){
    parts[exportLocalGids[i]] = exportToPart[i];
  }

  if (myRank == 0){
    printf("Object partition assignments after calling Zoltan\n");
  }

  showSimpleMeshPartitions(myRank, myData.numMyObjects, myData.myGlobalIDs, parts);

  /******************************************************************
  ** Free the arrays allocated by Zoltan_LB_Partition, and free
  ** the storage allocated for the Zoltan structure.
  ******************************************************************/

  Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, 
                      &importProcs, &importToPart);
  Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, 
                      &exportProcs, &exportToPart);

  Zoltan_Destroy(&zz);

  MPI_Finalize();

  return 0;
}
Ejemplo n.º 9
0
  int MESH_PartitionWithZoltan(Mesh_ptr mesh, int nparts, int **part, int noptions, 
                               char **options, MSTK_Comm comm) { 

  MEdge_ptr fedge;
  MFace_ptr mf, oppf, rface;
  MRegion_ptr mr, oppr;
  List_ptr fedges, efaces, rfaces, fregions;
  int  i, j, k, id;
  int  nv, ne, nf, nr=0, nfe, nef, nfr, nrf, idx, idx2;
  int  numflag, nedgecut, ipos;
  int  wtflag;

  int rc;
  float ver;
  struct Zoltan_Struct *zz;
  GRAPH_DATA graph;
  int changes, numGidEntries, numLidEntries, numImport, numExport;
  ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
  int *importProcs, *importToPart, *exportProcs, *exportToPart;

  int rank;
  MPI_Comm_rank(comm,&rank);
 
  rc = Zoltan_Initialize(0, NULL, &ver);

  if (rc != ZOLTAN_OK){
    MSTK_Report("MESH_PartitionWithZoltan","Could not initialize Zoltan",MSTK_FATAL);
    MPI_Finalize();
    exit(0);
  }

  /******************************************************************
  ** Create a Zoltan library structure for this instance of partition 
  ********************************************************************/
  zz = Zoltan_Create(comm);

  /*****************************************************************
   ** Figure out partitioning method
   *****************************************************************/
  
  char partition_method_str[32];
  strcpy(partition_method_str,"RCB");  /* Default - Recursive Coordinate Bisection */
  if (noptions) {
    for (i = 0; i < noptions; i++) {
      if (strncmp(options[i],"LB_PARTITION",12) == 0) {
        char *result = NULL, instring[256];
        strcpy(instring,options[i]);
        result = strtok(instring,"=");
        result = strtok(NULL," ");
        strcpy(partition_method_str,result);
      }
    }
  }
  
  if (rank == 0) {
    char mesg[256];
    sprintf(mesg,"Using partitioning method %s for ZOLTAN\n",partition_method_str);
    MSTK_Report("MESH_PartitionWithZoltan",mesg,MSTK_MESG);
  }

  /* General parameters for Zoltan */
  Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
  Zoltan_Set_Param(zz, "LB_METHOD", partition_method_str);
  Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION");
  Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
  Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
  Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");


  graph.numMyNodes = 0;
  graph.numAllNbors = 0;
  graph.nodeGID = NULL;
  graph.nodeCoords = NULL;
  graph.nborIndex = NULL;
  graph.nborGID = NULL;
  graph.nborProc = NULL;

  if (strcmp(partition_method_str,"RCB") == 0) {
    if (rank == 0) {
      nr = MESH_Num_Regions(mesh);
      nf = MESH_Num_Faces(mesh);

      if (!nf && !nr)
        MSTK_Report("MESH_PartitionWithZoltan","Cannot partition wire meshes",
                    MSTK_FATAL);

      if (nr == 0) { /* Surface or planar mesh */

        int ndim = 2;       /* assume mesh is planar */
        idx = 0;
        MVertex_ptr mv;
        while ((mv = MESH_Next_Vertex(mesh,&idx))) {
          double vxyz[3];
          MV_Coords(mv,vxyz);
          if (vxyz[2] != 0.0) {
            ndim = 3;  /* non-planar or planar with non-zero z */
            break;
          }
        }
        NDIM_4_ZOLTAN = ndim-1;  /* ignore last dimension to avoid partitioning in that dimension */

        graph.numMyNodes = nf;

        graph.nodeGID = (ZOLTAN_ID_TYPE *) malloc(sizeof(ZOLTAN_ID_TYPE) * nf);
        graph.nodeCoords = (double *) malloc(sizeof(double) * NDIM_4_ZOLTAN * nf);

        idx = 0;
        while ((mf = MESH_Next_Face(mesh,&idx))) {
          double fxyz[MAXPV2][3], cen[3];
          int nfv;

          MF_Coords(mf,&nfv,fxyz);
          cen[0] = cen[1] = cen[2] = 0.0;
          for (j = 0; j < nfv; j++)
            for (k = 0; k < NDIM_4_ZOLTAN; k++) 
              cen[k] += fxyz[j][k];              
          for (k = 0; k < NDIM_4_ZOLTAN; k++) cen[k] /= nfv;

          id = MF_ID(mf);
          graph.nodeGID[id-1] = id;
          memcpy(&(graph.nodeCoords[NDIM_4_ZOLTAN*(id-1)]),cen,NDIM_4_ZOLTAN*sizeof(double));
        }

      }
      else { /* Volume mesh */

        int ndim = 3;
        NDIM_4_ZOLTAN = ndim-1;  /* ignore last dimension  to avoid partitioning in that dimension */
        graph.numMyNodes = nr;

        graph.nodeGID = (ZOLTAN_ID_TYPE *) malloc(sizeof(ZOLTAN_ID_TYPE) * nr);
        graph.nodeCoords = (double *) malloc(sizeof(double) * NDIM_4_ZOLTAN * nr);

        idx = 0;
        while ((mr = MESH_Next_Region(mesh,&idx))) {
          double rxyz[MAXPV3][3], cen[3];
          int nrv;
          
          MR_Coords(mr,&nrv,rxyz);
          cen[0] = cen[1] = cen[2] = 0.0;
          for (j = 0; j < nrv; j++)
            for (k = 0; k < NDIM_4_ZOLTAN; k++)
              cen[k] += rxyz[j][k];
          for (k = 0; k < NDIM_4_ZOLTAN; k++) cen[k] /= nrv;
          for (k = 0; k < NDIM_4_ZOLTAN; k++) 
            if (fabs(cen[k]) < 1.0e-10) cen[k] = 0.0; 

          id = MR_ID(mr);
          graph.nodeGID[id-1] = id;
          memcpy(&(graph.nodeCoords[NDIM_4_ZOLTAN*(id-1)]),cen,NDIM_4_ZOLTAN*sizeof(double));
        }

      }
    }

    MPI_Bcast(&NDIM_4_ZOLTAN,1,MPI_INT,0,comm);

    /* Set some default values */
    Zoltan_Set_Param(zz, "RCB_RECTILINEAR_BLOCKS","1");
    //    Zoltan_Set_Param(zz, "AVERAGE_CUTS", "1");

    if (noptions > 1) {
      for (i = 1; i < noptions; i++) {
        char *paramstr = NULL, *valuestr = NULL, instring[256];
        strcpy(instring,options[i]);
        paramstr = strtok(instring,"=");
        valuestr = strtok(NULL," ");
        Zoltan_Set_Param(zz,paramstr,valuestr);
      }
    }

    /* Query functions - defined in simpleQueries.h */

    Zoltan_Set_Num_Obj_Fn(zz, get_number_of_nodes, &graph);
    Zoltan_Set_Obj_List_Fn(zz, get_node_list, &graph);
    Zoltan_Set_Num_Geom_Fn(zz, get_num_dimensions_reduced, &graph);    /* reduced dimensions */
    Zoltan_Set_Geom_Multi_Fn(zz, get_element_centers_reduced, &graph); /* reduced dimension centers */

  }
  else if (strcmp(partition_method_str,"GRAPH") == 0) {

    if(rank == 0) {
      nv = MESH_Num_Vertices(mesh);
      ne = MESH_Num_Edges(mesh);
      nf = MESH_Num_Faces(mesh);
      nr = MESH_Num_Regions(mesh);
      
      ipos = 0;
      
      /* build nodes and neighbors list, similar as in partition with metis
         Assign processor 0 the whole mesh, assign other processors a NULL mesh */
  
      if (nr == 0) {
        if (nf == 0) {
          MSTK_Report("MESH_PartitionWithZoltan",
                      "Cannot partition wire meshes with Zoltan",MSTK_FATAL);
          exit(-1);
      
        }

        graph.nodeGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * nf);
        graph.nborIndex = (int *)malloc(sizeof(int) * (nf + 1));
        graph.nborGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * 2*ne);
        graph.nborProc = (int *)malloc(sizeof(int) * 2*ne);
      
        graph.nborIndex[0] = 0;
      
        /* Surface mesh */
        idx = 0; i = 0;
        while ((mf = MESH_Next_Face(mesh,&idx))) {
          graph.nodeGID[i] = MF_ID(mf);
          fedges = MF_Edges(mf,1,0);
          nfe = List_Num_Entries(fedges);
	
          idx2 = 0;
          while ((fedge = List_Next_Entry(fedges,&idx2))) {
	  
            efaces = ME_Faces(fedge);
            nef = List_Num_Entries(efaces);
	  
            if (nef == 1) {
              continue;          /* boundary edge; nothing to do */
            } else {
              int j;
              for (j = 0; j < nef; j++) {
                oppf = List_Entry(efaces,j);
                if (oppf == mf) {
                  graph.nborGID[ipos] = MF_ID(oppf);
                  /* initially set all nodes on processor 0 */
                  graph.nborProc[ipos] = 0;
                  ipos++;
                }
              }
            }
	  
            List_Delete(efaces);
	  
          }
	
          List_Delete(fedges);
          i++;
          graph.nborIndex[i] = ipos;
        }
        graph.numMyNodes = i;
        graph.numAllNbors = ipos;
      }
      else {
        graph.nodeGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * nr);
        graph.nborIndex = (int *)malloc(sizeof(int) * (nr + 1));
        graph.nborGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * 2*nf);
        graph.nborProc = (int *)malloc(sizeof(int) * 2*nf);
      
        graph.nborIndex[0] = 0;
      
        /* Volume mesh */
      
        idx = 0; i = 0;
        while ((mr = MESH_Next_Region(mesh,&idx))) {
          graph.nodeGID[i] = MR_ID(mr);
          rfaces = MR_Faces(mr);
          nrf = List_Num_Entries(rfaces);
      
          idx2 = 0;
          while ((rface = List_Next_Entry(rfaces,&idx2))) {
	  
            fregions = MF_Regions(rface);
            nfr = List_Num_Entries(fregions);
	  
            if (nfr > 1) {
              oppr = List_Entry(fregions,0);
              if (oppr == mr)
                oppr = List_Entry(fregions,1);
	    
              graph.nborGID[ipos] = MR_ID(oppr);
              /* initially set all nodes on processor 0 */
              graph.nborProc[ipos] = 0;
              ipos++;
            }
	  
            List_Delete(fregions);
	  
          }
	
          List_Delete(rfaces);
	
          i++;
          graph.nborIndex[i] = ipos;
        }
        graph.numMyNodes = i;
        graph.numAllNbors = ipos;
      }
    }

    /* Graph parameters */

    /* Zoltan_Set_Param(zz, "CHECK_GRAPH", "2"); */
    Zoltan_Set_Param(zz, "PHG_EDGE_SIZE_THRESHOLD", ".35");  /* 0-remove all, 1-remove none */

    /* Query functions - defined in simpleQueries.h */

    Zoltan_Set_Num_Obj_Fn(zz, get_number_of_nodes, &graph);
    Zoltan_Set_Obj_List_Fn(zz, get_node_list, &graph);
    Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, &graph);
    Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, &graph);    
  }

  /* Partition the graph */
  /******************************************************************                                                                             
   ** Zoltan can now partition the graph.                                                                                                   
   ** We assume the number of partitions is                                                                                
   ** equal to the number of processes.  Process rank 0 will own                                                                                   
   ** partition 0, process rank 1 will own partition 1, and so on.                                                                                 
   ******************************************************************/
  rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
			   &changes,        /* 1 if partitioning was changed, 0 otherwise */
			   &numGidEntries,  /* Number of integers used for a global ID */
			   &numLidEntries,  /* Number of integers used for a local ID */
			   &numImport,      /* Number of nodes to be sent to me */
			   &importGlobalGids,  /* Global IDs of nodes to be sent to me */
			   &importLocalGids,   /* Local IDs of nodes to be sent to me */
			   &importProcs,    /* Process rank for source of each incoming node */
			   &importToPart,   /* New partition for each incoming node */
			   &numExport,      /* Number of nodes I must send to other processes*/
			   &exportGlobalGids,  /* Global IDs of the nodes I must send */
			   &exportLocalGids,   /* Local IDs of the nodes I must send */
			   &exportProcs,    /* Process to which I send each of the nodes */
			   &exportToPart);  /* Partition to which each node will belong */

  if (rc != ZOLTAN_OK){
    if (rank == 0)
      MSTK_Report("MESH_PartitionWithZoltan","Could not partition mesh with ZOLTAN",
                  MSTK_ERROR);
    Zoltan_Destroy(&zz);
    MPI_Finalize();
    return 0;
  }

  if(rank == 0) {
    *part = (int *) calloc(graph.numMyNodes,sizeof(int));
    for ( i = 0; i < numExport; i++ ) {
      (*part)[exportGlobalGids[i]-1] = exportToPart[i];
    }
    if (graph.nodeGID) free(graph.nodeGID);
    if (graph.nodeCoords) free(graph.nodeCoords);
    if (graph.nborIndex) free(graph.nborIndex);
    if (graph.nborGID) free(graph.nborGID);
    if (graph.nborProc) free(graph.nborProc);
  }
  else { 
    *part = NULL;
  }


  Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, &exportProcs, &exportToPart);
  Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, &importProcs, &importToPart);
  Zoltan_Destroy(&zz);                

  return 1;
}
Ejemplo n.º 10
0
PeridigmNS::ZoltanSearchTree::ZoltanSearchTree(int numPoints, double* coordinates) : SearchTree(numPoints, coordinates),
	callbackdata(numPoints,coordinates),zoltan(0),
	partToCoordIdx(new int[numPoints]),
	searchParts(new int[numPoints])
{



	zoltan = Zoltan_Create(MPI_COMM_SELF);
    Zoltan_Set_Param(zoltan, "debug_level", "0");
    Zoltan_Set_Param(zoltan, "rcb_output_level", "0");
	/*
	 * query function returns the number of objects that are currently assigned to the processor
	 */
	Zoltan_Set_Num_Obj_Fn(zoltan, &get_num_points, &callbackdata);

	/*
	 * query function fills two (three if weights are used) arrays with information about the objects
	 * currently assigned to the processor. Both arrays are allocated (and subsequently freed) by Zoltan;
	 * their size is determined by a call to a ZOLTAN_NUM_OBJ_FN query function to get the array size.
	 */
	Zoltan_Set_Obj_List_Fn(zoltan, &get_point_ids, &callbackdata);

	/*
	 * query function returns the number of values needed to express the geometry of an object.
	 * For example, for a two-dimensional mesh-based application, (x,y) coordinates are needed
	 * to describe an object's geometry; thus the ZOLTAN_NUM_GEOM_FN query function should return
	 * the value of two. For a similar three-dimensional application, the return value should be three.
	 */
	Zoltan_Set_Num_Geom_Fn(zoltan, &get_dimension, NULL);

	/*
	 * query function returns a vector of geometry values for a list of given objects. The geometry
	 * vector is allocated by Zoltan to be of size num_obj * num_dim.
	 */

	Zoltan_Set_Geom_Multi_Fn(zoltan, &get_point_coordinates, &callbackdata);

	/*
	 * setting method
	 */
	Zoltan_Set_Param(zoltan,"lb_method","rcb");

	/*
	 * keep cuts
	 */
	Zoltan_Set_Param(zoltan,"keep_cuts","1");

	Zoltan_Set_Param(zoltan,"return_lists","part");
	Zoltan_Set_Param(zoltan, "num_lid_entries", "0");
	char s[11];
	sprintf(s,"%d",numPoints);
	Zoltan_Set_Param(zoltan, "num_global_parts", s);

	int numimp = -1, numexp = -1;
	int numgid = 1, numlid = 0;
	ZOLTAN_ID_PTR impgid=NULL, implid=NULL;
	ZOLTAN_ID_PTR gid=NULL, lid=NULL;
	int *imppart = NULL, *impproc = NULL;
	int *procs = NULL, *parts=NULL;


	int changes;

	int ierr = Zoltan_LB_Partition(zoltan, &changes, &numgid, &numlid,
			&numimp, &impgid, &implid, &imppart, &impproc,
			&numexp, &gid, &lid, &procs, &parts);

    TEUCHOS_TEST_FOR_EXCEPT_MSG(ierr != 0, "Error in ZoltanSearchTree::ZoltanSearchTree(), call to Zoltan_LB_Partition() returned a nonzero error code.");

	for (int I = 0; I < numPoints; I++)
	    partToCoordIdx[parts[I]] = I;

	Zoltan_LB_Free_Part(&impgid, &implid, &impproc, &imppart);
	Zoltan_LB_Free_Part(&gid, &lid, &procs, &parts);
}
Ejemplo n.º 11
0
int main(int argc, char *argv[])
{
  int rc, do_hier, status;
  float ver;
  struct Zoltan_Struct *zz;
  int changes, numGidEntries, numLidEntries, numImport, numExport;
  int generate_files = 0;
  char *platform=NULL, *topology=NULL;
  char *graph_package=NULL;
  ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
  int *importProcs, *importToPart, *exportProcs, *exportToPart;
  struct option opts[10];
  double comm_time[10];
  float cut_weight[3] = {0., 0., 0.};
  long nvert=0;
  char *debug_level=NULL;

  status = 0;

  MPI_Init(&argc, &argv);
  MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
  MPI_Comm_size(MPI_COMM_WORLD, &numProcs);

  Zoltan_Initialize(argc, argv, &ver);
  zz = Zoltan_Create(MPI_COMM_WORLD);

  /******************************************************************
  ** Check that this test makes sense.
  ******************************************************************/

  if (sizeof(long) < sizeof(ZOLTAN_ID_TYPE)){
    if (myRank == 0){
      printf("ERROR: This code assumes that a long is at least %d bytes\n",(int)sizeof(ZOLTAN_ID_TYPE));
    }
    status = 1;
  }

  check_error_status(status, "configuration error");

  /******************************************************************
  ** Initialize zoltan
  ******************************************************************/

  /* options */

  opts[0].name = "platform";
  opts[0].has_arg = 1;
  opts[0].flag = NULL;
  opts[0].val = 1;

  opts[1].name = "topology";
  opts[1].has_arg = 1;
  opts[1].flag = NULL;
  opts[1].val = 2;

  opts[2].name = "size";
  opts[2].has_arg = 1;
  opts[2].flag = NULL;
  opts[2].val = 4;

  opts[3].name = "verbose";
  opts[3].has_arg = 0;
  opts[3].flag = NULL;
  opts[3].val = 5;

  opts[4].name = "help";
  opts[4].has_arg = 0;
  opts[4].flag = NULL;
  opts[4].val = 6;

  opts[5].name = "graph_package";
  opts[5].has_arg = 1;
  opts[5].flag = NULL;
  opts[5].val = 7;

  opts[6].name = "generate_files";
  opts[6].has_arg = 0;
  opts[6].flag = NULL;
  opts[6].val = 8;

  opts[7].name = "debug_level";
  opts[7].has_arg = 1;
  opts[7].flag = NULL;
  opts[7].val = 9;

  opts[8].name = 0;
  opts[8].has_arg = 0;
  opts[8].flag = NULL;
  opts[8].val = 0;

  status = 0;

  while (1){
    rc = getopt_long_only(argc, argv, "",  opts, NULL);

    if (rc == '?'){
      MPI_Barrier(MPI_COMM_WORLD);
      if (myRank == 0) usage();
      MPI_Finalize();
      exit(0);
    }
    else if (rc == 1){
      platform = optarg;
      if (myRank == 0)
        printf( "For platform %s\n",optarg );
    }
    else if (rc == 2){
      topology = optarg;
      if (myRank == 0)
        printf( "For topology %s\n",optarg);
    }
    else if (rc == 7){
      graph_package = optarg;
      if (myRank == 0)
        printf( "Zoltan parameter GRAPH_PACKAGE = %s\n",graph_package);
    }
    else if (rc == 8){
      generate_files = 1;
      if (myRank == 0)
        printf( "Zoltan_Generate_Files will be called for each level.\n");
    }
    else if (rc == 4){
      nvert = atol(optarg);
      if (nvert < 1) status = 1;
      check_error_status(status, "--size={approximate number of vertices}");
      if (myRank == 0){
        printf( "Graph will have approximately %ld vertices.\n",nvert);
      }
    }
    else if (rc == 5){
      verbose = 1;
    }
    else if (rc == 6){
      if (myRank == 0) usage();
      MPI_Finalize();
      exit(0);
    }
    else if (rc == 9){
      debug_level = optarg;
    }
    else if (rc <= 0){
      break;
    }
  }

  if ((platform==NULL) && (topology==NULL)){
    if (myRank == 0)
      fprintf(stdout,"No platform or topology, so we'll skip hierarchical partitioning\n");
    do_hier = 0;
  }
  else if (graph_package == NULL){
    if (myRank == 0)
      fprintf(stdout,"No graph package, so we'll skip hierarchical partitioning\n");
    do_hier = 0;
  }
  else{
    do_hier = 1;
  }

  /* start */

  Zoltan_Memory_Debug(0);

  if (nvert > 0)
    numGlobalVertices = nvert;
  else
    numGlobalVertices = NUM_GLOBAL_VERTICES;

  status = create_a_graph();
  check_error_status(status, "creating the graph");

  Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
  Zoltan_Set_Param(zz, "REMAP", "0");
  Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
  Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
  Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL"); /* export AND import lists */
  Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "1"); /* number of weights per vertex */
  Zoltan_Set_Param(zz, "EDGE_WEIGHT_DIM", "1");/* number of weights per hyperedge */

  Zoltan_Set_Num_Obj_Fn(zz, get_number_of_vertices, NULL);
  Zoltan_Set_Obj_List_Fn(zz, get_vertex_list, NULL);
  Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list,  NULL);
  Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list,  NULL);

  /* GRAPH PARTITION */

  Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH");
  Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION");

  if (graph_package)
    Zoltan_Set_Param(zz, "GRAPH_PACKAGE", graph_package);

  if (verbose){
    debug(zz, "Initial graph", 0);
  }

  if (generate_files){
    rc = Zoltan_Generate_Files(zz, "flat", myRank, 0, 1, 0);
    if (rc != ZOLTAN_OK) status = 1;
    check_error_status(status, "Zoltan_Generate_Files");
  }

  /* Performance before partitioning */
  time_communication(comm_time+0);
  cut_weight[0] = get_edge_cut_weight(zz);

  if (cut_weight[0] < 0.0) status = 1;
  check_error_status(status, "First call to get_edge_cut_weight");

  rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
        &changes,        /* 1 if partitioning was changed, 0 otherwise */
        &numGidEntries,  /* Number of integers used for a global ID */
        &numLidEntries,  /* Number of integers used for a local ID */
        &numImport,      /* Number of vertices to be sent to me */
        &importGlobalGids,  /* Global IDs of vertices to be sent to me */
        &importLocalGids,   /* Local IDs of vertices to be sent to me */
        &importProcs,    /* Process rank for source of each incoming vertex */
        &importToPart,   /* New partition for each incoming vertex */
        &numExport,      /* Number of vertices I must send to other processes*/
        &exportGlobalGids,  /* Global IDs of the vertices I must send */
        &exportLocalGids,   /* Local IDs of the vertices I must send */
        &exportProcs,    /* Process to which I send each of the vertices */
        &exportToPart);  /* Partition to which each vertex will belong */

  if (rc != ZOLTAN_OK) status = 1;
  check_error_status(status, "First call to LB_Partition");

  status = migrate_graph(numExport, numImport, exportLocalGids, importGlobalGids);
  check_error_status(status, "migration");

  if (verbose){
    debug(zz, "After flat partitioning and migration", 0);
  }

  time_communication(comm_time+1);      /* With graph partitioning */
  cut_weight[1] = get_edge_cut_weight(zz);

  if (cut_weight[1] < 0.0) status = 1;
  check_error_status(status, "Second call to get_edge_cut_weight");

  Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids,
                      &importProcs, &importToPart);
  Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids,
                      &exportProcs, &exportToPart);

  if (do_hier){

    /* HIERARCHICAL PARTITION */

    free_graph();
    status = create_a_graph();
    check_error_status(status, "create graph for hierarchical partitioning");

    Zoltan_Set_Param(zz, "LB_METHOD", "HIER");
    Zoltan_Set_Param(zz, "HIER_ASSIST", "1");
    if (generate_files){
      Zoltan_Set_Param(zz, "HIER_GENERATE_FILES", "1");
    }

    if (debug_level)   /* 1, 2 or 3 */
      Zoltan_Set_Param(zz, "HIER_DEBUG_LEVEL", debug_level);
    else
      Zoltan_Set_Param(zz, "HIER_DEBUG_LEVEL", "0");

    /* TODO: Suppose graph is not symmetric, and we request SYMMETRIZE.  Do we still get
     *  a "good" answer when each sub-graph in the hierarchy is symmetrized?
     */

    if (topology)
      Zoltan_Set_Param(zz, "TOPOLOGY", topology);
    else if (platform)
      Zoltan_Set_Param(zz, "PLATFORM", platform);

    rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
          &changes,        /* 1 if partitioning was changed, 0 otherwise */
          &numGidEntries,  /* Number of integers used for a global ID */
          &numLidEntries,  /* Number of integers used for a local ID */
          &numImport,      /* Number of vertices to be sent to me */
          &importGlobalGids,  /* Global IDs of vertices to be sent to me */
          &importLocalGids,   /* Local IDs of vertices to be sent to me */
          &importProcs,    /* Process rank for source of each incoming vertex */
          &importToPart,   /* New partition for each incoming vertex */
          &numExport,      /* Number of vertices I must send to other processes*/
          &exportGlobalGids,  /* Global IDs of the vertices I must send */
          &exportLocalGids,   /* Local IDs of the vertices I must send */
          &exportProcs,    /* Process to which I send each of the vertices */
          &exportToPart);  /* Partition to which each vertex will belong */

    if (rc != ZOLTAN_OK) status = 1;
    check_error_status(status, "Second call to LB_Partition");

    status = migrate_graph(numExport, numImport, exportLocalGids, importGlobalGids);
    check_error_status(status, "second migration");

    if (verbose){
      debug(zz, "After hierarchical partitioning and migration", 0);
    }

    time_communication(comm_time+2);      /* With hierarchical graph partitioning */
    cut_weight[2] = get_edge_cut_weight(zz);

    if (cut_weight[2] < 0.0) status = 1;
    check_error_status(status, "Third call to get_edge_cut_weight");

    Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids,
                        &importProcs, &importToPart);
    Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids,
                        &exportProcs, &exportToPart);
  }

  Zoltan_Destroy(&zz);

  free_graph();

  if (myRank == 0){
    fprintf(stdout,"Graph cut weight before partitioning: %f\n",cut_weight[0]);
    fprintf(stdout,"             after flat partitioning: %f\n",cut_weight[1]);
    if (do_hier)
      fprintf(stdout,"     after hierarchical partitioning: %f\n",cut_weight[2]);
    fflush(stdout);
  }

  if (cut_weight[1] >= cut_weight[0]){
    status = 1;
    if (zz->Proc == 0){
      fprintf(stderr,"FAILED: No improvement shown in flat partitioning");
    }
  }

  if (do_hier && (cut_weight[2] > cut_weight[0])){
    status = 1;
    if (zz->Proc == 0){
      fprintf(stderr,"FAILED: No improvement shown in hierarchical partitioning");
    }
  }


  MPI_Finalize();

  return status;
}