Ejemplo n.º 1
0
static
int
_SCOTCH_METIS_PartGraph (
const int * const           n,
const int * const           xadj,
const int * const           adjncy,
const int * const           vwgt,
const int * const           adjwgt,
const int * const           numflag,
const int * const           nparts,
int * const                 part)
{
  SCOTCH_Graph        grafdat;                    /* Scotch graph object to interface with libScotch */
  SCOTCH_Strat        stradat;
  SCOTCH_Num          baseval;
  SCOTCH_Num          vertnbr;
  int                 o;

  if (sizeof (SCOTCH_Num) != sizeof (int)) {
    errorPrint ("METIS_PartGraph* (as of SCOTCH): SCOTCH_Num type should equate to int");
    return (1);
  }

  SCOTCH_graphInit (&grafdat);

  baseval = *numflag;
  vertnbr = *n;

  o = 1;                                          /* Assume something will go wrong */
  if (SCOTCH_graphBuild (&grafdat,
                         baseval, vertnbr, xadj, xadj + 1, vwgt, NULL,
                         xadj[vertnbr] - baseval, adjncy, adjwgt) == 0) {
    SCOTCH_stratInit (&stradat);
#ifdef SCOTCH_DEBUG_ALL
    if (SCOTCH_graphCheck (&grafdat) == 0)        /* TRICK: next instruction called only if graph is consistent */
#endif /* SCOTCH_DEBUG_ALL */
    o = SCOTCH_graphPart (&grafdat, *nparts, &stradat, part);
    SCOTCH_stratExit (&stradat);
  }
  SCOTCH_graphExit (&grafdat);

  if (baseval != 0) {                             /* MeTiS part array is based, Scotch is not */
    SCOTCH_Num          vertnum;

    for (vertnum = 0; vertnum < vertnbr; vertnum ++)
      part[vertnum] += baseval;
  }

  return (o);
}
Ejemplo n.º 2
0
    void setGraph(const ordinal_type m,
                  const size_type_array rptr,
                  const ordinal_type_array cidx) {
        _is_ordered = false;
        _cblk  = 0;

        /// Scotch graph spec
        /// - no diagonals, symmetric

        _base  = 0;
        _m     = m;
        _nnz   = rptr[m];

        _rptr  = rptr;
        _cidx  = cidx;

        _perm  = ordinal_type_array("Scotch::PermutationArray", _m);
        _peri  = ordinal_type_array("Scotch::InvPermutationArray", _m);
        _range = ordinal_type_array("Scotch::RangeArray", _m);
        _tree  = ordinal_type_array("Scotch::TreeArray", _m);

        _strat = 0;
        _level = 0;

        int ierr = 0;
        ordinal_type *rptr_ptr = reinterpret_cast<ordinal_type*>(_rptr.ptr_on_device());
        ordinal_type *cidx_ptr = reinterpret_cast<ordinal_type*>(_cidx.ptr_on_device());

        ierr = SCOTCH_graphInit(&_graph);
        TACHO_TEST_FOR_ABORT(ierr, "Failed in SCOTCH_graphInit");
        ierr = SCOTCH_graphBuild(&_graph,             // scotch graph
                                 _base,               // base value
                                 _m,                  // # of vertices
                                 rptr_ptr,            // column index array pointer begin
                                 rptr_ptr+1,          // column index array pointer end
                                 NULL,                // weights on vertices (optional)
                                 NULL,                // label array on vertices (optional)
                                 _nnz,                // # of nonzeros
                                 cidx_ptr,            // column index array
                                 NULL);               // edge load array (optional)
        TACHO_TEST_FOR_ABORT(ierr, "Failed in SCOTCH_graphBuild");
        ierr = SCOTCH_graphCheck(&_graph);
        TACHO_TEST_FOR_ABORT(ierr, "Failed in SCOTCH_graphCheck");
    }
Ejemplo n.º 3
0
int
METISNAMEU(METIS_NodeWND) (
const SCOTCH_Num * const    n,
const SCOTCH_Num * const    xadj,
const SCOTCH_Num * const    adjncy,
const SCOTCH_Num * const    vwgt,
const SCOTCH_Num * const    numflag,
const SCOTCH_Num * const    options,
SCOTCH_Num * const          perm,
SCOTCH_Num * const          iperm)
{
  SCOTCH_Graph        grafdat;                    /* Scotch graph object to interface with libScotch    */
  SCOTCH_Ordering     ordedat;                    /* Scotch ordering object to interface with libScotch */
  SCOTCH_Strat        stradat;
  int                 o;

  o = METIS_ERROR;                                /* Assume an error */

  SCOTCH_graphInit (&grafdat);

  if (SCOTCH_graphBuild (&grafdat,
                         *numflag, *n, xadj, xadj + 1, vwgt, NULL,
                         xadj[*n] - *numflag, adjncy, NULL) == 0) {
    SCOTCH_stratInit (&stradat);
#ifdef SCOTCH_DEBUG_ALL
    if (SCOTCH_graphCheck (&grafdat) == 0)        /* TRICK: next instruction called only if graph is consistent */
#endif /* SCOTCH_DEBUG_ALL */
    {
      if (SCOTCH_graphOrderInit (&grafdat, &ordedat, iperm, perm, /* MeTiS and Scotch have opposite definitions for (inverse) permutations */
                                 NULL, NULL, NULL) == 0) {
        if (SCOTCH_graphOrderCompute (&grafdat, &ordedat, &stradat) == 0)
          o = METIS_OK;
        SCOTCH_graphOrderExit    (&grafdat, &ordedat);
      }
    }
    SCOTCH_stratExit (&stradat);
  }
  SCOTCH_graphExit (&grafdat);

  return (o);
}
Ejemplo n.º 4
0
// Call scotch with options from dictionary.
Foam::label Foam::scotchDecomp::decompose
(
    const List<int>& adjncy,
    const List<int>& xadj,
    const scalarField& cWeights,

    List<int>& finalDecomp
)
{
    // Dump graph
    if (decompositionDict_.found("scotchCoeffs"))
    {
        const dictionary& scotchCoeffs =
            decompositionDict_.subDict("scotchCoeffs");

        if (scotchCoeffs.found("writeGraph"))
        {
            Switch writeGraph(scotchCoeffs.lookup("writeGraph"));

            if (writeGraph)
            {
                OFstream str(mesh_.time().path() / mesh_.name() + ".grf");

                Info<< "Dumping Scotch graph file to " << str.name() << endl
                    << "Use this in combination with gpart." << endl;

                label version = 0;
                str << version << nl;
                // Numer of vertices
                str << xadj.size()-1 << ' ' << adjncy.size() << nl;
                // Numbering starts from 0
                label baseval = 0;
                // Has weights?
                label hasEdgeWeights = 0;
                label hasVertexWeights = 0;
                label numericflag = 10*hasEdgeWeights+hasVertexWeights;
                str << baseval << ' ' << numericflag << nl;
                for (label cellI = 0; cellI < xadj.size()-1; cellI++)
                {
                    label start = xadj[cellI];
                    label end = xadj[cellI+1];
                    str << end-start;

                    for (label i = start; i < end; i++)
                    {
                        str << ' ' << adjncy[i];
                    }
                    str << nl;
                }
            }
        }
    }


    // Strategy
    // ~~~~~~~~

    // Default.
    SCOTCH_Strat stradat;
    check(SCOTCH_stratInit(&stradat), "SCOTCH_stratInit");

    if (decompositionDict_.found("scotchCoeffs"))
    {
        const dictionary& scotchCoeffs =
            decompositionDict_.subDict("scotchCoeffs");


        string strategy;
        if (scotchCoeffs.readIfPresent("strategy", strategy))
        {
            if (debug)
            {
                Info<< "scotchDecomp : Using strategy " << strategy << endl;
            }
            SCOTCH_stratGraphMap(&stradat, strategy.c_str());
            //fprintf(stdout, "S\tStrat=");
            //SCOTCH_stratSave(&stradat, stdout);
            //fprintf(stdout, "\n");
        }
    }


    // Graph
    // ~~~~~

    List<int> velotab;


    // Check for externally provided cellweights and if so initialise weights
    scalar minWeights = gMin(cWeights);
    if (cWeights.size() > 0)
    {
        if (minWeights <= 0)
        {
            WarningIn
            (
                "scotchDecomp::decompose"
                "(const pointField&, const scalarField&)"
            )   << "Illegal minimum weight " << minWeights
                << endl;
        }

        if (cWeights.size() != xadj.size()-1)
        {
            FatalErrorIn
            (
                "scotchDecomp::decompose"
                "(const pointField&, const scalarField&)"
            )   << "Number of cell weights " << cWeights.size()
                << " does not equal number of cells " << xadj.size()-1
                << exit(FatalError);
        }

        // Convert to integers.
        velotab.setSize(cWeights.size());
        forAll(velotab, i)
        {
            velotab[i] = int(cWeights[i]/minWeights);
        }
    }



    SCOTCH_Graph grafdat;
    check(SCOTCH_graphInit(&grafdat), "SCOTCH_graphInit");
    check
    (
        SCOTCH_graphBuild
        (
            &grafdat,
            0,                      // baseval, c-style numbering
            xadj.size()-1,          // vertnbr, nCells
            xadj.begin(),           // verttab, start index per cell into adjncy
            &xadj[1],               // vendtab, end index  ,,
            velotab.begin(),        // velotab, vertex weights
            NULL,                   // vlbltab
            adjncy.size(),          // edgenbr, number of arcs
            adjncy.begin(),         // edgetab
            NULL                    // edlotab, edge weights
        ),
        "SCOTCH_graphBuild"
    );
    check(SCOTCH_graphCheck(&grafdat), "SCOTCH_graphCheck");


    // Architecture
    // ~~~~~~~~~~~~
    // (fully connected network topology since using switch)

    SCOTCH_Arch archdat;
    check(SCOTCH_archInit(&archdat), "SCOTCH_archInit");

    List<label> processorWeights;
    if (decompositionDict_.found("scotchCoeffs"))
    {
        const dictionary& scotchCoeffs =
            decompositionDict_.subDict("scotchCoeffs");

        scotchCoeffs.readIfPresent("processorWeights", processorWeights);
    }
    if (processorWeights.size())
    {
        if (debug)
        {
            Info<< "scotchDecomp : Using procesor weights " << processorWeights
                << endl;
        }
        check
        (
            SCOTCH_archCmpltw(&archdat, nProcessors_, processorWeights.begin()),
            "SCOTCH_archCmpltw"
        );
    }
    else
    {
        check
        (
            SCOTCH_archCmplt(&archdat, nProcessors_),
            "SCOTCH_archCmplt"
        );
    }


    //SCOTCH_Mapping mapdat;
    //SCOTCH_graphMapInit(&grafdat, &mapdat, &archdat, NULL);
    //SCOTCH_graphMapCompute(&grafdat, &mapdat, &stradat); /* Perform mapping */
    //SCOTCH_graphMapExit(&grafdat, &mapdat);


    // Hack:switch off fpu error trapping
#   ifdef LINUX_GNUC
    int oldExcepts = fedisableexcept
    (
        FE_DIVBYZERO
      | FE_INVALID
      | FE_OVERFLOW
    );
#   endif

    finalDecomp.setSize(xadj.size()-1);
    finalDecomp = 0;
    check
    (
        SCOTCH_graphMap
        (
            &grafdat,
            &archdat,
            &stradat,           // const SCOTCH_Strat *
            finalDecomp.begin() // parttab
        ),
        "SCOTCH_graphMap"
    );

#   ifdef LINUX_GNUC
    feenableexcept(oldExcepts);
#   endif



    //finalDecomp.setSize(xadj.size()-1);
    //check
    //(
    //    SCOTCH_graphPart
    //    (
    //        &grafdat,
    //        nProcessors_,       // partnbr
    //        &stradat,           // const SCOTCH_Strat *
    //        finalDecomp.begin() // parttab
    //    ),
    //    "SCOTCH_graphPart"
    //);

    // Release storage for graph
    SCOTCH_graphExit(&grafdat);
    // Release storage for strategy
    SCOTCH_stratExit(&stradat);
    // Release storage for network topology
    SCOTCH_archExit(&archdat);

    return 0;
}
Ejemplo n.º 5
0
void ScotchRefineLB::work(LDStats *stats) {
  /** ========================== INITIALIZATION ============================= */
  ProcArray *parr = new ProcArray(stats);
  ObjGraph *ogr = new ObjGraph(stats);
  int cost_array[10] = {64, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536};

  /** ============================= STRATEGY ================================ */
  // convert ObjGraph to the Scotch graph
  SCOTCH_Num baseval = 0;			// starting index of vertices
  SCOTCH_Num vertnbr = ogr->vertices.size();	// number of vertices
  SCOTCH_Num edgenbr = 0;			// number of edges

  SCOTCH_Num *oldpemap = (SCOTCH_Num *)malloc(sizeof(SCOTCH_Num) * vertnbr);

  double maxLoad = 0.0;
  double minLoad = 0.0;
  if (vertnbr > 0) {
    minLoad = ogr->vertices[baseval].getVertexLoad();
  }

  long maxBytes = 1;
  int i, j, k, vert;
  

  /** remove duplicate edges from recvFrom */
  for(i = baseval; i < vertnbr; i++) {
    for(j = 0; j < ogr->vertices[i].sendToList.size(); j++) {
      vert = ogr->vertices[i].sendToList[j].getNeighborId();
      for(k = 0; k < ogr->vertices[i].recvFromList.size(); k++) {
        if(ogr->vertices[i].recvFromList[k].getNeighborId() == vert) {
          ogr->vertices[i].sendToList[j].setNumBytes(ogr->vertices[i].sendToList[j].getNumBytes() + ogr->vertices[i].recvFromList[k].getNumBytes());
          ogr->vertices[i].recvFromList.erase(ogr->vertices[i].recvFromList.begin() + k);
        }
      }
    }
  }

  /** the object load is normalized to an integer between 0 and 256 */
  for(i = baseval; i < vertnbr; i++) {
    if(ogr->vertices[i].getVertexLoad() > maxLoad)
      maxLoad = ogr->vertices[i].getVertexLoad();

    if (ogr->vertices[i].getVertexLoad() < minLoad) {
      minLoad = ogr->vertices[i].getVertexLoad();
    }
    edgenbr += ogr->vertices[i].sendToList.size() + ogr->vertices[i].recvFromList.size();
    oldpemap[i] = ogr->vertices[i].getCurrentPe();
  }

  for(i = baseval; i < vertnbr; i++) {
    for(j = 0; j < ogr->vertices[i].sendToList.size(); j++) {
      if (ogr->vertices[i].sendToList[j].getNumBytes() > maxBytes) {
        maxBytes = ogr->vertices[i].sendToList[j].getNumBytes();
      }
    }
    for(j = 0; j < ogr->vertices[i].recvFromList.size(); j++) {
      if (ogr->vertices[i].recvFromList[j].getNumBytes() > maxBytes) {
        maxBytes = ogr->vertices[i].recvFromList[j].getNumBytes();
      }
    }
  }

  /* adjacency list */
  SCOTCH_Num *verttab = (SCOTCH_Num *)malloc(sizeof(SCOTCH_Num) * (vertnbr+1));
  /* loads of vertices */
  SCOTCH_Num *velotab = (SCOTCH_Num *)malloc(sizeof(SCOTCH_Num) * vertnbr);
  /* id of the neighbors */
  SCOTCH_Num *edgetab = (SCOTCH_Num *)malloc(sizeof(SCOTCH_Num) * edgenbr);
  /* number of bytes exchanged */
  SCOTCH_Num *edlotab = (SCOTCH_Num *)malloc(sizeof(SCOTCH_Num) * edgenbr);

  int edgeNum = 0;
  double ratio = 256.0/maxLoad;
  double byteRatio = 1024.0/maxBytes;
  
  for(i = baseval; i < vertnbr; i++) {
    verttab[i] = edgeNum;
    velotab[i] = (int)ceil(ogr->vertices[i].getVertexLoad() * ratio);
    for(j = 0; j < ogr->vertices[i].sendToList.size(); j++) {
      edgetab[edgeNum] = ogr->vertices[i].sendToList[j].getNeighborId();
      edlotab[edgeNum] = (int) ceil(ogr->vertices[i].sendToList[j].getNumBytes()
          * byteRatio);
      edgeNum++;
    }
    for(j = 0; j < ogr->vertices[i].recvFromList.size(); j++) {
      edgetab[edgeNum] = ogr->vertices[i].recvFromList[j].getNeighborId();
      edlotab[edgeNum] = (int)
          ceil(ogr->vertices[i].recvFromList[j].getNumBytes() * byteRatio);
      edgeNum++;
    }
  }
  verttab[i] = edgeNum;
  CkAssert(edgeNum == edgenbr);

  SCOTCH_Graph graph;		// Graph to partition
  SCOTCH_Strat strat;		// Strategy to achieve partitioning

  /* Initialize data structures */
  SCOTCH_graphInit (&graph);
  SCOTCH_stratInit (&strat);

  SCOTCH_graphBuild (&graph, baseval, vertnbr, verttab, NULL, velotab, NULL, edgenbr, edgetab, edlotab); 
  SCOTCH_graphCheck (&graph);

  double migration_cost = 1024.0;

    if (step() == 0) {
      SCOTCH_stratGraphMapBuild (&strat, SCOTCH_STRATBALANCE, parr->procs.size (), 0.01);
    } else {
      SCOTCH_stratGraphMapBuild (&strat, SCOTCH_STRATBALANCE | SCOTCH_STRATREMAP, parr->procs.size (), 0.01);
    }

  SCOTCH_Num *pemap = (SCOTCH_Num *)malloc(sizeof(SCOTCH_Num) * vertnbr);

  // Takes as input the graph, arch graph, strategy, migration cost in
  // double, old mapping and new mapping

  if (step() == 0) {
    SCOTCH_graphPart(&graph, parr->procs.size(), &strat, pemap);
  } else {
    SCOTCH_graphRepart(&graph, parr->procs.size(), oldpemap, migration_cost, NULL, &strat, pemap);
  }
  
  SCOTCH_graphExit (&graph);
  SCOTCH_stratExit (&strat);

  free(verttab);
  free(velotab);
  free(edgetab);
  free(edlotab);

  for(i = baseval; i < vertnbr; i++) {
    if(pemap[i] != ogr->vertices[i].getCurrentPe())
      ogr->vertices[i].setNewPe(pemap[i]);
  }

  free(pemap);
  free(oldpemap);
  /** ============================== CLEANUP ================================ */
  ogr->convertDecisions(stats);
  delete parr;
  delete ogr;
}
Ejemplo n.º 6
0
static PetscErrorCode MatPartitioningApply_PTScotch_Private(MatPartitioning part, PetscBool useND, IS *partitioning)
{
  MPI_Comm                 pcomm,comm;
  MatPartitioning_PTScotch *scotch = (MatPartitioning_PTScotch*)part->data;
  PetscErrorCode           ierr;
  PetscMPIInt              rank;
  Mat                      mat  = part->adj;
  Mat_MPIAdj               *adj = (Mat_MPIAdj*)mat->data;
  PetscBool                flg,distributed;
  PetscBool                proc_weight_flg;
  PetscInt                 i,j,p,bs=1,nold;
  PetscInt                 *NDorder = NULL;
  PetscReal                *vwgttab,deltval;
  SCOTCH_Num               *locals,*velotab,*veloloctab,*edloloctab,vertlocnbr,edgelocnbr,nparts=part->n;

  PetscFunctionBegin;
  ierr = PetscObjectGetComm((PetscObject)part,&pcomm);CHKERRQ(ierr);
  /* Duplicate the communicator to be sure that PTSCOTCH attribute caching does not interfere with PETSc. */
  ierr = MPI_Comm_dup(pcomm,&comm);CHKERRQ(ierr);
  ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
  ierr = PetscObjectTypeCompare((PetscObject)mat,MATMPIADJ,&flg);CHKERRQ(ierr);
  if (!flg) {
    /* bs indicates if the converted matrix is "reduced" from the original and hence the
       resulting partition results need to be stretched to match the original matrix */
    nold = mat->rmap->n;
    ierr = MatConvert(mat,MATMPIADJ,MAT_INITIAL_MATRIX,&mat);CHKERRQ(ierr);
    if (mat->rmap->n > 0) bs = nold/mat->rmap->n;
    adj  = (Mat_MPIAdj*)mat->data;
  }

  proc_weight_flg = PETSC_TRUE;
  ierr = PetscOptionsGetBool(NULL, NULL, "-mat_partitioning_ptscotch_proc_weight", &proc_weight_flg, NULL);CHKERRQ(ierr);

  ierr = PetscMalloc1(mat->rmap->n+1,&locals);CHKERRQ(ierr);

  if (useND) {
#if defined(PETSC_HAVE_SCOTCH_PARMETIS_V3_NODEND)
    PetscInt    *sizes, *seps, log2size, subd, *level, base = 0;
    PetscMPIInt size;

    ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
    log2size = PetscLog2Real(size);
    subd = PetscPowInt(2,log2size);
    if (subd != size) SETERRQ(comm,PETSC_ERR_SUP,"Only power of 2 communicator sizes");
    ierr = PetscMalloc1(mat->rmap->n,&NDorder);CHKERRQ(ierr);
    ierr = PetscMalloc3(2*size,&sizes,4*size,&seps,size,&level);CHKERRQ(ierr);
    SCOTCH_ParMETIS_V3_NodeND(mat->rmap->range,adj->i,adj->j,&base,NULL,NDorder,sizes,&comm);
    ierr = MatPartitioningSizesToSep_Private(subd,sizes,seps,level);CHKERRQ(ierr);
    for (i=0;i<mat->rmap->n;i++) {
      PetscInt loc;

      ierr = PetscFindInt(NDorder[i],2*subd,seps,&loc);CHKERRQ(ierr);
      if (loc < 0) {
        loc = -(loc+1);
        if (loc%2) { /* part of subdomain */
          locals[i] = loc/2;
        } else {
          ierr = PetscFindInt(NDorder[i],2*(subd-1),seps+2*subd,&loc);CHKERRQ(ierr);
          loc = loc < 0 ? -(loc+1)/2 : loc/2;
          locals[i] = level[loc];
        }
      } else locals[i] = loc/2;
    }
    ierr = PetscFree3(sizes,seps,level);CHKERRQ(ierr);
#else
    SETERRQ(pcomm,PETSC_ERR_SUP,"Need libptscotchparmetis.a compiled with -DSCOTCH_METIS_PREFIX");
#endif
  } else {
    velotab = NULL;
    if (proc_weight_flg) {
      ierr = PetscMalloc1(nparts,&vwgttab);CHKERRQ(ierr);
      ierr = PetscMalloc1(nparts,&velotab);CHKERRQ(ierr);
      for (j=0; j<nparts; j++) {
        if (part->part_weights) vwgttab[j] = part->part_weights[j]*nparts;
        else vwgttab[j] = 1.0;
      }
      for (i=0; i<nparts; i++) {
        deltval = PetscAbsReal(vwgttab[i]-PetscFloorReal(vwgttab[i]+0.5));
        if (deltval>0.01) {
          for (j=0; j<nparts; j++) vwgttab[j] /= deltval;
        }
      }
      for (i=0; i<nparts; i++) velotab[i] = (SCOTCH_Num)(vwgttab[i] + 0.5);
      ierr = PetscFree(vwgttab);CHKERRQ(ierr);
    }

    vertlocnbr = mat->rmap->range[rank+1] - mat->rmap->range[rank];
    edgelocnbr = adj->i[vertlocnbr];
    veloloctab = part->vertex_weights;
    edloloctab = adj->values;

    /* detect whether all vertices are located at the same process in original graph */
    for (p = 0; !mat->rmap->range[p+1] && p < nparts; ++p);
    distributed = (mat->rmap->range[p+1] == mat->rmap->N) ? PETSC_FALSE : PETSC_TRUE;

    if (distributed) {
      SCOTCH_Arch              archdat;
      SCOTCH_Dgraph            grafdat;
      SCOTCH_Dmapping          mappdat;
      SCOTCH_Strat             stradat;

      ierr = SCOTCH_dgraphInit(&grafdat,comm);CHKERRQ(ierr);
      ierr = SCOTCH_dgraphBuild(&grafdat,0,vertlocnbr,vertlocnbr,adj->i,adj->i+1,veloloctab,
                                NULL,edgelocnbr,edgelocnbr,adj->j,NULL,edloloctab);CHKERRQ(ierr);

#if defined(PETSC_USE_DEBUG)
      ierr = SCOTCH_dgraphCheck(&grafdat);CHKERRQ(ierr);
#endif

      ierr = SCOTCH_archInit(&archdat);CHKERRQ(ierr);
      ierr = SCOTCH_stratInit(&stradat);CHKERRQ(ierr);
      ierr = SCOTCH_stratDgraphMapBuild(&stradat,scotch->strategy,nparts,nparts,scotch->imbalance);CHKERRQ(ierr);

      if (velotab) {
        ierr = SCOTCH_archCmpltw(&archdat,nparts,velotab);CHKERRQ(ierr);
      } else {
        ierr = SCOTCH_archCmplt( &archdat,nparts);CHKERRQ(ierr);
      }
      ierr = SCOTCH_dgraphMapInit(&grafdat,&mappdat,&archdat,locals);CHKERRQ(ierr);
      ierr = SCOTCH_dgraphMapCompute(&grafdat,&mappdat,&stradat);CHKERRQ(ierr);

      SCOTCH_dgraphMapExit(&grafdat,&mappdat);
      SCOTCH_archExit(&archdat);
      SCOTCH_stratExit(&stradat);
      SCOTCH_dgraphExit(&grafdat);

    } else if (rank == p) {
      SCOTCH_Graph   grafdat;
      SCOTCH_Strat   stradat;

      ierr = SCOTCH_graphInit(&grafdat);CHKERRQ(ierr);
      ierr = SCOTCH_graphBuild(&grafdat,0,vertlocnbr,adj->i,adj->i+1,veloloctab,NULL,edgelocnbr,adj->j,edloloctab);CHKERRQ(ierr);

#if defined(PETSC_USE_DEBUG)
      ierr = SCOTCH_graphCheck(&grafdat);CHKERRQ(ierr);
#endif

      ierr = SCOTCH_stratInit(&stradat);CHKERRQ(ierr);
      ierr = SCOTCH_stratGraphMapBuild(&stradat,scotch->strategy,nparts,scotch->imbalance);CHKERRQ(ierr);

      ierr = SCOTCH_graphPart(&grafdat,nparts,&stradat,locals);CHKERRQ(ierr);

      SCOTCH_stratExit(&stradat);
      SCOTCH_graphExit(&grafdat);
    }

    ierr = PetscFree(velotab);CHKERRQ(ierr);
  }
  ierr = MPI_Comm_free(&comm);CHKERRQ(ierr);

  if (bs > 1) {
    PetscInt *newlocals;
    ierr = PetscMalloc1(bs*mat->rmap->n,&newlocals);CHKERRQ(ierr);
    for (i=0;i<mat->rmap->n;i++) {
      for (j=0;j<bs;j++) {
        newlocals[bs*i+j] = locals[i];
      }
    }
    ierr = PetscFree(locals);CHKERRQ(ierr);
    ierr = ISCreateGeneral(pcomm,bs*mat->rmap->n,newlocals,PETSC_OWN_POINTER,partitioning);CHKERRQ(ierr);
  } else {
    ierr = ISCreateGeneral(pcomm,mat->rmap->n,locals,PETSC_OWN_POINTER,partitioning);CHKERRQ(ierr);
  }
  if (useND) {
    IS ndis;

    if (bs > 1) {
      ierr = ISCreateBlock(pcomm,bs,mat->rmap->n,NDorder,PETSC_OWN_POINTER,&ndis);CHKERRQ(ierr);
    } else {
      ierr = ISCreateGeneral(pcomm,mat->rmap->n,NDorder,PETSC_OWN_POINTER,&ndis);CHKERRQ(ierr);
    }
    ierr = ISSetPermutation(ndis);CHKERRQ(ierr);
    ierr = PetscObjectCompose((PetscObject)(*partitioning),"_petsc_matpartitioning_ndorder",(PetscObject)ndis);CHKERRQ(ierr);
    ierr = ISDestroy(&ndis);CHKERRQ(ierr);
  }

  if (!flg) {
    ierr = MatDestroy(&mat);CHKERRQ(ierr);
  }
  PetscFunctionReturn(0);
}
bool
ScotchSplitter(unsigned dim, 
	       const int* ptRows, const int* indCols, 
	       unsigned& nbMaxLevels, unsigned minSize,
	       int* loc2glob, int* glob2loc, int& nbDoms, 
	       int*& ptOnDomains, int*& sizeOfDomains,
	       bool checkData, const bool verbose, FILE *fp)
{
  int ierr;
  // check consistency between Scotch and Dissection library
  CHECK(sizeof(int) == sizeof(SCOTCH_Num),
	"Incompatible integer representation between Scotch and Dissection");
  if (verbose) {
    int vers, rela, patc;
    SCOTCH_version(&vers, &rela, &patc);
    diss_printf(verbose, fp, "%s %d : Soctch version : %d.%d.%d\n",
		__FILE__, __LINE__, vers, rela, patc);
  }
  // Allocate and initialize the Scotch graph
  SCOTCH_Graph ptGraph;
  ierr = SCOTCH_graphInit(&ptGraph);
  CHECK(ierr==0,"Fail initializing Scotch graph");
  //  TRACE("Building Scotch graph\n");
  ierr = SCOTCH_graphBuild(&ptGraph,
			   (SCOTCH_Num)0, // offset,
			   (SCOTCH_Num)dim,
			   (const SCOTCH_Num*)ptRows,
			   (const SCOTCH_Num*)ptRows+1,
			   NULL, NULL, (SCOTCH_Num)ptRows[dim],
			   (const SCOTCH_Num*)indCols, NULL);
  //  if (verbose) {
  // fprintf(fp, "%s %d : 1 : ptRows = %p indCols = %p\n", 
  //	    __FILE__, __LINE__, (void *)ptRows, (void *)indCols);
  // }
  CHECK(ierr==0,"Scotch graph building failed !");
  if (checkData) {
    //TRACE("Check Scotch graph\n");
    ierr = SCOTCH_graphCheck(&ptGraph);
    if (ierr) {
      diss_printf(verbose, fp,
		  "Failed the checking of the graph : bad data ?\n");
      SCOTCH_graphFree(&ptGraph);
      return false;
    }
  }
  // Allocate and initialize the strategy wanted for Scotch
  // TRACE("Initialize strategy for splitting\n");
  SCOTCH_Strat ptStrat;
  ierr = SCOTCH_stratInit(&ptStrat);
  CHECK(ierr==0,
	"Failed initializing Scotch strategy structure");

  char *str_Strat = new char[1024];
  int nbLvls = std::min(unsigned(nbMaxLevels),
			highestbit(unsigned(dim/minSize)));
  diss_printf(verbose, fp, "nbLevels = %d\n", nbLvls);
  sprintf(str_Strat,"c{rat=0.7,cpr=n{sep=/((levl<%d)|(vert>%d))?m{type=h,rat=0.7,vert=100,low=h{pass=10},asc=b{width=3,bnd=f{bal=0.2},org=h{pass=10}f{bal=0.2}}}|m{type=h,rat=0.7,vert=100,low=h{pass=10},asc=b{width=3,bnd=f{bal=0.2},org=h{pass=10}f{bal=0.2}}};,ole=f{cmin=%d,cmax=%d,frat=0.05},ose=s},unc=n{sep=/(levl<%d)?(m{type=h,rat=0.7,vert=100,low=h{pass=10},asc=b{width=3,bnd=f{bal=0.2},org=h{pass=10}f{bal=0.2}}})|m{type=h,rat=0.7,vert=100,low=h{pass=10},asc=b{width=3,bnd=f{bal=0.2},org=h{pass=10}f{bal=0.2}}};,ole=f{cmin=%d,cmax=%d,frat=0.05},ose=s}}",
	  nbLvls-1,2*minSize-1,minSize,dim,nbLvls-1,minSize,dim);
  //  DBG_PRINT("Strategy string : %s\n", str_Strat);
  ierr = SCOTCH_stratGraphOrder(&ptStrat, str_Strat);
  delete [] str_Strat;
  CHECK(ierr==0,
	"Failed build graph ordering strategy for Scotch");  
  // Ordering with nested bisection :
  // TRACE("Split the graph\n");
  int* rangtab = new int[dim+1];
  int* treetab = new int[dim];
  int nbSplitDoms;
  bool repeat = true;
  int lastCompleteLevel;
  int *levels, *nbDomsPerLevels;
  SCOTCH_randomReset();
  while (repeat) {
    ierr = SCOTCH_graphOrder(&ptGraph, &ptStrat,
			     (SCOTCH_Num*)loc2glob, 
			     (SCOTCH_Num*)glob2loc, 
			     (SCOTCH_Num*)&nbSplitDoms,
			     (SCOTCH_Num*)rangtab,
			     (SCOTCH_Num*)treetab);
    if (ierr) {
      diss_printf(verbose, fp, "Failed reordering sparse matrix graph !\n");
      SCOTCH_stratExit(&ptStrat);
      SCOTCH_graphFree(&ptGraph);
      return false;
    }

    levels = new int[nbSplitDoms];
//    int *nbDomsPerLevels;// = new int[nbLvls];
    unsigned nbLvlsScotch= compLevelOfDoms(nbSplitDoms, nbLvls, treetab, levels,
					   nbDomsPerLevels);
    /** Search last level where number of domains is a power of two
     */
    lastCompleteLevel = 0;
    while ((lastCompleteLevel<nbLvlsScotch) && 
	   ((1<<lastCompleteLevel) == nbDomsPerLevels[lastCompleteLevel]) ) {
      lastCompleteLevel ++;
    }
    lastCompleteLevel = std::min(lastCompleteLevel, nbLvls);

    nbDoms = (1<<lastCompleteLevel)-1;

    // Search where start each domain per bisection level 
    // and compute the size of each subdomain :
    //  int indDom = 0;
    bool flag_size_check = false;
    for (int i = 0; i < nbSplitDoms; i++) {
      int sz = 0;
      while (levels[i]>=lastCompleteLevel) {
	sz += rangtab[i+1]-rangtab[i];
	i++;
      }
      if (sz+rangtab[i+1]-rangtab[i] <= TOO_SMALL) {
	flag_size_check = true;
	break;
      }
      //DBG_PRINT("Domain %d begin at %d\n",indDom+1,begDom);
    } // loop : i
    if (!flag_size_check) {
      repeat = false;
      break;
    }
    else {
      delete [] levels;
      delete [] nbDomsPerLevels;
    }
  } // while (repeat)
  ptOnDomains   = new int[nbDoms+1];
  sizeOfDomains = new int[nbDoms];
  memset(sizeOfDomains, 0, nbDoms*sizeof(int));
  int* indDomPerLevel = new int[lastCompleteLevel+1];
  memset(indDomPerLevel,0,(lastCompleteLevel+1)*sizeof(int));
  int begDom = 0;
  for (int i = 0; i < nbSplitDoms; i++) {
    int sz = 0;
    while (levels[i]>=lastCompleteLevel) {
      sz += rangtab[i+1]-rangtab[i];
      i++;
    }
    int indDom = (1<<levels[i])-1+indDomPerLevel[levels[i]];
    // DBG_PRINT("level %d : current dom = %d\n", levels[i],indDom+1);
    ptOnDomains[indDom] = begDom;
    //DBG_PRINT("Domain %d begin at %d\n",indDom+1,begDom);
    sizeOfDomains[indDom] = sz+rangtab[i+1]-rangtab[i];
    //DBG_PRINT("Domain %d size of %d\n",indDom+1,sizeOfDomains[indDom]);
    begDom += sizeOfDomains[indDom];
    indDomPerLevel[levels[i]] += 1;
  }
  diss_printf(verbose, fp, "%s %d : indDomPerlevel[lastCompleteLevel] = %d\n",
	      __FILE__, __LINE__, indDomPerLevel[lastCompleteLevel]);
  ptOnDomains[nbDoms] = dim;
  nbMaxLevels = lastCompleteLevel;
  delete [] indDomPerLevel;
  delete [] nbDomsPerLevels;
  delete [] levels;
  delete [] treetab;
  delete [] rangtab;
  // Cleaning all Scotch structures :
  SCOTCH_stratExit(&ptStrat);
  SCOTCH_graphFree(&ptGraph);
  return true;
}
Ejemplo n.º 8
0
Archivo: gcv.c Proyecto: Hartorn/AN304
int
main (
int                         argc,
char *                      argv[])
{
  SCOTCH_Graph        grafdat;
  SCOTCH_Geom         geomdat;
  int                 i, j;

  errorProg ("gcv");

  if ((argc >= 2) && (argv[1][0] == '?')) {       /* If need for help */
    usagePrint (stdout, C_usageList);
    return     (0);
  }

  for (i = 0; i < C_FILENBR; i ++)                /* Set default stream pointers */
    C_fileTab[i].pntr = (C_fileTab[i].mode[0] == 'r') ? stdin : stdout;
  for (i = 1; i < argc; i ++) {                   /* Loop for all option codes                        */
    if ((argv[i][0] != '-') || (argv[i][1] == '\0') || (argv[i][1] == '.')) { /* If found a file name */
      if (C_fileNum < C_FILEARGNBR)               /* File name has been given                         */
        C_fileTab[C_fileNum ++].name = argv[i];
      else {
        errorPrint ("main: too many file names given");
        return     (1);
      }
    }
    else {                                       /* If found an option name */
      switch (argv[i][1]) {
        case 'H' :                               /* Give help */
        case 'h' :
          usagePrint (stdout, C_usageList);
          return     (0);
        case 'I' :                               /* Select input file type */
        case 'i' :
          for (j = 0; C_inpFormatTab[j].code != '\0'; j ++) { /* Find proper format code */
            if (C_inpFormatTab[j].code == argv[i][2]) {
              C_inpFormatType = j;
              C_inpFormatData = &argv[i][3];
              break;
            }
          }
          if (C_inpFormatTab[j].code == '\0') {
            errorPrint ("main: unprocessed option (\"%s\")", argv[i]);
            return     (1);
          }
          break;
        case 'O' :                               /* Select input file type */
        case 'o' :
          for (j = 0; C_outFormatTab[j].code != '\0'; j ++) { /* Find proper format code */
            if (C_outFormatTab[j].code == argv[i][2]) {
              C_outFormatType = j;
              C_outFormatData = &argv[i][3];
              break;
            }
          }
          if (C_inpFormatTab[j].code == '\0') {
            errorPrint ("main: unprocessed option (\"%s\")", argv[i]);
            return     (1);
          }
          break;
        case 'V' :
          fprintf (stderr, "gcv, version " SCOTCH_VERSION_STRING "\n");
          fprintf (stderr, "Copyright 2004,2007,2008,2010 ENSEIRB, INRIA & CNRS, France\n");
          fprintf (stderr, "This software is libre/free software under CeCILL-C -- see the user's manual for more information\n");
          return  (0);
        default :
          errorPrint ("main: unprocessed option (\"%s\")", argv[i]);
          return     (1);
      }
    }
  }

  fileBlockOpen (C_fileTab, C_FILENBR);           /* Open all files */

  SCOTCH_graphInit (&grafdat);
  SCOTCH_geomInit  (&geomdat);
  C_inpFormatTab[C_inpFormatType].func (&grafdat, &geomdat, C_filepntrsrcinp, NULL, C_inpFormatData);
#ifdef SCOTCH_DEBUG_ALL
  if (SCOTCH_graphCheck (&grafdat) != 0) {
    errorPrint ("main: bad graph structure");
    return (1);
  }
#endif /* SCOTCH_DEBUG_ALL */
  C_outFormatTab[C_outFormatType].func (&grafdat, &geomdat, C_filepntrsrcout, C_filepntrgeoout, C_outFormatData);

  fileBlockClose (C_fileTab, C_FILENBR);          /* Always close explicitely to end eventual (un)compression tasks */

  SCOTCH_geomExit  (&geomdat);
  SCOTCH_graphExit (&grafdat);

#ifdef COMMON_PTHREAD
  pthread_exit ((void *) 0);                      /* Allow potential (un)compression tasks to complete */
#endif /* COMMON_PTHREAD */
  return (0);
}
Ejemplo n.º 9
0
static PetscErrorCode MatPartitioningApply_Scotch(MatPartitioning part, IS * partitioning)
{
    PetscErrorCode ierr;
    int  *parttab, *locals = PETSC_NULL, rank, i, size;
    size_t                 j;
    Mat                    mat = part->adj, matMPI, matSeq;
    int                    nb_locals = mat->rmap->n;
    Mat_MPIAdj             *adj = (Mat_MPIAdj *) mat->data;
    MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;
    PetscTruth             flg;
#ifdef PETSC_HAVE_UNISTD_H
    int                    fd_stdout, fd_pipe[2], count,err;
#endif

    PetscFunctionBegin;

    /* check if the matrix is sequential, use MatGetSubMatrices if necessary */
    ierr = MPI_Comm_size(((PetscObject)mat)->comm, &size);CHKERRQ(ierr);
    ierr = PetscTypeCompare((PetscObject) mat, MATMPIADJ, &flg);CHKERRQ(ierr);
    if (size > 1) {
        int M, N;
        IS isrow, iscol;
        Mat *A;

        if (flg) {
            SETERRQ(0, "Distributed matrix format MPIAdj is not supported for sequential partitioners");
        }
        PetscPrintf(((PetscObject)part)->comm, "Converting distributed matrix to sequential: this could be a performance loss\n");CHKERRQ(ierr);

        ierr = MatGetSize(mat, &M, &N);CHKERRQ(ierr);
        ierr = ISCreateStride(PETSC_COMM_SELF, M, 0, 1, &isrow);CHKERRQ(ierr);
        ierr = ISCreateStride(PETSC_COMM_SELF, N, 0, 1, &iscol);CHKERRQ(ierr);
        ierr = MatGetSubMatrices(mat, 1, &isrow, &iscol, MAT_INITIAL_MATRIX, &A);CHKERRQ(ierr);
        matSeq = *A; 
        ierr = PetscFree(A);CHKERRQ(ierr);
        ierr = ISDestroy(isrow);CHKERRQ(ierr);
        ierr = ISDestroy(iscol);CHKERRQ(ierr);
    } else
        matSeq = mat;

    /* convert the the matrix to MPIADJ type if necessary */
    if (!flg) {
        ierr = MatConvert(matSeq, MATMPIADJ, MAT_INITIAL_MATRIX, &matMPI);CHKERRQ(ierr);
    } else {
        matMPI = matSeq;
    }

    adj = (Mat_MPIAdj *) matMPI->data;  /* finaly adj contains adjacency graph */

    ierr = MPI_Comm_rank(((PetscObject)part)->comm, &rank);CHKERRQ(ierr);

    {
        /* definition of Scotch library arguments */
        SCOTCH_Strat stratptr;      /* scotch strategy */
        SCOTCH_Graph grafptr;       /* scotch graph */
#if defined(DOES_NOT_COMPILE_DUE_TO_BROKEN_INTERFACE)
        int vertnbr = mat->rmap->N; /* number of vertices in full graph */
        int *verttab = adj->i;      /* start of edge list for each vertex */
        int *edgetab = adj->j;      /* edge list data */
        int edgenbr = adj->nz;      /* number of edges */
        int *velotab = NULL;        /* not used by petsc interface */
        int *vlbltab = NULL;    
        int *edlotab = NULL; 
        int flagval = 3;            /* (cf doc scotch no weight edge & vertices) */
#endif  
        int baseval = 0;            /* 0 for C array indexing */
        char strategy[256];

        ierr = PetscMalloc((mat->rmap->N) * sizeof(int), &parttab);CHKERRQ(ierr); 

        /* redirect output to buffer scotch -> mesg_log */
#ifdef PETSC_HAVE_UNISTD_H
        fd_stdout = dup(1);
        pipe(fd_pipe);
        close(1);
        dup2(fd_pipe[1], 1);
        ierr = PetscMalloc(SIZE_LOG * sizeof(char), &(scotch->mesg_log));CHKERRQ(ierr);
#endif

        /* library call */

        /* Construction of the scotch graph object */
        ierr = SCOTCH_graphInit(&grafptr);
#if defined(DOES_NOT_COMPILE_DUE_TO_BROKEN_INTERFACE)
        ierr = SCOTCH_graphBuild((SCOTCH_Graph *)   &grafptr, 
				 (const SCOTCH_Num)  vertnbr, 
				 (const SCOTCH_Num)  verttab, 
				 (const SCOTCH_Num *)velotab,
				 (const SCOTCH_Num *)vlbltab, 
				 (const SCOTCH_Num *)edgenbr, 
				 (const SCOTCH_Num *)edgetab, 
				 (const SCOTCH_Num)  edlotab, 
				 (const SCOTCH_Num *)baseval, 
				 (const SCOTCH_Num *)flagval);CHKERRQ(ierr);
#else
        SETERRQ(PETSC_ERR_SUP,"Scotch interface currently broken");
#endif
        ierr = SCOTCH_graphCheck(&grafptr);CHKERRQ(ierr);

        /* Construction of the strategy */
        if (scotch->strategy[0] != 0) {
            ierr = PetscStrcpy(strategy, scotch->strategy);CHKERRQ(ierr);
        } else {
            PetscStrcpy(strategy, "b{strat=");

            if (scotch->multilevel) {
                /* PetscStrcat(strategy,"m{vert=");
                   sprintf(strategy+strlen(strategy),"%d",scotch->nbvtxcoarsed);
                   PetscStrcat(strategy,",asc="); */
                sprintf(strategy, "b{strat=m{vert=%d,asc=",
                    scotch->nbvtxcoarsed);
            } else
                PetscStrcpy(strategy, "b{strat=");

            switch (scotch->global_method) {
            case MP_SCOTCH_GREEDY:
                PetscStrcat(strategy, "h");
                break;
            case MP_SCOTCH_GPS:
                PetscStrcat(strategy, "g");
                break;
            case MP_SCOTCH_GR_GPS:
                PetscStrcat(strategy, "g|h");
            }

            switch (scotch->local_method) {
            case MP_SCOTCH_KERNIGHAN_LIN:
                if (scotch->multilevel)
                    PetscStrcat(strategy, ",low=f}");
                else
                    PetscStrcat(strategy, " f");
                break;
            case MP_SCOTCH_NONE:
                if (scotch->multilevel)
                    PetscStrcat(strategy, ",asc=x}");
            default:
                break;
            }

            PetscStrcat(strategy, " x}");
        }

        PetscPrintf(((PetscObject)part)->comm, "strategy=[%s]\n", strategy);

        ierr = SCOTCH_stratInit(&stratptr);CHKERRQ(ierr);
	/*

	  TODO: Correct this part

	  Commented because this doesn't exists anymore 

	  
	  ierr = SCOTCH_stratMap(&stratptr, strategy);CHKERRQ(ierr);
	*/
        /* check for option mapping */
        if (!scotch->map) {
	  /* ********************************************
	   *						*
	   *        TODO: Correct this part		*
	   *						*
	   * Won't work with this tmp SCOTCH_Strat...	*
	   *						*
	   * I just modified it to make scotch compile, *
	   * to be able to use PaStiX...		*
	   *						*
	   **********************************************/
#if defined (DOES_NOT_COMPILE_DUE_TO_BROKEN_INTERFACE)
	  SCOTCH_Strat tmp;
	  ierr = SCOTCH_graphPart((const SCOTCH_Graph *)&grafptr, 
				  (const SCOTCH_Num)    &stratptr, 
				  (const SCOTCH_Strat *)&tmp,        /* The Argument changed from scotch 3.04 it was part->n, */ 
				  (SCOTCH_Num *)        parttab);CHKERRQ(ierr);
#else
        SETERRQ(PETSC_ERR_SUP,"Scotch interface currently broken");
#endif
            ierr = PetscPrintf(PETSC_COMM_SELF, "Partition simple without mapping\n");
        } else {
            SCOTCH_Graph grafarch;
            SCOTCH_Num *listtab;
            SCOTCH_Num listnbr = 0;
            SCOTCH_Arch archptr;        /* file in scotch architecture format */
            SCOTCH_Strat archstrat;
            int arch_total_size, *parttab_tmp,err;
            int cpt;
            char buf[256];
            FILE *file1, *file2;
            char host_buf[256];

            /* generate the graph that represents the arch */
            file1 = fopen(scotch->arch, "r");
            if (!file1) SETERRQ1(PETSC_ERR_FILE_OPEN, "Scotch: unable to open architecture file %s", scotch->arch);

            ierr = SCOTCH_graphInit(&grafarch);CHKERRQ(ierr);
            ierr = SCOTCH_graphLoad(&grafarch, file1, baseval, 3);CHKERRQ(ierr);

            ierr = SCOTCH_graphCheck(&grafarch);CHKERRQ(ierr);
            SCOTCH_graphSize(&grafarch, &arch_total_size, &cpt);

            err = fclose(file1);
            if (err) SETERRQ(PETSC_ERR_SYS,"fclose() failed on file");    

            printf("total size = %d\n", arch_total_size);

            /* generate the list of nodes currently working */
            ierr = PetscGetHostName(host_buf, 256);CHKERRQ(ierr);
            ierr = PetscStrlen(host_buf, &j);CHKERRQ(ierr);

            file2 = fopen(scotch->host_list, "r");
            if (!file2) SETERRQ1(PETSC_ERR_FILE_OPEN, "Scotch: unable to open host list file %s", scotch->host_list);

            i = -1;
            flg = PETSC_FALSE;
            while (!feof(file2) && !flg) {
                i++;
                fgets(buf, 256, file2);
                PetscStrncmp(buf, host_buf, j, &flg);
            }
            err = fclose(file2);
            if (err) SETERRQ(PETSC_ERR_SYS,"fclose() failed on file");    
            if (!flg) SETERRQ1(PETSC_ERR_LIB, "Scotch: unable to find '%s' in host list file", host_buf);

            listnbr = size;
            ierr = PetscMalloc(sizeof(SCOTCH_Num) * listnbr, &listtab);CHKERRQ(ierr);

            ierr = MPI_Allgather(&i, 1, MPI_INT, listtab, 1, MPI_INT, ((PetscObject)part)->comm);CHKERRQ(ierr);

            printf("listnbr = %d, listtab = ", listnbr);
            for (i = 0; i < listnbr; i++)
                printf("%d ", listtab[i]);

            printf("\n");
            err = fflush(stdout);
            if (err) SETERRQ(PETSC_ERR_SYS,"fflush() failed on file");    

            ierr = SCOTCH_stratInit(&archstrat);CHKERRQ(ierr);
	    /**************************************************************
	     *								  *
	     * TODO: Correct this part					  *
	     * 								  *
	     * Commented because this doesn't exists anymore 		  *
	     * 								  *
	     * ierr = SCOTCH_stratBipart(&archstrat, "fx");CHKERRQ(ierr); *
	     **************************************************************/
            ierr = SCOTCH_archInit(&archptr);CHKERRQ(ierr);
            ierr = SCOTCH_archBuild(&archptr, &grafarch, listnbr, listtab,
                &archstrat);CHKERRQ(ierr);

            ierr = PetscMalloc((mat->rmap->N) * sizeof(int), &parttab_tmp);CHKERRQ(ierr);
	    /************************************************************************************
	     *											*
	     * TODO: Correct this part								*
	     *											*
	     * Commented because this doesn't exists anymore 					*
	     *											*
	     * ierr = SCOTCH_mapInit(&mappptr, &grafptr, &archptr, parttab_tmp);CHKERRQ(ierr);	*
	     *											*
	     * ierr = SCOTCH_mapCompute(&mappptr, &stratptr);CHKERRQ(ierr);			*
	     * 											*
	     * ierr = SCOTCH_mapView(&mappptr, stdout);CHKERRQ(ierr);				*
	     ************************************************************************************/
            /* now we have to set in the real parttab at the good place */
            /* because the ranks order are different than position in */
            /* the arch graph */
            for (i = 0; i < mat->rmap->N; i++) {
                parttab[i] = parttab_tmp[i];
            }

            ierr = PetscFree(listtab);CHKERRQ(ierr);
            SCOTCH_archExit(&archptr);
	    /*************************************************
   	     * TODO: Correct this part			     *
	     * 						     *
	     * Commented because this doesn't exists anymore *
	     * SCOTCH_mapExit(&mappptr);		     *
	     *************************************************/
            SCOTCH_stratExit(&archstrat);
        }

        /* dump to mesg_log... */
#ifdef PETSC_HAVE_UNISTD_H
        err = fflush(stdout);
        if (err) SETERRQ(PETSC_ERR_SYS,"fflush() failed on stdout");    

        count = read(fd_pipe[0], scotch->mesg_log, (SIZE_LOG - 1) * sizeof(char));
        if (count < 0)
            count = 0;
        scotch->mesg_log[count] = 0;
        close(1);
        dup2(fd_stdout, 1);
        close(fd_stdout);
        close(fd_pipe[0]);
        close(fd_pipe[1]);
#endif

        SCOTCH_graphExit(&grafptr);
        SCOTCH_stratExit(&stratptr);
    }

    if (ierr)
        SETERRQ(PETSC_ERR_LIB, scotch->mesg_log);

    /* Creation of the index set */

    ierr = MPI_Comm_rank(((PetscObject)part)->comm, &rank);CHKERRQ(ierr);
    ierr = MPI_Comm_size(((PetscObject)part)->comm, &size);CHKERRQ(ierr);
    nb_locals = mat->rmap->N / size;
    locals = parttab + rank * nb_locals;
    if (rank < mat->rmap->N % size) {
        nb_locals++;
        locals += rank;
    } else
        locals += mat->rmap->N % size;
    ierr = ISCreateGeneral(((PetscObject)part)->comm, nb_locals, locals, partitioning);CHKERRQ(ierr);

    /* destroying old objects */
    ierr = PetscFree(parttab);CHKERRQ(ierr);
    if (matSeq != mat) {
        ierr = MatDestroy(matSeq);CHKERRQ(ierr);
    }
    if (matMPI != mat) {
        ierr = MatDestroy(matMPI);CHKERRQ(ierr);
    }

    PetscFunctionReturn(0);
}
Ejemplo n.º 10
0
      static
      void run_scotch( Container         &c,
                       MapContainer      &mapping,
                       const std::size_t cores,
                       weight_function_t weight_func,
                       void              *weight )
   {
#if 0
      static_assert( std::is_signed< 
         std::remove_reference< decltype( c.end() ) >::type >::value, 
            "Container must have signed types so that -1 may signify no mapping" );
#endif            
      raftgraph_t raft_graph;
      get_graph_info( c, 
                      raft_graph, 
                      weight_func, 
                      nullptr );
      SCOTCH_Graph graph;
      if( SCOTCH_graphInit( &graph ) != 0 )
      {
         /** TODO, add RaftLib Exception **/
         std::cerr << "Failed to initialize graph!!\n";
         exit( EXIT_FAILURE );
      }
      auto table( raft_graph.getScotchTables() );
      if( SCOTCH_graphBuild( 
            &graph                  /** graph ptr     **/,
            0                       /** base value    **/,
            table.num_vertices      /** vertex nmbr (zero indexed)   **/,
            table.vtable            /** vertex tab **/,
            &table.vtable[ 1 ]      /** vendtab **/,
            nullptr           /** velotab **/,
            nullptr           /** vlbltab **/,
            table.num_edges                 /** edge number **/,
            table.etable             /** edge tab **/,
            table.eweight         /** edlotab **/
          ) != 0 )
      {
         /** TODO, add RaftLib Exception **/
         std::cerr << "Failed to build graph\n";
         exit( EXIT_FAILURE );
      }
      if( SCOTCH_graphCheck( &graph ) != 0 )
      {
         /** TODO, add RaftLib Exception **/
         std::cerr << "Graph is inconsistent\n";
         std::remove_reference< decltype( table ) >::type::print( std::cerr, table );
         std::cerr << "\n";
         raft_graph.print( std::cerr );
         
         exit( EXIT_FAILURE );
      }
      /** TODO, we can do much more with this arch file **/
      SCOTCH_Arch archdat;
      if( SCOTCH_archInit( &archdat )  != 0 )
      {
         /** TODO, add RaftLib Exception **/
         std::cerr << "Architecture initialization failed\n";
         exit( EXIT_FAILURE );
      }
      /** core are equal **/
      if( SCOTCH_archCmplt( &archdat, cores /** num cores **/) != 0 )
      {
         /** TODO, add RaftLib Exception **/
         std::cerr << "Failed to create architecture file\n";
         exit( EXIT_FAILURE );
      }
      /** strategy **/
      SCOTCH_Strat stradat;
      if( SCOTCH_stratInit( &stradat ) != 0 )
      {
         /** TODO, add RaftLib Exception **/
         std::cerr << "Failed to init strategy!!\n";
         exit( EXIT_FAILURE );
      }
      /** build recursive strategy **/
      if( SCOTCH_stratGraphClusterBuild(
                                   &stradat,
                                   SCOTCH_STRATSPEED,
                                   cores,
                                   .75,
                                   .01) != 0 )
      {
         /** TODO, add RaftLib Exception **/
         std::cerr << "Failed to map strategy graph!!\n";
         exit( EXIT_FAILURE );
      }
      if( SCOTCH_graphMap( 
            &graph             /** graph ptr **/,
            &archdat,
            &stradat,
            table.partition    /** parttab **/
            ) != 0 )
      {
         /** TODO, add RaftLib Exception **/
         std::cerr << "Failed to map!!\n";
         exit( EXIT_FAILURE );
      }
      /**
       * first case is for if we've mapped all vertices, 
       * second is for when some of the kernels are innactive
       * in which case the number of vertices in the 
       * table will be less than the size of c in which case
       * we need to get which vertices (the actual number id
       * from the application) are mapped and to where, the 
       * returned table in mapping must include even the 
       * vertices that aren't active (indicated by a -1) so
       * that the returning loop can be as simple as possible
       */
      if( c.size() == table.num_vertices )
      {
         /** copy mapping **/ 
         for( auto i( 0 ); i < table.num_vertices; i++ )
         {
            mapping.emplace_back( table.partition[ i ] );
         }
      }
      else
      {
         const auto &vmapping( raft_graph.getVertexNumbersAtIndicies() );
         auto it_map_index( vmapping.cbegin() );
         auto table_index( 0 );
         const auto size( c.size() );
         for( auto i( 0 ); i < size; i++ )
         {
            if( i == (*it_map_index) &&  it_map_index != vmapping.cend() )
            {
               mapping.emplace_back( table.partition[ table_index++ ] );
               ++it_map_index;
            }
            else
            {
               mapping.emplace_back( -1 );
            }
         }
      }
      /** call exit graph **/
      SCOTCH_graphExit( &graph    );
      SCOTCH_stratExit( &stradat );
      SCOTCH_archExit ( &archdat );
      return;
   }
Ejemplo n.º 11
0
/* Function : renumbering
 *  it modifies the numbering of each node to prevent from cache missing.
 *
 *  - boxVertNbr : number of vertices by box
 *  - mesh : the input mesh which is modified
 *  
 *  returning 0 if OK, 1 else
 */
int renumbering(int boxVertNbr, MMG_pMesh mesh, MMG_pSol sol) {
  MMG_pPoint ppt;
  MMG_pPoint points;
  MMG_pTria ptri, trias;
  MMG_pTetra ptet, tetras;
  SCOTCH_Num edgeNbr;
  SCOTCH_Num *vertTab, *vendTab, *edgeTab, *permVrtTab;
  SCOTCH_Graph graf ;
  int vertNbr, nodeGlbIdx, triaIdx, tetraIdx, ballTetIdx;
  int i, j, k, addrNew, addrOld;
  int edgeSiz;
  int *vertOldTab, *permNodTab, ntreal, nereal, npreal;
  int      *adja,iadr;
  double *metNew;


  /* Computing the number of vertices and a contiguous tabular of vertices */
  vertNbr = 0;
  vertOldTab = (int *)M_calloc(mesh->ne + 1, sizeof(int), "renumbering");

  if (!memset(vertOldTab, 0, sizeof(int)*(mesh->ne+1))) {
    perror("memset");
    return 1;
  }

  for(tetraIdx = 1 ; tetraIdx < mesh->ne + 1 ; tetraIdx++) {

    /* Testing if the tetra exists */
    if (!mesh->tetra[tetraIdx].v[0]) continue;
    vertOldTab[tetraIdx] = vertNbr+1;
    vertNbr++;
  }


  /* Allocating memory to compute adjacency lists */
  vertTab = (SCOTCH_Num *)M_calloc(vertNbr + 1, sizeof(SCOTCH_Num), "renumbering");

  if (!memset(vertTab, ~0, sizeof(SCOTCH_Num)*(vertNbr + 1))) {
    perror("memset");
    return 1;
  }

  vendTab = (SCOTCH_Num *)M_calloc(vertNbr + 1, sizeof(SCOTCH_Num), "renumbering");

  edgeNbr = 1;
  edgeSiz = vertNbr*2;
  edgeTab = (SCOTCH_Num *)M_calloc(edgeSiz, sizeof(SCOTCH_Num), "renumbering");



  /* Computing the adjacency list for each vertex */
  for(tetraIdx = 1 ; tetraIdx < mesh->ne + 1 ; tetraIdx++) {

    /* Testing if the tetra exists */
    if (!mesh->tetra[tetraIdx].v[0]) continue;





    iadr = 4*(tetraIdx-1) + 1;
    adja = &mesh->adja[iadr];
    for (i=0; i<4; i++) {
      ballTetIdx = adja[i] >> 2;

      if (!ballTetIdx) continue;






      /* Testing if one neighbour of tetraIdx has already been added */
      if (vertTab[vertOldTab[tetraIdx]] < 0)
        vertTab[vertOldTab[tetraIdx]] = edgeNbr;
      vendTab[vertOldTab[tetraIdx]] = edgeNbr+1;

      /* Testing if edgeTab memory is enough */
      if (edgeNbr >= edgeSiz) {
        edgeSiz += EDGEGAP;
        edgeTab = (SCOTCH_Num *)M_realloc(edgeTab, edgeSiz * sizeof(SCOTCH_Num), "renumbering");
      }

      edgeTab[edgeNbr++] = vertOldTab[ballTetIdx];
    }
  }

  edgeNbr--;


  /* Building the graph by calling Scotch functions */

  SCOTCH_graphInit(&graf) ;
  CHECK_SCOTCH(SCOTCH_graphBuild(&graf, (SCOTCH_Num) 1, vertNbr, vertTab+1, vendTab+1, NULL, NULL, edgeNbr, edgeTab+1, NULL), "scotch_graphbuild", 0) ;
  CHECK_SCOTCH(SCOTCH_graphCheck(&graf), "scotch_graphcheck", 0) ;

  permVrtTab = (SCOTCH_Num *)M_calloc(vertNbr + 1, sizeof(SCOTCH_Num), "renumbering");

  CHECK_SCOTCH(kPartBoxCompute(graf, vertNbr, boxVertNbr, permVrtTab), "boxCompute", 0);

  SCOTCH_graphExit(&graf) ;

  M_free(vertTab);
  M_free(vendTab);
  M_free(edgeTab);


  permNodTab = (int *)M_calloc(mesh->np + 1, sizeof(int), "renumbering");

  /* Computing the new point list and modifying the sol structures*/
  tetras = (MMG_pTetra)M_calloc(mesh->nemax+1,sizeof(MMG_Tetra),"renumbering");

  points = (MMG_pPoint)M_calloc(mesh->npmax+1,sizeof(MMG_Point),"renumbering");

  metNew = (double*)M_calloc(sol->npmax+1,sol->offset*sizeof(double),"renumbering");

  nereal = 0;
  npreal = 1;
  for(tetraIdx = 1 ; tetraIdx < mesh->ne + 1 ; tetraIdx++) {
    ptet = &mesh->tetra[tetraIdx];

    /* Testing if the tetra exists */
    if (!ptet->v[0]) continue;

    /* Building the new point list */
    tetras[permVrtTab[vertOldTab[tetraIdx]]] = *ptet;  
    nereal++;

    for(j = 0 ; j <= 3 ; j++) {

      nodeGlbIdx = mesh->tetra[tetraIdx].v[j];

      if (permNodTab[nodeGlbIdx]) continue;

      ppt = &mesh->point[nodeGlbIdx];

      if (!(ppt->tag & M_UNUSED)) {
        /* Building the new point list */
        permNodTab[nodeGlbIdx] = npreal++;

        points[permNodTab[nodeGlbIdx]] = *ppt;  

        /* Building the new sol met */
        addrOld = (nodeGlbIdx-1)*sol->offset + 1;
        addrNew = (permNodTab[nodeGlbIdx]-1)*sol->offset + 1;
        memcpy(&metNew[addrNew], &sol->met[addrOld], sol->offset*sizeof(double));
      }
    }
  }


  M_free(mesh->tetra);
  mesh->tetra = tetras;
  mesh->ne = nereal;

  M_free(mesh->point);
  mesh->point = points;
  mesh->np    = npreal - 1;

  M_free(sol->met);
  sol->met = metNew;

  trias = (MMG_pTria)M_calloc(mesh->ntmax+1,sizeof(MMG_Tria),"renumbering");

  ntreal = 1;
  for(triaIdx = 1 ; triaIdx < mesh->nt + 1 ; triaIdx++) {
    ptri = &mesh->tria[triaIdx];

    /* Testing if the tetra exists */
    if (!ptri->v[0]) continue;

    /* Building the new point list */
    trias[ntreal] = *ptri;  
    ntreal++;
  }

  M_free(mesh->tria);
  mesh->tria = trias;
  mesh->nt = ntreal - 1;

  mesh->npnil = mesh->np + 1;
  mesh->nenil = mesh->ne + 1;

  for (k=mesh->npnil; k<mesh->npmax-1; k++)
    mesh->point[k].tmp  = k+1;

  for (k=mesh->nenil; k<mesh->nemax-1; k++)
    mesh->tetra[k].v[3] = k+1;

  if ( mesh->nt ) {
    mesh->ntnil = mesh->nt + 1;
    for (k=mesh->ntnil; k<mesh->ntmax-1; k++)
      mesh->tria[k].v[2] = k+1;
  }



  /* Modifying the numbering of the nodes of each tetra */
  for(tetraIdx = 1 ; tetraIdx < mesh->ne + 1 ; tetraIdx++) {
    if (!mesh->tetra[tetraIdx].v[0]) continue;
    for(j = 0 ; j <= 3 ; j++) {
      mesh->tetra[tetraIdx].v[j] = permNodTab[mesh->tetra[tetraIdx].v[j]];
    }
  }

  /* Modifying the numbering of the nodes of each triangle */
  for(triaIdx = 1 ; triaIdx < mesh->nt + 1 ; triaIdx++) {  
    if (!mesh->tria[triaIdx].v[0]) continue;  
    for(j = 0 ; j <= 2 ; j++) {
      mesh->tria[triaIdx].v[j] = permNodTab[mesh->tria[triaIdx].v[j]];
    } 
  }

  M_free(permVrtTab);

  return 1;
}