Beispiel #1
0
void Cluster::subdivide(unsigned bmin)
{
    const unsigned size(_end - _beg);
    if (size > bmin) {

        idx_t n_rows(static_cast<idx_t>(_l_adj_mat->getNRows()));

        idx_t *xadj(new idx_t[n_rows+1]);
        unsigned const*const original_row_ptr(_l_adj_mat->getRowPtrArray());
        for(idx_t k(0); k<=n_rows; k++) {
            xadj[k] = original_row_ptr[k];
        }

        unsigned nnz(_l_adj_mat->getNNZ());
        idx_t *adjncy(new idx_t[nnz]);
        unsigned const*const original_adjncy(_l_adj_mat->getColIdxArray());
        for(unsigned k(0); k<nnz; k++) {
            adjncy[k] = original_adjncy[k];
        }
//		unsigned nparts = 2;
        idx_t options[METIS_NOPTIONS]; // for METIS
        METIS_SetDefaultOptions(options);
//		options[METIS OPTION PTYPE] = METIS PTYPE RB;
//		options[METIS OPTION OBJTYPE] = METIS OBJTYPE CUT;
//		options[METIS OPTION CTYPE] = METIS CTYPE SHEM;
//		options[] = ;
//		options[] = ;
//		options[] = ;

//		unsigned sepsize(0); // for METIS
        idx_t *vwgt(new idx_t[n_rows + 1]);
//		const unsigned nnz(xadj[n_rows]);
//		unsigned *adjwgt(new unsigned[nnz]);
        for (idx_t k(0); k < n_rows + 1; k++)
            vwgt[k] = 1;
//		for (unsigned k(0); k < nnz; k++)
//			adjwgt[k] = 1;
//		unsigned *part(new unsigned[n_rows + 1]);

        // subdivide the index set into three parts employing METIS
//		METIS_ComputeVertexSeparator(&n_rows, xadj, adjncy, vwgt, &options,
//				&sepsize, part);
        idx_t *loc_op_perm(new idx_t[n_rows]);
        idx_t *loc_po_perm(new idx_t[n_rows]);
        for (idx_t k(0); k<n_rows; k++) {
            loc_op_perm[k] = _g_op_perm[k];
        }
        for (idx_t k(0); k<n_rows; k++) {
            loc_po_perm[k] = _g_po_perm[k];
        }
        METIS_NodeND(&n_rows, xadj, adjncy, vwgt, options, loc_op_perm, loc_po_perm);
        for (idx_t k(0); k<n_rows; k++) {
            _g_op_perm[k] = loc_op_perm[k];
        }
        for (idx_t k(0); k<n_rows; k++) {
            _g_po_perm[k] = loc_po_perm[k];
        }
        delete [] loc_op_perm;
        delete [] loc_po_perm;
        delete [] vwgt;
        delete [] adjncy;
        delete [] xadj;
//		// create and init local permutations
//		unsigned *l_op_perm(new unsigned[size]);
//		unsigned *l_po_perm(new unsigned[size]);
//		for (unsigned i = 0; i < size; ++i)
//			l_op_perm[i] = l_po_perm[i] = i;
//
//		unsigned isep1, isep2;
//		updatePerm(part, isep1, isep2, l_op_perm, l_po_perm);
//		delete[] part;
//
//		// update global permutation
//		unsigned *t_op_perm = new unsigned[size];
//		for (unsigned k = 0; k < size; ++k)
//			t_op_perm[k] = _g_op_perm[_beg + l_op_perm[k]];
//
//		for (unsigned k = _beg; k < _end; ++k) {
//			_g_op_perm[k] = t_op_perm[k - _beg];
//			_g_po_perm[_g_op_perm[k]] = k;
//		}
//		delete[] t_op_perm;
//
//		// next recursion step
//		if ((isep1 >= bmin) && (isep2 - isep1 >= bmin)) {
//			// construct adj matrices for [0, isep1), [isep1,isep2), [isep2, _end)
//			AdjMat *l_adj0(_l_adj_mat->getMat(0, isep1, l_op_perm, l_po_perm));
//			AdjMat *l_adj1(_l_adj_mat->getMat(isep1, isep2, l_op_perm, l_po_perm));
//			AdjMat *l_adj2(_l_adj_mat->getMat(isep2, size, l_op_perm, l_po_perm));
//
//			delete[] l_op_perm;
//			delete[] l_po_perm;
//			delete _l_adj_mat;
//			_l_adj_mat = NULL;
//
//			_n_sons = 3;
//			_sons = new ClusterBase*[_n_sons];
//
//			isep1 += _beg;
//			isep2 += _beg;
//
//			// constructing child nodes for index cluster tree
//			_sons[0] = new Cluster(this, _beg, isep1, _g_op_perm, _g_po_perm, _g_adj_mat, l_adj0);
//			_sons[1] = new Cluster(this, isep1, isep2, _g_op_perm, _g_po_perm, _g_adj_mat, l_adj1);
//			_sons[2] = new Separator(this, isep2, _end, _g_op_perm,	_g_po_perm, _g_adj_mat, l_adj2);
//
//			dynamic_cast<Cluster*>(_sons[0])->subdivide(bmin);
//			dynamic_cast<Cluster*>(_sons[1])->subdivide(bmin);
//
//		} else {
//			delete _l_adj_mat;
//			_l_adj_mat = NULL;
//		} // end if next recursion step
    } // end if ( connected && size () > bmin )

}
//- Does prevention of 0 cell domains and calls ptscotch.
Foam::label Foam::ptscotchDecomp::decomposeZeroDomains
(
    const fileName& meshPath,
    const List<int>& initadjncy,
    const List<int>& initxadj,
    const scalarField& initcWeights,

    List<int>& finalDecomp
) const
{
    globalIndex globalCells(initxadj.size()-1);

    bool hasZeroDomain = false;
    for (label procI = 0; procI < Pstream::nProcs(); procI++)
    {
        if (globalCells.localSize(procI) == 0)
        {
            hasZeroDomain = true;
            break;
        }
    }

    if (!hasZeroDomain)
    {
        return decompose
        (
            meshPath,
            initadjncy,
            initxadj,
            initcWeights,
            finalDecomp
        );
    }


    if (debug)
    {
        Info<< "ptscotchDecomp : have graphs with locally 0 cells."
            << " trickling down." << endl;
    }

    // Make sure every domain has at least one cell
    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    // (scotch does not like zero sized domains)
    // Trickle cells from processors that have them up to those that
    // don't.


    // Number of cells to send to the next processor
    // (is same as number of cells next processor has to receive)
    List<int> nSendCells(Pstream::nProcs(), 0);

    for (label procI = nSendCells.size()-1; procI >=1; procI--)
    {
        label nLocalCells = globalCells.localSize(procI);
        if (nLocalCells-nSendCells[procI] < 1)
        {
            nSendCells[procI-1] = nSendCells[procI]-nLocalCells+1;
        }
    }

    // First receive (so increasing the sizes of all arrays)

    Field<int> xadj(initxadj);
    Field<int> adjncy(initadjncy);
    scalarField cWeights(initcWeights);

    if (Pstream::myProcNo() >= 1 && nSendCells[Pstream::myProcNo()-1] > 0)
    {
        // Receive cells from previous processor
        IPstream fromPrevProc(Pstream::blocking, Pstream::myProcNo()-1);

        Field<int> prevXadj(fromPrevProc);
        Field<int> prevAdjncy(fromPrevProc);
        scalarField prevCellWeights(fromPrevProc);

        if (prevXadj.size() != nSendCells[Pstream::myProcNo()-1])
        {
            FatalErrorIn("ptscotchDecomp::decompose(..)")
                << "Expected from processor " << Pstream::myProcNo()-1
                << " connectivity for " << nSendCells[Pstream::myProcNo()-1]
                << " nCells but only received " << prevXadj.size()
                << abort(FatalError);
        }

        // Insert adjncy
        prepend(prevAdjncy, adjncy);
        // Adapt offsets and prepend xadj
        xadj += prevAdjncy.size();
        prepend(prevXadj, xadj);
        // Weights
        prepend(prevCellWeights, cWeights);
    }


    // Send to my next processor

    if (nSendCells[Pstream::myProcNo()] > 0)
    {
        // Send cells to next processor
        OPstream toNextProc(Pstream::blocking, Pstream::myProcNo()+1);

        int nCells = nSendCells[Pstream::myProcNo()];
        int startCell = xadj.size()-1 - nCells;
        int startFace = xadj[startCell];
        int nFaces = adjncy.size()-startFace;

        // Send for all cell data: last nCells elements
        // Send for all face data: last nFaces elements
        toNextProc
            << Field<int>::subField(xadj, nCells, startCell)-startFace
            << Field<int>::subField(adjncy, nFaces, startFace)
            <<
            (
                cWeights.size()
              ? static_cast<const scalarField&>
                (
                    scalarField::subField(cWeights, nCells, startCell)
                )
              : scalarField(0)
            );

        // Remove data that has been sent
        if (cWeights.size())
        {
            cWeights.setSize(cWeights.size()-nCells);
        }
        adjncy.setSize(adjncy.size()-nFaces);
        xadj.setSize(xadj.size() - nCells);
    }


    // Do decomposition as normal. Sets finalDecomp.
    label result = decompose(meshPath, adjncy, xadj, cWeights, finalDecomp);


    if (debug)
    {
        Info<< "ptscotchDecomp : have graphs with locally 0 cells."
            << " trickling up." << endl;
    }


    // If we sent cells across make sure we undo it
    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    // Receive back from next processor if I sent something
    if (nSendCells[Pstream::myProcNo()] > 0)
    {
        IPstream fromNextProc(Pstream::blocking, Pstream::myProcNo()+1);

        List<int> nextFinalDecomp(fromNextProc);

        if (nextFinalDecomp.size() != nSendCells[Pstream::myProcNo()])
        {
            FatalErrorIn("parMetisDecomp::decompose(..)")
                << "Expected from processor " << Pstream::myProcNo()+1
                << " decomposition for " << nSendCells[Pstream::myProcNo()]
                << " nCells but only received " << nextFinalDecomp.size()
                << abort(FatalError);
        }

        append(nextFinalDecomp, finalDecomp);
    }

    // Send back to previous processor.
    if (Pstream::myProcNo() >= 1 && nSendCells[Pstream::myProcNo()-1] > 0)
    {
        OPstream toPrevProc(Pstream::blocking, Pstream::myProcNo()-1);

        int nToPrevious = nSendCells[Pstream::myProcNo()-1];

        toPrevProc <<
            SubList<int>
            (
                finalDecomp,
                nToPrevious,
                finalDecomp.size()-nToPrevious
            );

        // Remove locally what has been sent
        finalDecomp.setSize(finalDecomp.size()-nToPrevious);
    }
    return result;
}