コード例 #1
0
/*
   ----------------------------------------------------------
   purpose -- to construct the map from fronts to processors,
      and compute operations for each processor.

   maptype -- type of map for parallel factorization
      maptype = 1 --> wrap map
      maptype = 2 --> balanced map
      maptype = 3 --> subtree-subset map
      maptype = 4 --> domain decomposition map
   cutoff -- used when maptype = 4 as upper bound on
      relative domain size

   return value --
      1 -- success
     -1 -- bridge is NULL
     -2 -- front tree is NULL

   created -- 98sep25, cca
   ----------------------------------------------------------
*/
int
BridgeMPI_factorSetup (
    BridgeMPI   *bridge,
    int         maptype,
    double      cutoff
) {
    double   t1, t2 ;
    DV       *cumopsDV ;
    ETree    *frontETree ;
    FILE     *msgFile ;
    int      msglvl, nproc ;
    /*
       ---------------
       check the input
       ---------------
    */
    MARKTIME(t1) ;
    if ( bridge == NULL ) {
        fprintf(stderr, "\n error in BridgeMPI_factorSetup()"
                "\n bridge is NULL") ;
        return(-1) ;
    }
    if ( (frontETree = bridge->frontETree) == NULL ) {
        fprintf(stderr, "\n error in BridgeMPI_factorSetup()"
                "\n frontETree is NULL") ;
        return(-2) ;
    }
    nproc   = bridge->nproc   ;
    msglvl  = bridge->msglvl  ;
    msgFile = bridge->msgFile ;
    /*
       -------------------------------------------
       allocate and initialize the cumopsDV object
       -------------------------------------------
    */
    if ( (cumopsDV = bridge->cumopsDV) == NULL ) {
        cumopsDV = bridge->cumopsDV = DV_new() ;
    }
    DV_setSize(cumopsDV, nproc) ;
    DV_zero(cumopsDV) ;
    /*
       ----------------------------
       create the owners map object
       ----------------------------
    */
    switch ( maptype ) {
    case 1 :
        bridge->ownersIV = ETree_wrapMap(frontETree, bridge->type,
                                         bridge->symmetryflag, cumopsDV) ;
        break ;
    case 2 :
        bridge->ownersIV = ETree_balancedMap(frontETree, bridge->type,
                                             bridge->symmetryflag, cumopsDV) ;
        break ;
    case 3 :
        bridge->ownersIV = ETree_subtreeSubsetMap(frontETree, bridge->type,
                           bridge->symmetryflag, cumopsDV) ;
        break ;
    case 4 :
        bridge->ownersIV = ETree_ddMap(frontETree, bridge->type,
                                       bridge->symmetryflag, cumopsDV, cutoff) ;
        break ;
    default :
        bridge->ownersIV = ETree_ddMap(frontETree, bridge->type,
                                       bridge->symmetryflag, cumopsDV, 1./(2*nproc)) ;
        break ;
    }
    MARKTIME(t2) ;
    bridge->cpus[7] = t2 - t1 ;
    if ( msglvl > 1 ) {
        fprintf(msgFile, "\n\n parallel factor setup") ;
        fprintf(msgFile, "\n type = %d, symmetryflag = %d",
                bridge->type, bridge->symmetryflag) ;
        fprintf(msgFile, "\n total factor operations = %.0f",
                DV_sum(cumopsDV)) ;
        fprintf(msgFile,
                "\n upper bound on speedup due to load balance = %.2f",
                DV_max(cumopsDV)/DV_sum(cumopsDV)) ;
        fprintf(msgFile, "\n operations distributions over threads") ;
        DV_writeForHumanEye(cumopsDV, msgFile) ;
        fflush(msgFile) ;
    }
    if ( msglvl > 2 ) {
        fprintf(msgFile, "\n\n owners map IV object") ;
        IV_writeForHumanEye(bridge->ownersIV, msgFile) ;
        fflush(msgFile) ;
    }
    /*
       ----------------------------
       create the vertex map object
       ----------------------------
    */
    bridge->vtxmapIV = IV_new() ;
    IV_init(bridge->vtxmapIV, bridge->neqns, NULL) ;
    IVgather(bridge->neqns, IV_entries(bridge->vtxmapIV),
             IV_entries(bridge->ownersIV),
             ETree_vtxToFront(bridge->frontETree)) ;
    if ( msglvl > 2 ) {
        fprintf(msgFile, "\n\n vertex map IV object") ;
        IV_writeForHumanEye(bridge->vtxmapIV, msgFile) ;
        fflush(msgFile) ;
    }

    return(1) ;
}
コード例 #2
0
ファイル: colmapMPI.c プロジェクト: damiannz/spooles
/*
   -------------------------------------------------------------
   purpose -- after pivoting for a nonsymmetric factorization,
              some delayed columns may belong to a process other
              than its original owner. this method returns an
              IV object that maps columns to owning processes.

   created -- 98may22, cca
   -------------------------------------------------------------
*/
IV *
FrontMtx_MPI_colmapIV (
   FrontMtx   *frontmtx,
   IV         *frontOwnersIV,
   int        msglvl,
   FILE       *msgFile,
   MPI_Comm   comm
) {
int   buffersize, ii, iproc, J, myid, nDJ, neqns, nfront, nproc, 
      ncolJ, nToSend, v ;
int   *buffer, *counts, *frontOwners, *inbuffer, *outbuffer, 
      *colindJ, *colmap, *vtxToFront ;
IV    *colmapIV ;
/*
   -------------------------------------------
   get the process id and number of processors
   -------------------------------------------
*/
MPI_Comm_rank(comm, &myid) ;
MPI_Comm_size(comm, &nproc) ;
neqns      = frontmtx->neqns ;
vtxToFront = ETree_vtxToFront(frontmtx->frontETree) ;
IV_sizeAndEntries(frontOwnersIV, &nfront, &frontOwners) ;
if ( msglvl > 2 ) {
   fprintf(msgFile, "\n\n inside FrontMtx_MPI_colmapIV()"
           "\n myid = %d, nproc = %d, nfront = %d, neqns = %d",
           myid, nproc, nfront, neqns) ;
   fflush(msgFile) ;
}
/*
   ----------------------------------------------------------
   loop through the owned fronts and store each column in an 
   owned front that was originally owned by another processor
   ----------------------------------------------------------
*/
outbuffer = IVinit(neqns, -1) ;
for ( J = nToSend = 0 ; J < nfront ; J++ ) {
   if (  frontOwners[J] == myid 
      && (nDJ = FrontMtx_frontSize(frontmtx, J)) > 0 ) {
      FrontMtx_columnIndices(frontmtx, J, &ncolJ, &colindJ) ;
      if ( msglvl > 2 ) {
         fprintf(msgFile, "\n front %d owned, nDJ = %d, ncolJ = %d",
                 J, nDJ, ncolJ) ;
         fflush(msgFile) ;
      }
      for ( ii = 0 ; ii < nDJ ; ii++ ) {
         v = colindJ[ii] ;
         if ( frontOwners[vtxToFront[v]] != myid ) {
            if ( msglvl > 2 ) {
               fprintf(msgFile, "\n column %d originally owned by %d",
                       v, frontOwners[vtxToFront[v]]) ;
               fflush(msgFile) ;
            }
            outbuffer[nToSend++] = v ;
         }
      }
   }
}
IVqsortUp(nToSend, outbuffer) ;
if ( msglvl > 2 ) {
   fprintf(msgFile, "\n shifted vertices") ;
   IVfprintf(msgFile, nToSend, outbuffer) ;
   fflush(msgFile) ;
}
counts = IVinit(nproc, 0) ;
/*
   --------------------------------------------
   use an all-gather call to get the number of
   moved columns that are owned by each process
   --------------------------------------------
*/
MPI_Allgather((void *) &nToSend, 1, MPI_INT, counts, 1, MPI_INT, comm) ;
if ( msglvl > 2 ) {
   fprintf(msgFile, "\n after the all-gather operation, counts") ;
   IVfprintf(msgFile, nproc, counts) ;
   fflush(msgFile) ;
}
buffersize = IVmax(nproc, counts, &iproc) ;
inbuffer   = IVinit(buffersize, -1) ;
/*
   -----------------------------------
   initialize the column map IV object
   -----------------------------------
*/
colmapIV = IV_new() ;
IV_init(colmapIV, neqns, NULL) ;
colmap = IV_entries(colmapIV) ;
IVgather(neqns, colmap, frontOwners, vtxToFront) ;
/*
   --------------------------------------------------------------
   loop over the other processes, receive vector of moved columns
   --------------------------------------------------------------
*/
for ( iproc = 0 ; iproc < nproc ; iproc++ ) {
   if ( counts[iproc] > 0 ) {
      if ( iproc == myid ) {
/*
        -------------------------------------
        send buffer vector to other processes
        -------------------------------------
*/
         if ( msglvl > 2 ) {
            fprintf(msgFile, "\n sending outbuffer to all processes") ;
            IVfprintf(msgFile, nToSend, outbuffer) ;
            fflush(msgFile) ;
         }
         MPI_Bcast(outbuffer, nToSend, MPI_INT, iproc, comm) ;
         buffer = outbuffer ;
      } else {
/*
        -----------------------------------------
        receive the vector from the other process
        -----------------------------------------
*/
         MPI_Bcast(inbuffer, counts[iproc], MPI_INT, iproc, comm) ;
         if ( msglvl > 2 ) {
            fprintf(msgFile, "\n received inbuffer from process %d",
                    iproc) ;
            IVfprintf(msgFile, counts[iproc], inbuffer) ;
            fflush(msgFile) ;
         }
         buffer = inbuffer ;
      }
/*
      -------------------------
      set the column map values
      -------------------------
*/
      for ( ii = 0 ; ii < counts[iproc] ; ii++ ) {
         v = buffer[ii] ;
         colmap[v] = iproc ;
      }
   }
}
/*
   ------------------------
   free the working storage
   ------------------------
*/
IVfree(inbuffer)  ;
IVfree(outbuffer) ;
IVfree(counts)    ;

return(colmapIV) ; }
コード例 #3
0
ファイル: init.c プロジェクト: bialk/SPOOLES
/*
   --------------------------------------------------------------------
   purpose -- to fill submtx with a submatrix of the front matrix.
      the fronts that form the submatrix are found in frontidsIV.

      all information in submtx is local, front #'s are from 0 to
      one less than the number of fronts in the submatrix, equation
      #'s are from 0 to one less than the number of rows and columns
      in the submatrix. the global row and column ids for the submatrix
      are stored in rowsIV and colsIV on return.

   return values ---
      1 -- normal return
     -1 -- submtx is NULL
     -2 -- frontmtx is NULL
     -3 -- frontmtx is not in 2-D mode
     -4 -- frontidsIV is NULL
     -5 -- frontidsIV is invalid
     -6 -- rowsIV is NULL
     -7 -- colsIV is NULL
     -8 -- unable to create front tree
     -9 -- unable to create symbfacIVL
    -10 -- unable to create coladjIVL
    -11 -- unable to create rowadjIVL
    -12 -- unable to create upperblockIVL
    -13 -- unable to create lowerblockIVL

   created -- 98oct17, cca
   --------------------------------------------------------------------
*/
int
FrontMtx_initFromSubmatrix (
   FrontMtx   *submtx,
   FrontMtx   *frontmtx,
   IV         *frontidsIV,
   IV         *rowsIV,
   IV         *colsIV,
   int        msglvl,
   FILE       *msgFile
) {
ETree    *etreeSub ;
int      ii, J, Jsub, K, Ksub, ncol, nfront, nfrontSub, neqnSub, nJ,
         nrow, offset, rc, size, vSub ;
int      *bndwghts, *colind, *colmap, *cols, *frontSubIds, 
         *list, *nodwghts, *rowind, *rowmap, *rows ;
IV       *frontsizesIVsub, *vtxIV ;
IVL      *coladjIVLsub, *lowerblockIVLsub, *rowadjIVLsub, 
         *symbfacIVLsub, *upperblockIVLsub ;
SubMtx   *mtx ;
/*
   ---------------
   check the input
   ---------------
*/
if ( submtx == NULL ) {
   fprintf(stderr, "\n error in FrontMtx_initFromSubmatrix()"
           "\n submtx is NULL\n") ;
   return(-1) ;
}
if ( frontmtx == NULL ) {
   fprintf(stderr, "\n error in FrontMtx_initFromSubmatrix()"
           "\n frontmtx is NULL\n") ;
   return(-2) ;
}
if ( ! FRONTMTX_IS_2D_MODE(frontmtx) ) {
   fprintf(stderr, "\n error in FrontMtx_initFromSubmatrix()"
           "\n frontmtx mode is not 2D\n") ;
   return(-3) ;
}
if ( frontidsIV == NULL ) {
   fprintf(stderr, "\n error in FrontMtx_initFromSubmatrix()"
           "\n frontidsIV is NULL\n") ;
   return(-4) ;
}
nfront = FrontMtx_nfront(frontmtx) ;
IV_sizeAndEntries(frontidsIV, &nfrontSub, &frontSubIds) ;
if ( nfrontSub < 0 || nfrontSub > nfront ) {
   fprintf(stderr, "\n error in FrontMtx_initFromSubmatrix()"
           "\n invalid frontidsIV"
           "\n nfrontSub = %d, nfront %d\n", nfrontSub, nfront) ;
   return(-5) ;
}
for ( ii = 0 ; ii < nfrontSub ; ii++ ) {
   if ( (J = frontSubIds[ii]) < 0 || J >= nfront ) {
      fprintf(stderr, "\n error in FrontMtx_initFromSubmatrix()"
              "\n invalid frontidsIV"
              "\n frontSubIds[%d] = %d, nfront = %d\n",
              ii, J, nfront) ;
      return(-5) ;
   }
}
if ( rowsIV == NULL ) {
   fprintf(stderr, "\n error in FrontMtx_initFromSubmatrix()"
           "\n rowsIV is NULL\n") ;
   return(-6) ;
}
if ( colsIV == NULL ) {
   fprintf(stderr, "\n error in FrontMtx_initFromSubmatrix()"
           "\n colsIV is NULL\n") ;
   return(-7) ;
}
/*--------------------------------------------------------------------*/
/*
   -----------------------------------------------------
   clear the data for the submatrix and set the 
   scalar values (some inherited from the global matrix)
   -----------------------------------------------------
*/
FrontMtx_clearData(submtx) ;
submtx->nfront       = nfrontSub ;
submtx->type         = frontmtx->type ;
submtx->symmetryflag = frontmtx->symmetryflag ;
submtx->sparsityflag = frontmtx->sparsityflag ;
submtx->pivotingflag = frontmtx->pivotingflag ;
submtx->dataMode     = FRONTMTX_2D_MODE ;
/*
   ---------------------------------------------------------------
   initialize the front tree for the submatrix.

   note: on return, vtxIV is filled with the vertices originally
   in the submatrix, (pivoting may change this), needed to find
   symbolic factorization IVL object

   note: at return, the boundary weights are likely to be invalid,
   since we have no way of knowing what boundary indices for a
   front are really in the domain. this will be changed after we
   have the symbolic factorization.
   ---------------------------------------------------------------
*/
etreeSub = submtx->frontETree = ETree_new() ;
vtxIV = IV_new() ;
rc = ETree_initFromSubtree(etreeSub, frontidsIV, 
                           frontmtx->frontETree, vtxIV) ;
if ( rc != 1 ) {
   fprintf(stderr, "\n error in FrontMtx_initFromSubmatrix()"
         "\n unable to create submatrix's front ETree, rc = %d\n", rc) ;
   return(-8) ;
}
if ( msglvl > 4 ) {
   fprintf(msgFile, "\n\n submatrix ETree") ;
   ETree_writeForHumanEye(etreeSub, msgFile) ;
   fprintf(msgFile, "\n\n submatrix original equations") ;
   IV_writeForHumanEye(vtxIV, msgFile) ;
   fflush(msgFile) ;
}
/*
   ------------------------------------------------------
   set the # of equations (perhap temporarily if pivoting 
   has delayed some rows and columns), and the tree.
   ------------------------------------------------------
*/
submtx->neqns = neqnSub = IV_size(vtxIV) ;
submtx->tree  = etreeSub->tree ;
/*
   -----------------------------------------------------
   initialize the symbolic factorization for the subtree
   -----------------------------------------------------
*/
symbfacIVLsub = submtx->symbfacIVL = IVL_new() ;
rc = IVL_initFromSubIVL(symbfacIVLsub, frontmtx->symbfacIVL,
                        frontidsIV, vtxIV) ;
if ( rc != 1 ) {
   fprintf(stderr, "\n error in FrontMtx_initFromSubmatrix()"
         "\n unable to create submatrix's symbfac, rc = %d\n", rc) ;
   return(-9) ;
}
if ( msglvl > 4 ) {
   fprintf(msgFile, "\n\n submatrix symbolic factorizatio") ;
   IVL_writeForHumanEye(symbfacIVLsub, msgFile) ;
   fflush(msgFile) ;
}
/*
   ---------------------------------------------
   adjust the boundary weights of the front tree
   ---------------------------------------------
*/
nodwghts = ETree_nodwghts(etreeSub) ;
bndwghts = ETree_bndwghts(etreeSub) ;
for ( J = 0 ; J < nfrontSub ; J++ ) {
   IVL_listAndSize(symbfacIVLsub, J, &size, &list) ;
   bndwghts[J] = size - nodwghts[J] ;
}
if ( msglvl > 4 ) {
   fprintf(msgFile, "\n\n submatrix ETree after bndweight adjustment") ;
   ETree_writeForHumanEye(etreeSub, msgFile) ;
   fflush(msgFile) ;
}
/*
   -------------------------------------
   set the front sizes for the submatrix
   -------------------------------------
*/
frontsizesIVsub = submtx->frontsizesIV = IV_new() ;
IV_init(frontsizesIVsub, nfrontSub, NULL) ;
IVgather(nfrontSub, IV_entries(frontsizesIVsub), 
         IV_entries(frontmtx->frontsizesIV),
         IV_entries(frontidsIV)) ;
neqnSub = submtx->neqns = IV_sum(frontsizesIVsub) ;
if ( msglvl > 4 ) {
   fprintf(msgFile, "\n\n %d equations in submatrix", neqnSub) ;
   fprintf(msgFile, "\n\n front sizes for submatrix") ;
   IV_writeForHumanEye(frontsizesIVsub, msgFile) ;
   fflush(msgFile) ;
}
/*
   -------------------------------------------------------------------
   fill rowsIV and colsIV with the row and column ids of the submatrix
   -------------------------------------------------------------------
*/
IV_setSize(rowsIV, neqnSub) ;
IV_setSize(colsIV, neqnSub) ;
rows = IV_entries(rowsIV) ;
cols = IV_entries(colsIV) ;
for ( Jsub = offset = 0 ; Jsub < nfrontSub ; Jsub++ ) {
   if ( (nJ = FrontMtx_frontSize(submtx, Jsub)) > 0 ) {
      J = frontSubIds[Jsub] ;
      FrontMtx_columnIndices(frontmtx, J, &size, &list) ;
      IVcopy(nJ, cols + offset, list) ;
      FrontMtx_rowIndices(frontmtx, J, &size, &list) ;
      IVcopy(nJ, rows + offset, list) ;
      offset += nJ ;
   }
}
if ( msglvl > 4 ) {
   fprintf(msgFile, "\n\n row ids for submatrix") ;
   IV_writeForHumanEye(rowsIV, msgFile) ;
   fprintf(msgFile, "\n\n column ids for submatrix") ;
   IV_writeForHumanEye(colsIV, msgFile) ;
   fflush(msgFile) ;
}
/*
   ----------------------------------
   get the row and column adjacencies
   ----------------------------------
*/
if ( FRONTMTX_IS_PIVOTING(frontmtx) ) {
   submtx->neqns = neqnSub ;
   coladjIVLsub  = submtx->coladjIVL = IVL_new() ;
   rc = IVL_initFromSubIVL(coladjIVLsub, frontmtx->coladjIVL,
                           frontidsIV, colsIV) ;
   if ( rc != 1 ) {
      fprintf(stderr, "\n error in FrontMtx_initFromSubmatrix()"
           "\n unable to create submatrix's coladjIVL, rc = %d\n", rc) ;
      return(-10) ;
   }
   if ( msglvl > 4 ) {
      fprintf(msgFile, "\n\n submatrix col adjacency") ;
      IVL_writeForHumanEye(coladjIVLsub, msgFile) ;
      fflush(msgFile) ;
   }
   if ( FRONTMTX_IS_NONSYMMETRIC(frontmtx) ) {
      rowadjIVLsub = submtx->rowadjIVL = IVL_new() ;
      rc = IVL_initFromSubIVL(rowadjIVLsub, frontmtx->rowadjIVL,
                              frontidsIV, rowsIV) ;
      if ( rc != 1 ) {
         fprintf(stderr, "\n error in FrontMtx_initFromSubmatrix()"
           "\n unable to create submatrix's rowadjIVL, rc = %d\n", rc) ;
         return(-11) ;
      }
      if ( msglvl > 4 ) {
         fprintf(msgFile, "\n\n submatrix row adjacency") ;
         IVL_writeForHumanEye(rowadjIVLsub, msgFile) ;
         fflush(msgFile) ;
      }
   }
}
IV_free(vtxIV) ;
/*
   ----------------------------------------------
   get the rowmap[] and colmap[] vectors,
   needed to translate indices in the submatrices
   ----------------------------------------------
*/
colmap = IVinit(frontmtx->neqns, -1) ;
for ( ii = 0 ; ii < neqnSub ; ii++ ) {
   colmap[cols[ii]] = ii ;
}
if ( FRONTMTX_IS_NONSYMMETRIC(frontmtx) ) {
   rowmap = IVinit(frontmtx->neqns, -1) ;
   for ( ii = 0 ; ii < neqnSub ; ii++ ) {
      rowmap[rows[ii]] = ii ;
   }
} else {
   rowmap = colmap ;
}
/*
   -----------------------------------------------------------
   get the upper and lower block IVL objects for the submatrix
   -----------------------------------------------------------
*/
upperblockIVLsub = submtx->upperblockIVL = IVL_new() ;
rc = IVL_initFromSubIVL(upperblockIVLsub, frontmtx->upperblockIVL,
                        frontidsIV, frontidsIV) ;
if ( rc != 1 ) {
   fprintf(stderr, "\n error in FrontMtx_initFromSubmatrix()"
        "\n unable to create upperblockIVL, rc = %d\n", rc) ;
   return(-12) ;
}
if ( msglvl > 4 ) {
   fprintf(msgFile, "\n\n upper block adjacency IVL object") ;
   IVL_writeForHumanEye(upperblockIVLsub, msgFile) ;
   fflush(msgFile) ;
}
if ( FRONTMTX_IS_NONSYMMETRIC(frontmtx) ) {
   lowerblockIVLsub = submtx->lowerblockIVL = IVL_new() ;
   rc = IVL_initFromSubIVL(lowerblockIVLsub, frontmtx->lowerblockIVL,
                           frontidsIV, frontidsIV) ;
   if ( rc != 1 ) {
      fprintf(stderr, "\n error in FrontMtx_initFromSubmatrix()"
           "\n unable to create lowerblockIVL, rc = %d\n", rc) ;
      return(-13) ;
   }
   if ( msglvl > 4 ) {
      fprintf(msgFile, "\n\n lower block adjacency IVL object") ;
      IVL_writeForHumanEye(lowerblockIVLsub, msgFile) ;
      fflush(msgFile) ;
   }
}
/*
   ----------------------------------------------------------------
   allocate the vector and hash table(s) for the factor submatrices
   ----------------------------------------------------------------
*/
ALLOCATE(submtx->p_mtxDJJ, struct _SubMtx *, nfrontSub) ;
for ( J = 0 ; J < nfrontSub ; J++ ) {
   submtx->p_mtxDJJ[J] = NULL ;
}
submtx->upperhash = I2Ohash_new() ;
I2Ohash_init(submtx->upperhash, nfrontSub, nfrontSub, nfrontSub) ;
if ( FRONTMTX_IS_NONSYMMETRIC(frontmtx) ) {
   submtx->lowerhash = I2Ohash_new() ;
   I2Ohash_init(submtx->lowerhash, nfrontSub, nfrontSub, nfrontSub) ;
}
/*
   -----------------------------------------------------------------
   remove the diagonal submatrices from the factor matrix
   and insert into the submatrix object. note: front row and column
   ids must be changed to their local values, and the row and column
   indices must be mapped to local indices.
   -----------------------------------------------------------------
*/
for ( Jsub = 0 ; Jsub < nfrontSub ; Jsub++ ) {
   J = frontSubIds[Jsub] ;
   if ( (mtx = frontmtx->p_mtxDJJ[J]) != NULL ) {
      SubMtx_setIds(mtx, Jsub, Jsub) ;
      SubMtx_columnIndices(mtx, &ncol, &colind) ;
      IVgather(ncol, colind, colmap, colind) ;
      SubMtx_rowIndices(mtx, &nrow, &rowind) ;
      IVgather(nrow, rowind, rowmap, rowind) ;
      submtx->p_mtxDJJ[Jsub] = mtx ;
      frontmtx->p_mtxDJJ[J]  = NULL ;
      submtx->nentD += mtx->nent ;
   }
}
/*
   ----------------------------------------------------------------
   remove the upper triangular submatrices from the factor matrix
   and insert into the submatrix object. note: front row and column
   ids must be changed to their local values. if the matrix is on
   the diagonal, i.e., U(J,J), its row and column indices must be 
   mapped to local indices.
   ----------------------------------------------------------------
*/
for ( Jsub = 0 ; Jsub < nfrontSub ; Jsub++ ) {
   J = frontSubIds[Jsub] ;
   FrontMtx_upperAdjFronts(submtx, Jsub, &size, &list) ;
   for ( ii = 0 ; ii < size ; ii++ ) {
      Ksub = list[ii] ;
      K = frontSubIds[Ksub] ;
      if ( 1 == I2Ohash_remove(frontmtx->upperhash, 
                               J, K, (void *) &mtx) ) {
         SubMtx_setIds(mtx, Jsub, Ksub) ;
         if ( K == J ) {
            SubMtx_columnIndices(mtx, &ncol, &colind) ;
            IVgather(ncol, colind, colmap, colind) ;
            SubMtx_rowIndices(mtx, &nrow, &rowind) ;
            IVgather(nrow, rowind, rowmap, rowind) ;
         }
         I2Ohash_insert(submtx->upperhash, Jsub, Ksub, (void *) mtx) ;
         submtx->nentU += mtx->nent ;
      }
   }
}
if ( FRONTMTX_IS_NONSYMMETRIC(frontmtx) ) {
/*
   ----------------------------------------------------------------
   remove the lower triangular submatrices from the factor matrix
   and insert into the submatrix object. note: front row and column
   ids must be changed to their local values. if the matrix is on
   the diagonal, i.e., L(J,J), its row and column indices must be 
   mapped to local indices.
   ----------------------------------------------------------------
*/
   for ( Jsub = 0 ; Jsub < nfrontSub ; Jsub++ ) {
      J = frontSubIds[Jsub] ;
      FrontMtx_lowerAdjFronts(submtx, Jsub, &size, &list) ;
      for ( ii = 0 ; ii < size ; ii++ ) {
         Ksub = list[ii] ;
         K = frontSubIds[Ksub] ;
         if ( 1 == I2Ohash_remove(frontmtx->lowerhash, 
                                  K, J, (void *) &mtx) ) {
            SubMtx_setIds(mtx, Ksub, Jsub) ;
            if ( K == J ) {
               SubMtx_columnIndices(mtx, &ncol, &colind) ;
               IVgather(ncol, colind, colmap, colind) ;
               SubMtx_rowIndices(mtx, &nrow, &rowind) ;
               IVgather(nrow, rowind, rowmap, rowind) ;
            }
            I2Ohash_insert(submtx->lowerhash, Ksub, Jsub, (void *) mtx);
            submtx->nentL += mtx->nent ;
         }
      }
   }
}
/*
   ------------------------
   free the working storage
   ------------------------
*/
IVfree(colmap) ;
if ( FRONTMTX_IS_NONSYMMETRIC(frontmtx) ) {
   IVfree(rowmap) ;
}
return(1) ; }