//--------------------------------------------------------------------------
int Epetra_FEVbrMatrix::SetupForNonlocalSubmits(int BlockRow,
int NumBlockEntries,
int * BlockIndices,
bool indicesAreLocal,
Epetra_CombineMode SubmitMode)
{
(void)indicesAreLocal;
if (ignoreNonLocalEntries_) {
curRowOffset_ = 0;
return(0);
}
int insertPoint = -1;
//find offset of this row in our list of nonlocal rows
int rowoffset = Epetra_Util_binary_search(BlockRow, nonlocalBlockRows_,
numNonlocalBlockRows_, insertPoint);
//if this row is not already present, insert it
if (rowoffset < 0) {
EPETRA_CHK_ERR( InsertNonlocalRow(BlockRow, insertPoint, NumBlockEntries) );
rowoffset = insertPoint;
}
//now insert each incoming block-column-index in this list of column-indices,
//maintaining sortedness.
for(int i=0; i<NumBlockEntries; ++i) {
int col = BlockIndices[i];
int coloffset = Epetra_Util_binary_search(col, nonlocalBlockCols_[rowoffset],
nonlocalBlockRowLengths_[rowoffset],
insertPoint);
if (coloffset < 0) {
int tmp1 = nonlocalBlockRowLengths_[rowoffset];
int tmp2 = nonlocalBlockRowAllocLengths_[rowoffset];
EPETRA_CHK_ERR( Epetra_Util_insert(col, insertPoint,
nonlocalBlockCols_[rowoffset],
nonlocalBlockRowLengths_[rowoffset],
nonlocalBlockRowAllocLengths_[rowoffset]));
EPETRA_CHK_ERR( Epetra_Util_insert((Epetra_SerialDenseMatrix*)NULL,
insertPoint,
nonlocalCoefs_[rowoffset],
tmp1, tmp2) );
}
}
curRowOffset_ = rowoffset;
curColOffset_ = 0;
curNumCols_ = NumBlockEntries;
curCols_ = new int[NumBlockEntries];
for(int j=0; j<NumBlockEntries; ++j) {
curCols_[j] = BlockIndices[j];
}
curMode_ = SubmitMode;
return(0);
}
int EpetraVector<T>::inputNonlocalValues(int GID,
int numValues,
const double * values,
bool accumulate)
{
int insertPoint = -1;
//find offset of GID in nonlocalIDs_
int offset = Epetra_Util_binary_search(GID, nonlocalIDs_, numNonlocalIDs_,
insertPoint);
if (offset >= 0) {
//if offset >= 0
// put value in nonlocalCoefs_[offset][0]
if (numValues != nonlocalElementSize_[offset]) {
libMesh::err << "Epetra_FEVector ERROR: block-size for GID " << GID << " is "
<< numValues<<" which doesn't match previously set block-size of "
<< nonlocalElementSize_[offset] << std::endl;
return(-1);
}
if (accumulate) {
for(int j=0; j<numValues; ++j) {
nonlocalCoefs_[offset][j] += values[j];
}
}
else {
for(int j=0; j<numValues; ++j) {
nonlocalCoefs_[offset][j] = values[j];
}
}
}
else {
//else
// insert GID in nonlocalIDs_
// insert numValues in nonlocalElementSize_
// insert values in nonlocalCoefs_
int tmp1 = numNonlocalIDs_;
int tmp2 = allocatedNonlocalLength_;
int tmp3 = allocatedNonlocalLength_;
EPETRA_CHK_ERR( Epetra_Util_insert(GID, insertPoint, nonlocalIDs_,
tmp1, tmp2) );
--tmp1;
EPETRA_CHK_ERR( Epetra_Util_insert(numValues, insertPoint, nonlocalElementSize_,
tmp1, tmp3) );
double * newvalues = new double[numValues];
for(int j=0; j<numValues; ++j) {
newvalues[j] = values[j];
}
EPETRA_CHK_ERR( Epetra_Util_insert(newvalues, insertPoint, nonlocalCoefs_,
numNonlocalIDs_, allocatedNonlocalLength_) );
}
return(0);
}
double* matrix_data::coefs(int row, int col)
{
int insertPoint = -1;
int row_idx = Epetra_Util_binary_search(row, rows_, numrows_,
insertPoint);
if (row_idx < 0) {
cerr << "ERROR, row " << row
<< " not found in matrix_data"<<endl;
return 0;
}
int col_idx = Epetra_Util_binary_search(col, colindices_[row_idx],
rowlengths_[row_idx], insertPoint);
if (col_idx < 0) {
cerr << "ERROR, col " << col
<< " not found in matrix_data"<<endl;
return 0;
}
int dim = blocksize_*blocksize_;
return( &(coefs_[row_idx][col_idx*dim]) );
}
int EpetraVector<T>::inputNonlocalValue(int GID, double value, bool accumulate)
{
int insertPoint = -1;
//find offset of GID in nonlocalIDs_
int offset = Epetra_Util_binary_search(GID, nonlocalIDs_, numNonlocalIDs_,
insertPoint);
if (offset >= 0) {
//if offset >= 0
// put value in nonlocalCoefs_[offset][0]
if (accumulate) {
nonlocalCoefs_[offset][0] += value;
}
else {
nonlocalCoefs_[offset][0] = value;
}
}
else {
//else
// insert GID in nonlocalIDs_
// insert 1 in nonlocalElementSize_
// insert value in nonlocalCoefs_
int tmp1 = numNonlocalIDs_;
int tmp2 = allocatedNonlocalLength_;
int tmp3 = allocatedNonlocalLength_;
EPETRA_CHK_ERR( Epetra_Util_insert(GID, insertPoint, nonlocalIDs_,
tmp1, tmp2) );
--tmp1;
EPETRA_CHK_ERR( Epetra_Util_insert(1, insertPoint, nonlocalElementSize_,
tmp1, tmp3) );
double * values = new double[1];
values[0] = value;
EPETRA_CHK_ERR( Epetra_Util_insert(values, insertPoint, nonlocalCoefs_,
numNonlocalIDs_, allocatedNonlocalLength_) );
}
return(0);
}
//----------------------------------------------------------------------------
int Epetra_FEVbrMatrix::InputNonlocalBlockEntry(double *values, int LDA,
int NumRows, int NumCols)
{
if (curRowOffset_ < 0) {
return(-1);
}
int insertPoint;
int col = curCols_[curColOffset_++];
int coloffset =
Epetra_Util_binary_search(col, nonlocalBlockCols_[curRowOffset_],
nonlocalBlockRowLengths_[curRowOffset_],
insertPoint);
if (coloffset < 0) return(-1);
Epetra_SerialDenseMatrix*& subblock = nonlocalCoefs_[curRowOffset_][coloffset];
if (subblock == NULL) {
//For the construction of the serial dense matrix here, we choose Copy mode
//in case the user deletes or reuses the Values array after this method is
//called.
subblock = new Epetra_SerialDenseMatrix(Copy, values, LDA, NumRows, NumCols);
if (subblock == NULL) {
return(-1);
}
}
else {
int nrows = subblock->M();
int ncols = subblock->N();
if (nrows != NumRows || ncols != NumCols) {
return(-1);
}
int Target_LDA = subblock->LDA();
int Source_LDA = LDA;
double* tptr = subblock->A();
double* sptr = values;
if (curMode_ == Add) {
for(int j=0; j<NumCols; ++j) {
for(int i=0; i<NumRows; ++i) {
tptr[i] += sptr[i];
}
tptr += Target_LDA;
sptr += Source_LDA;
}
}
else {
for(int j=0; j<NumCols; ++j) {
for(int i=0; i<NumRows; ++i) {
tptr[i] = sptr[i];
}
tptr += Target_LDA;
sptr += Source_LDA;
}
}
}
return(0);
}
int Epetra_FEVbrMatrix::GlobalAssemble(bool callFillComplete)
{
if(Map().Comm().NumProc() < 2 || ignoreNonLocalEntries_) {
if(callFillComplete) {
EPETRA_CHK_ERR(FillComplete());
}
return(0);
}
int i;
//In this method we need to gather all the non-local (overlapping) data
//that's been input on each processor, into the
//non-overlapping distribution defined by the map that 'this' matrix was
//constructed with.
//Need to build a map that describes our nonlocal data.
//First, create a list of the sizes (point-rows per block-row) of the
//nonlocal rows we're holding.
int* pointRowsPerNonlocalBlockRow = numNonlocalBlockRows_>0 ?
new int[numNonlocalBlockRows_] : NULL;
for(i=0; i<numNonlocalBlockRows_; ++i) {
pointRowsPerNonlocalBlockRow[i] = nonlocalCoefs_[i][0]->M();
}
//We'll use the arbitrary distribution constructor of BlockMap.
Epetra_BlockMap sourceMap(-1, numNonlocalBlockRows_, nonlocalBlockRows_, // CJ TODO FIXME long long
pointRowsPerNonlocalBlockRow,
RowMap().IndexBase(), RowMap().Comm());
delete [] pointRowsPerNonlocalBlockRow;
//If sourceMap has global size 0, then no nonlocal data exists and we can
//skip most of this function.
if(sourceMap.NumGlobalElements64() < 1) {
if(callFillComplete) {
EPETRA_CHK_ERR(FillComplete());
}
return(0);
}
//We also need to build a column-map, containing the columns in our
//nonlocal data. To do that, create a list of all column-indices that
//occur in our nonlocal rows.
int numCols = 0, allocLen = 0;
int* cols = NULL;
int* pointColsPerBlockCol = NULL;
int ptColAllocLen = 0;
int insertPoint = -1;
for(i=0; i<numNonlocalBlockRows_; ++i) {
for(int j=0; j<nonlocalBlockRowLengths_[i]; ++j) {
int col = nonlocalBlockCols_[i][j];
int offset = Epetra_Util_binary_search(col, cols, numCols, insertPoint);
if (offset < 0) {
EPETRA_CHK_ERR( Epetra_Util_insert(col, insertPoint, cols,
numCols, allocLen) );
int tmpNumCols = numCols-1;
EPETRA_CHK_ERR( Epetra_Util_insert(nonlocalCoefs_[i][j]->N(),
insertPoint,
pointColsPerBlockCol,
tmpNumCols, ptColAllocLen) );
}
}
}
Epetra_BlockMap colMap(-1, numCols, cols, // CJ TODO FIXME long long
pointColsPerBlockCol,
RowMap().IndexBase(), RowMap().Comm());
delete [] cols;
delete [] pointColsPerBlockCol;
numCols = 0;
allocLen = 0;
//now we need to create a matrix with sourceMap and colMap, and fill it with
//our nonlocal data so we can then export it to the correct owning
//processors.
Epetra_VbrMatrix tempMat(Copy, sourceMap, colMap, nonlocalBlockRowLengths_);
//Next we need to make sure the 'indices-are-global' attribute of tempMat's
//graph is set to true, in case this processor doesn't end up calling the
//InsertGlobalValues method...
const Epetra_CrsGraph& graph = tempMat.Graph();
Epetra_CrsGraph& nonconst_graph = const_cast<Epetra_CrsGraph&>(graph);
nonconst_graph.SetIndicesAreGlobal(true);
for(i=0; i<numNonlocalBlockRows_; ++i) {
EPETRA_CHK_ERR( tempMat.BeginInsertGlobalValues(nonlocalBlockRows_[i],
nonlocalBlockRowLengths_[i],
nonlocalBlockCols_[i]) );
for(int j=0; j<nonlocalBlockRowLengths_[i]; ++j) {
Epetra_SerialDenseMatrix& subblock = *(nonlocalCoefs_[i][j]);
EPETRA_CHK_ERR( tempMat.SubmitBlockEntry(subblock.A(),
subblock.LDA(),
subblock.M(),
subblock.N()) );
}
EPETRA_CHK_ERR( tempMat.EndSubmitEntries() );
}
//Now we need to call FillComplete on our temp matrix. We need to
//pass a DomainMap and RangeMap, which are not the same as the RowMap
//and ColMap that we constructed the matrix with.
EPETRA_CHK_ERR(tempMat.FillComplete(RowMap(), sourceMap));
//Finally, we're ready to create the exporter and export non-local data to
//the appropriate owning processors.
Epetra_Export exporter(sourceMap, RowMap());
EPETRA_CHK_ERR( Export(tempMat, exporter, Add) );
if(callFillComplete) {
EPETRA_CHK_ERR(FillComplete());
}
destroyNonlocalData();
return(0);
}
int MatrixMarketFileToBlockMaps(const char* filename,
const Epetra_Comm& comm,
Epetra_BlockMap*& rowmap,
Epetra_BlockMap*& colmap,
Epetra_BlockMap*& rangemap,
Epetra_BlockMap*& domainmap)
{
FILE* infile = fopen(filename, "r");
if (infile == NULL) {
return(-1);
}
MM_typecode matcode;
int err = mm_read_banner(infile, &matcode);
if (err != 0) return(err);
if (!mm_is_matrix(matcode) || !mm_is_coordinate(matcode) ||
!mm_is_real(matcode) || !mm_is_general(matcode)) {
return(-1);
}
int numrows, numcols, nnz;
err = mm_read_mtx_crd_size(infile, &numrows, &numcols, &nnz);
if (err != 0) return(err);
//for this case, we'll assume that the row-map is the same as
//the range-map.
//create row-map and range-map with linear distributions.
rowmap = new Epetra_BlockMap(numrows, 1, 0, comm);
rangemap = new Epetra_BlockMap(numrows, 1, 0, comm);
int I, J;
double val, imag;
int num_map_cols = 0, insertPoint, foundOffset;
int allocLen = numcols;
int* map_cols = new int[allocLen];
//read through all matrix data and construct a list of the column-
//indices that occur in rows that are local to this processor.
for(int i=0; i<nnz; ++i) {
err = mm_read_mtx_crd_entry(infile, &I, &J, &val,
&imag, matcode);
if (err == 0) {
--I;
--J;
if (rowmap->MyGID(I)) {
foundOffset = Epetra_Util_binary_search(J, map_cols, num_map_cols,
insertPoint);
if (foundOffset < 0) {
Epetra_Util_insert(J, insertPoint, map_cols,
num_map_cols, allocLen);
}
}
}
}
//create colmap with the list of columns associated with rows that are
//local to this processor.
colmap = new Epetra_Map(-1, num_map_cols, map_cols, 0, comm);
//create domainmap which has a linear distribution
domainmap = new Epetra_BlockMap(numcols, 1, 0, comm);
delete [] map_cols;
return(0);
}
matrix_data::matrix_data(int num_quad_elements,
int num_dof_per_node,
bool make_numerically_nonsymmetric)
: numrows_(0),
numcols_(0),
rows_(0),
rowlengths_(0),
blocksize_(num_dof_per_node),
colindices_(0),
coefs_(0)
{
//Set up matrix-data representing a simple finite-element
//mesh containing 2-D quad elements
//
// *-----*-----*-----*
// 0| 2| 4| 6|
// | 0 | 1 | ne-1|
// | | | |
// *-----*-----*-----*
// 1 3 5 7
//
//In the above drawing, 'ne' means num-elems. node-numbers are to the
//lower-left of each node (*).
numrows_ = num_quad_elements*2+2;
if (numrows_ > 0) {
rows_ = new int[numrows_];
rowlengths_ = new int[numrows_];
colindices_ = new int*[numrows_];
coefs_ = new double*[numrows_];
int i, j, k;
for(i=0; i<numrows_; ++i) {
rows_[i] = i;
rowlengths_[i] = 0;
}
int* nodes = new int[nodes_per_elem];
for(i=0; i<num_quad_elements; ++i) {
get_node_ids(i, nodes);
for(j=0; j<nodes_per_elem; ++j) {
int node_j = nodes[j];
for(k=0; k<nodes_per_elem; ++k) {
int insertPoint = -1;
int alloclen = rowlengths_[node_j];
int offset = Epetra_Util_binary_search(nodes[k], colindices_[node_j],
rowlengths_[node_j], insertPoint);
if (offset < 0) {
Epetra_Util_insert(nodes[k], insertPoint,
colindices_[node_j], rowlengths_[node_j],
alloclen);
}
}
}
}
int dim = blocksize_*blocksize_;
for(i=0; i<numrows_; ++i) {
int len = rowlengths_[i]*dim;
coefs_[i] = new double[len];
for(j=0; j<len; ++j) {
if (make_numerically_nonsymmetric) {
coefs_[i][j] = 1.0*(j+1);
}
else {
coefs_[i][j] = 1.0;
}
}
}
}
}