int
UmfpackGenLinSOE::sendSelf(int cTag, Channel &theChannel)
{
LinearSOESolver *theSoeSolver = this->getSolver();
if (theSoeSolver != 0) {
if (theSoeSolver->sendSelf(cTag, theChannel) < 0) {
opserr <<"WARNING MumpsParallelSOE::sendSelf() - failed to send solver\n";
return -1;
}
} else {
opserr <<"WARNING MumpsParallelSOE::sendSelf() - no solver to send!\n";
return -1;
}
return 0;
}
int
DistributedSparseGenColLinSOE::setSize(Graph &theGraph)
{
int result = 0;
int oldSize = size;
int maxNumSubVertex = 0;
// if subprocess, collect graph, send it off,
// vector back containing size of system, etc.
if (processID != 0) {
Channel *theChannel = theChannels[0];
theGraph.sendSelf(0, *theChannel);
static ID data(2);
theChannel->recvID(0, 0, data);
size = data(0);
nnz = data(1);
ID *subMap = new ID(theGraph.getNumVertex());
localCol[0] = subMap;
Vertex *vertex;
VertexIter &theSubVertices = theGraph.getVertices();
int cnt = 0;
while((vertex = theSubVertices()) != 0)
(*subMap)(cnt++) = vertex->getTag();
theChannel->sendID(0, 0, *subMap);
if (nnz > Asize) { // we have to get more space for A and rowA
if (rowA != 0)
delete [] rowA;
rowA = new int[nnz];
if (rowA == 0) {
opserr << "WARNING SparseGenColLinSOE::SparseGenColLinSOE :";
opserr << " ran out of memory for A and rowA with nnz = ";
opserr << nnz << " \n";
size = 0; Asize = 0; nnz = 0;
result = -1;
}
}
if (size > Bsize) { // we have to get space for the vectors
if (colStartA != 0)
delete [] colStartA;
colStartA = new int[size+1];
if (colStartA == 0) {
opserr << "WARNING SparseGenColLinSOE::SparseGenColLinSOE :";
opserr << " ran out of memory for vectors (size) (";
opserr << size << ") \n";
size = 0; Bsize = 0;
result = -1;
}
}
ID rowAdata(rowA, nnz);
ID colStartAdata(colStartA, size+1);
theChannel->recvID(0, 0, rowAdata);
theChannel->recvID(0, 0, colStartAdata);
}
// if main domain, collect graphs from all subdomains,
// merge into 1, number this one, send to subdomains the
// id containing dof tags & start id's.
else {
// from each distributed soe recv it's graph
// and merge them into master graph
FEM_ObjectBroker theBroker;
for (int j=0; j<numChannels; j++) {
Channel *theChannel = theChannels[j];
Graph theSubGraph;
theSubGraph.recvSelf(0, *theChannel, theBroker);
theGraph.merge(theSubGraph);
int numSubVertex = theSubGraph.getNumVertex();
ID *subMap = new ID(numSubVertex);
localCol[j] = subMap;
if (numSubVertex > maxNumSubVertex)
maxNumSubVertex = numSubVertex;
}
size = theGraph.getNumVertex();
//
// determine the number of non-zeros
//
Vertex *theVertex;
VertexIter &theVertices = theGraph.getVertices();
nnz = 0;
while ((theVertex = theVertices()) != 0) {
const ID &theAdjacency = theVertex->getAdjacency();
nnz += theAdjacency.Size() +1; // the +1 is for the diag entry
}
static ID data(2);
data(0) = size;
data(1) = nnz;
// to each distributed soe send the size data
// and merge them into master graph
for (int j=0; j<numChannels; j++) {
Channel *theChannel = theChannels[j];
theChannel->sendID(0, 0, data);
ID *subMap = localCol[j];
theChannel->recvID(0, 0, *subMap);
}
if (nnz > Asize) { // we have to get more space for A and rowA
if (rowA != 0)
delete [] rowA;
if (workArea != 0)
delete [] workArea;
rowA = new int[nnz];
workArea = new double[nnz];
sizeWork = nnz;
if (rowA == 0 || workArea == 0) {
opserr << "WARNING SparseGenColLinSOE::SparseGenColLinSOE :";
opserr << " ran out of memory for A and rowA with nnz = ";
opserr << nnz << " \n";
size = 0; Asize = 0; nnz = 0;
result = -1;
}
}
if (size > Bsize) { // we have to get space for the vectors
if (colStartA != 0)
delete [] colStartA;
colStartA = new int[size+1];
if (colStartA == 0) {
opserr << "WARNING SparseGenColLinSOE::SparseGenColLinSOE :";
opserr << " ran out of memory for vectors (size) (";
opserr << size << ") \n";
size = 0; Bsize = 0;
result = -1;
}
}
// fill in colStartA and rowA
if (size != 0) {
colStartA[0] = 0;
int startLoc = 0;
int lastLoc = 0;
for (int a=0; a<size; a++) {
theVertex = theGraph.getVertexPtr(a);
if (theVertex == 0) {
opserr << "WARNING:SparseGenColLinSOE::setSize :";
opserr << " vertex " << a << " not in graph! - size set to 0\n";
size = 0;
return -1;
}
rowA[lastLoc++] = theVertex->getTag(); // place diag in first
const ID &theAdjacency = theVertex->getAdjacency();
int idSize = theAdjacency.Size();
// now we have to place the entries in the ID into order in rowA
for (int i=0; i<idSize; i++) {
int row = theAdjacency(i);
bool foundPlace = false;
// find a place in rowA for current col
for (int j=startLoc; j<lastLoc; j++)
if (rowA[j] > row) {
// move the entries already there one further on
// and place col in current location
for (int k=lastLoc; k>j; k--)
rowA[k] = rowA[k-1];
rowA[j] = row;
foundPlace = true;
j = lastLoc;
}
if (foundPlace == false) // put in at the end
rowA[lastLoc] = row;
lastLoc++;
}
colStartA[a+1] = lastLoc;;
startLoc = lastLoc;
}
}
ID rowAdata(rowA, nnz);
ID colStartAdata(colStartA, size+1);
for (int j=0; j<numChannels; j++) {
Channel *theChannel = theChannels[j];
theChannel->sendID(0, 0, rowAdata);
theChannel->sendID(0, 0, colStartAdata);
}
}
if (nnz > Asize) { // we have to get more space for A and rowA
if (A != 0)
delete [] A;
A = new double[nnz];
if (A == 0 || rowA == 0) {
opserr << "WARNING SparseGenColLinSOE::SparseGenColLinSOE :";
opserr << " ran out of memory for A and rowA with nnz = ";
opserr << nnz << " \n";
size = 0; Asize = 0; nnz = 0;
result = -1;
}
Asize = nnz;
}
// zero the matrix
for (int i=0; i<Asize; i++)
A[i] = 0;
factored = false;
if (size > Bsize) { // we have to get space for the vectors
// delete the old
if (B != 0) delete [] B;
if (X != 0) delete [] X;
if (myB != 0) delete [] myB;
// create the new
B = new double[size];
X = new double[size];
myB = new double[size];
if (B == 0 || X == 0 || colStartA == 0 || myB == 0) {
opserr << "WARNING SparseGenColLinSOE::SparseGenColLinSOE :";
opserr << " ran out of memory for vectors (size) (";
opserr << size << ") \n";
size = 0; Bsize = 0;
result = -1;
}
else
Bsize = size;
}
// zero the vectors
for (int j=0; j<size; j++) {
B[j] = 0;
X[j] = 0;
myB[j] = 0;
}
// create new Vectors objects
if (size != oldSize) {
if (vectX != 0)
delete vectX;
if (vectB != 0)
delete vectB;
vectX = new Vector(X,size);
vectB = new Vector(B,size);
myVectB = new Vector(myB, size);
}
LinearSOESolver *theSolvr = this->getSolver();
int solverOK = theSolvr->setSize();
if (solverOK < 0) {
opserr << "WARNING:DistributedSparseGenColLinSOE::setSize :";
opserr << " solver failed setSize()\n";
return solverOK;
}
return result;
}
int
DistributedSparseGenColLinSOE::solve(void)
{
static ID result(1);
//
// if subprocess send B and A and receive back result X, B & result
//
if (processID != 0) {
Channel *theChannel = theChannels[0];
// send B
theChannel->sendVector(0, 0, *myVectB);
// send A in packets placed in vector X
if (factored == false) {
Vector vectA(A, nnz);
theChannel->sendVector(0, 0, vectA);
}
LinearSOESolver *theSoeSolver = this->getSolver();
if (theSoeSolver->getClassTag() == SOLVER_TAGS_DistributedSuperLU)
this->LinearSOE::solve();
// receive X,B and result
theChannel->recvVector(0, 0, *vectX);
theChannel->recvVector(0, 0, *vectB);
theChannel->recvID(0, 0, result);
factored = true;
}
//
// if main process, recv B & A from all, solve and send back X, B & result
//
else {
// add P0 contribution to B
*vectB = *myVectB;
// receive X and A contribution from subprocess & add them in
for (int j=0; j<numChannels; j++) {
// get X & add
Channel *theChannel = theChannels[j];
theChannel->recvVector(0, 0, *vectX);
*vectB += *vectX;
if (factored == false) {
Vector vectA(workArea, nnz);
theChannel->recvVector(0, 0, vectA);
for (int i=0; i<nnz; i++)
A[i] += workArea[i];
}
/*
// get A & add using local map
const ID &localMap = *(localCol[j]);
int localSize = localMap.Size() * half_band;
Vector vectA(workArea, localSize);
theChannel->recvVector(0, 0, vectA);
int loc = 0;
for (int i=0; i<localMap.Size(); i++) {
int pos = localMap(i)*half_band;
for (int k=0; k<half_band; k++)
A[pos++] += workArea[loc++];
}
*/
}
// solve
result(0) = this->LinearSOE::solve();
// send results back
for (int j=0; j<numChannels; j++) {
Channel *theChannel = theChannels[j];
theChannel->sendVector(0, 0, *vectX);
theChannel->sendVector(0, 0, *vectB);
theChannel->sendID(0, 0, result);
}
}
return result(0);
}
int
BandGenLinSOE::setSize(Graph &theGraph)
{
int result = 0;
int oldSize = size;
size = theGraph.getNumVertex();
/*
* determine the number of superdiagonals and subdiagonals
*/
numSubD = 0;
numSuperD = 0;
Vertex *vertexPtr;
VertexIter &theVertices = theGraph.getVertices();
while ((vertexPtr = theVertices()) != 0) {
int vertexNum = vertexPtr->getTag();
const ID &theAdjacency = vertexPtr->getAdjacency();
for (int i=0; i<theAdjacency.Size(); i++) {
int otherNum = theAdjacency(i);
int diff = vertexNum - otherNum;
if (diff > 0) {
if (diff > numSuperD)
numSuperD = diff;
} else
if (diff < numSubD)
numSubD = diff;
}
}
numSubD *= -1;
int newSize = size * (2*numSubD + numSuperD +1);
if (newSize > Asize) { // we have to get another space for A
if (A != 0)
delete [] A;
A = new double[newSize];
if (A == 0) {
opserr << "WARNING BandGenLinSOE::BandGenLinSOE :";
opserr << " ran out of memory for A (size,super,sub) (";
opserr << size <<", " << numSuperD << ", " << numSubD << ") \n";
Asize = 0; size = 0; numSubD = 0; numSuperD = 0;
result= -1;
}
else
Asize = newSize;
}
// zero the matrix
for (int i=0; i<Asize; i++)
A[i] = 0;
factored = false;
if (size > Bsize) { // we have to get space for the vectors
// delete the old
if (B != 0) delete [] B;
if (X != 0) delete [] X;
// create the new
B = new double[size];
X = new double[size];
if (B == 0 || X == 0) {
opserr << "WARNING BandGenLinSOE::BandGenLinSOE :";
opserr << " ran out of memory for vectors (size) (";
opserr << size << ") \n";
Bsize = 0; size = 0; numSubD = 0; numSuperD = 0;
result = -1;
}
else
Bsize = size;
}
// zero the vectors
for (int j=0; j<size; j++) {
B[j] = 0;
X[j] = 0;
}
// get new Vector objects if size has changes
if (oldSize != size) {
if (vectX != 0)
delete vectX;
if (vectB != 0)
delete vectB;
vectX = new Vector(X,size);
vectB = new Vector(B,size);
}
// invoke setSize() on the Solver
LinearSOESolver *theSolvr = this->getSolver();
int solverOK = theSolvr->setSize();
if (solverOK < 0) {
opserr << "WARNING:BandGenLinSOE::setSize :";
opserr << " solver failed setSize()\n";
return solverOK;
}
return result;
}
int
FullGenLinSOE::setSize(Graph &theGraph)
{
int result = 0;
int oldSize = size;
size = theGraph.getNumVertex();
if (size*size > Asize) { // we have to get another space for A
if (A != 0)
delete [] A;
A = new (nothrow) double[size*size];
if (A == 0) {
opserr << "WARNING FullGenLinSOE::FullGenLinSOE :";
opserr << " ran out of memory for A (size,size) (";
opserr << size <<", " << size << ") \n";
size = 0; Asize = 0;
result = -1;
} else
Asize = size*size;
}
// zero the matrix
for (int i=0; i<Asize; i++)
A[i] = 0;
factored = false;
if (size > Bsize) { // we have to get space for the vectors
// delete the old
if (B != 0) delete [] B;
if (X != 0) delete [] X;
// create the new
B = new (nothrow) double[size];
X = new (nothrow) double[size];
if (B == 0 || X == 0) {
opserr << "WARNING FullGenLinSOE::FullGenLinSOE :";
opserr << " ran out of memory for vectors (size) (";
opserr << size << ") \n";
size = 0; Bsize = 0;
result = -1;
}
else
Bsize = size;
}
// zero the vectors
for (int j=0; j<Bsize; j++) {
B[j] = 0;
X[j] = 0;
}
// create new Vectors
if (size != oldSize) {
if (vectX != 0)
delete vectX;
if (vectB != 0)
delete vectB;
vectX = new Vector(X,Bsize);
vectB = new Vector(B,Bsize);
}
// invoke setSize() on the Solver
LinearSOESolver *theSolvr = this->getSolver();
int solverOK = theSolvr->setSize();
if (solverOK < 0) {
opserr << "WARNING:FullGenLinSOE::setSize :";
opserr << " solver failed setSize()\n";
return solverOK;
}
return result;
}
int
MumpsParallelSOE::setSize(Graph &theGraph)
{
int result = 0;
int oldSize = size;
int maxNumSubVertex = 0;
// fist itearte through the vertices of the graph to get nnzLoc and n
int maxVertexTag = -1;
Vertex *theVertex;
int newNNZ = 0;
size = theGraph.getNumVertex();
int mySize = size;
//opserr << "MumpsParallelSOE: size : " << size << endln;
VertexIter &theVertices = theGraph.getVertices();
while ((theVertex = theVertices()) != 0) {
int vertexTag = theVertex->getTag();
if (vertexTag > maxVertexTag)
maxVertexTag = vertexTag;
const ID &theAdjacency = theVertex->getAdjacency();
newNNZ += theAdjacency.Size() +1; // the +1 is for the diag entry
}
if (matType != 0) {
// symmetric - allows us to reduce nnz by almost half
newNNZ -= size;
newNNZ /= 2;
newNNZ += size;
}
nnz = newNNZ;
if (processID != 0) {
//
// if subprocess, send local max vertexTag (n)
// recv ax n from P0
//
static ID data(1);
data(0) = maxVertexTag;
Channel *theChannel = theChannels[0];
theChannel->sendID(0, 0, data);
theChannel->recvID(0, 0, data);
size = data(0);
} else {
//
// from each distributed soe recv it's max n and compare; return max n to all
//
static ID data(1);
FEM_ObjectBroker theBroker;
for (int j=0; j<numChannels; j++) {
Channel *theChannel = theChannels[j];
theChannel->recvID(0, 0, data);
if (data(0) > maxVertexTag)
maxVertexTag = data(0);
}
data(0) = maxVertexTag;
for (int j=0; j<numChannels; j++) {
Channel *theChannel = theChannels[j];
theChannel->sendID(0, 0, data);
}
size = maxVertexTag;
}
size+=1; // vertices numbered 0 through n-1
if (nnz > Asize) { // we have to get more space for A and rowA and colA
if (A != 0) delete [] A;
if (rowA != 0) delete [] rowA;
if (colA != 0) delete [] colA;
A = new double[nnz];
rowA = new int[nnz];
colA = new int[nnz];
for (int i=0; i<nnz; i++) {
A[i]=0;
rowA[i]=0;
colA[i]=0;
}
if (rowA == 0 || A == 0 || colA == 0) {
opserr << "WARNING SparseGenColLinSOE::SparseGenColLinSOE :";
opserr << " ran out of memory for A and rowA with nnz = ";
opserr << nnz << " \n";
size = 0; Asize = 0; nnz = 0;
result = -1;
}
Asize = nnz;
}
if (size > Bsize) { // we have to get space for the vectors
if (B != 0) delete [] B;
if (X != 0) delete [] X;
if (myB != 0) delete [] myB;
if (workArea != 0) delete [] workArea;
if (colStartA != 0) delete [] colStartA;
// create the new
B = new double[size];
X = new double[size];
myB = new double[size];
workArea = new double[size];
colStartA = new int[size+1];
if (B == 0 || X == 0 || colStartA == 0 || workArea == 0 || myB == 0) {
opserr << "WARNING MumpsSOE::MumpsSOE :";
opserr << " ran out of memory for vectors (size) (";
opserr << size << ") \n";
size = 0; Bsize = 0;
result = -1;
}
else
Bsize = size;
}
// zero the vectors
for (int j=0; j<size; j++) {
B[j] = 0;
X[j] = 0;
myB[j] = 0;
}
// create new Vectors objects
if (size != oldSize) {
if (vectX != 0) delete vectX;
if (vectB != 0) delete vectB;
if (myVectB != 0) delete myVectB;
vectX = new Vector(X,size);
vectB = new Vector(B,size);
myVectB = new Vector(myB, size);
}
// fill in colStartA and rowA
if (size != 0) {
colStartA[0] = 0;
int startLoc = 0;
int lastLoc = 0;
for (int a=0; a<size; a++) {
theVertex = theGraph.getVertexPtr(a);
if (theVertex != 0) {
int vertexTag = theVertex->getTag();
rowA[lastLoc++] = vertexTag; // place diag in first
const ID &theAdjacency = theVertex->getAdjacency();
int idSize = theAdjacency.Size();
// now we have to place the entries in the ID into order in rowA
if (matType != 0) {
// symmetric
for (int i=0; i<idSize; i++) {
int row = theAdjacency(i);
if (row > vertexTag) {
bool foundPlace = false;
// find a place in rowA for current col
for (int j=startLoc; j<lastLoc; j++)
if (rowA[j] > row) {
// move the entries already there one further on
// and place col in current location
for (int k=lastLoc; k>j; k--)
rowA[k] = rowA[k-1];
rowA[j] = row;
foundPlace = true;
j = lastLoc;
}
if (foundPlace == false) // put in at the end
rowA[lastLoc] = row;
lastLoc++;
}
}
} else {
// unsymmetric
for (int i=0; i<idSize; i++) {
int row = theAdjacency(i);
bool foundPlace = false;
// find a place in rowA for current col
for (int j=startLoc; j<lastLoc; j++)
if (rowA[j] > row) {
// move the entries already there one further on
// and place col in current location
for (int k=lastLoc; k>j; k--)
rowA[k] = rowA[k-1];
rowA[j] = row;
foundPlace = true;
j = lastLoc;
}
if (foundPlace == false) // put in at the end
rowA[lastLoc] = row;
lastLoc++;
}
}
}
colStartA[a+1] = lastLoc;
startLoc = lastLoc;
}
}
// fill in colA
int count = 0;
for (int i=0; i<size; i++) {
for (int k=colStartA[i]; k<colStartA[i+1]; k++)
colA[count++] = i;
}
LinearSOESolver *theSolvr = this->getSolver();
int solverOK = theSolvr->setSize();
if (solverOK < 0) {
opserr << "WARNING:MumpsParallelSOE::setSize :";
opserr << " solver failed setSize()\n";
return solverOK;
}
return result;
}
int
MumpsParallelSOE::sendSelf(int commitTag, Channel &theChannel)
{
int sendID =0;
// if P0 check if already sent. If already sent use old processID; if not allocate a new process
// id for remote part of object, enlarge channel * to hold a channel * for this remote object.
// if not P0, send current processID
if (processID == 0) {
// check if already using this object
bool found = false;
for (int i=0; i<numChannels; i++)
if (theChannels[i] == &theChannel) {
sendID = i+1;
found = true;
}
// if new object, enlarge Channel pointers to hold new channel * & allocate new ID
if (found == false) {
int nextNumChannels = numChannels + 1;
Channel **nextChannels = new Channel *[nextNumChannels];
if (nextNumChannels == 0) {
opserr << "MumpsParallelSOE::sendSelf() - failed to allocate channel array of size: " <<
nextNumChannels << endln;
return -1;
}
for (int i=0; i<numChannels; i++)
nextChannels[i] = theChannels[i];
nextChannels[numChannels] = &theChannel;
numChannels = nextNumChannels;
if (theChannels != 0)
delete [] theChannels;
theChannels = nextChannels;
if (localCol != 0)
delete [] localCol;
localCol = new ID *[numChannels];
if (localCol == 0) {
opserr << "MumpsParallelSOE::sendSelf() - failed to allocate id array of size: " <<
nextNumChannels << endln;
return -1;
}
for (int i=0; i<numChannels; i++)
localCol[i] = 0;
// allocate new processID for remote object
sendID = numChannels;
}
} else
sendID = processID;
// send remotes processID
ID idData(2);
idData(0) = sendID;
idData(1) = matType;
int res = theChannel.sendID(0, commitTag, idData);
if (res < 0) {
opserr <<"WARNING MumpsParallelSOE::sendSelf() - failed to send data\n";
return -1;
}
LinearSOESolver *theSoeSolver = this->getSolver();
if (theSoeSolver != 0) {
if (theSoeSolver->sendSelf(commitTag, theChannel) < 0) {
opserr <<"WARNING MumpsParallelSOE::sendSelf() - failed to send solver\n";
return -1;
}
} else {
opserr <<"WARNING MumpsParallelSOE::sendSelf() - no solver to send!\n";
return -1;
}
return 0;
}
