int UmfpackGenLinSOE::sendSelf(int cTag, Channel &theChannel) { LinearSOESolver *theSoeSolver = this->getSolver(); if (theSoeSolver != 0) { if (theSoeSolver->sendSelf(cTag, theChannel) < 0) { opserr <<"WARNING MumpsParallelSOE::sendSelf() - failed to send solver\n"; return -1; } } else { opserr <<"WARNING MumpsParallelSOE::sendSelf() - no solver to send!\n"; return -1; } return 0; }
int DistributedSparseGenColLinSOE::setSize(Graph &theGraph) { int result = 0; int oldSize = size; int maxNumSubVertex = 0; // if subprocess, collect graph, send it off, // vector back containing size of system, etc. if (processID != 0) { Channel *theChannel = theChannels[0]; theGraph.sendSelf(0, *theChannel); static ID data(2); theChannel->recvID(0, 0, data); size = data(0); nnz = data(1); ID *subMap = new ID(theGraph.getNumVertex()); localCol[0] = subMap; Vertex *vertex; VertexIter &theSubVertices = theGraph.getVertices(); int cnt = 0; while((vertex = theSubVertices()) != 0) (*subMap)(cnt++) = vertex->getTag(); theChannel->sendID(0, 0, *subMap); if (nnz > Asize) { // we have to get more space for A and rowA if (rowA != 0) delete [] rowA; rowA = new int[nnz]; if (rowA == 0) { opserr << "WARNING SparseGenColLinSOE::SparseGenColLinSOE :"; opserr << " ran out of memory for A and rowA with nnz = "; opserr << nnz << " \n"; size = 0; Asize = 0; nnz = 0; result = -1; } } if (size > Bsize) { // we have to get space for the vectors if (colStartA != 0) delete [] colStartA; colStartA = new int[size+1]; if (colStartA == 0) { opserr << "WARNING SparseGenColLinSOE::SparseGenColLinSOE :"; opserr << " ran out of memory for vectors (size) ("; opserr << size << ") \n"; size = 0; Bsize = 0; result = -1; } } ID rowAdata(rowA, nnz); ID colStartAdata(colStartA, size+1); theChannel->recvID(0, 0, rowAdata); theChannel->recvID(0, 0, colStartAdata); } // if main domain, collect graphs from all subdomains, // merge into 1, number this one, send to subdomains the // id containing dof tags & start id's. else { // from each distributed soe recv it's graph // and merge them into master graph FEM_ObjectBroker theBroker; for (int j=0; j<numChannels; j++) { Channel *theChannel = theChannels[j]; Graph theSubGraph; theSubGraph.recvSelf(0, *theChannel, theBroker); theGraph.merge(theSubGraph); int numSubVertex = theSubGraph.getNumVertex(); ID *subMap = new ID(numSubVertex); localCol[j] = subMap; if (numSubVertex > maxNumSubVertex) maxNumSubVertex = numSubVertex; } size = theGraph.getNumVertex(); // // determine the number of non-zeros // Vertex *theVertex; VertexIter &theVertices = theGraph.getVertices(); nnz = 0; while ((theVertex = theVertices()) != 0) { const ID &theAdjacency = theVertex->getAdjacency(); nnz += theAdjacency.Size() +1; // the +1 is for the diag entry } static ID data(2); data(0) = size; data(1) = nnz; // to each distributed soe send the size data // and merge them into master graph for (int j=0; j<numChannels; j++) { Channel *theChannel = theChannels[j]; theChannel->sendID(0, 0, data); ID *subMap = localCol[j]; theChannel->recvID(0, 0, *subMap); } if (nnz > Asize) { // we have to get more space for A and rowA if (rowA != 0) delete [] rowA; if (workArea != 0) delete [] workArea; rowA = new int[nnz]; workArea = new double[nnz]; sizeWork = nnz; if (rowA == 0 || workArea == 0) { opserr << "WARNING SparseGenColLinSOE::SparseGenColLinSOE :"; opserr << " ran out of memory for A and rowA with nnz = "; opserr << nnz << " \n"; size = 0; Asize = 0; nnz = 0; result = -1; } } if (size > Bsize) { // we have to get space for the vectors if (colStartA != 0) delete [] colStartA; colStartA = new int[size+1]; if (colStartA == 0) { opserr << "WARNING SparseGenColLinSOE::SparseGenColLinSOE :"; opserr << " ran out of memory for vectors (size) ("; opserr << size << ") \n"; size = 0; Bsize = 0; result = -1; } } // fill in colStartA and rowA if (size != 0) { colStartA[0] = 0; int startLoc = 0; int lastLoc = 0; for (int a=0; a<size; a++) { theVertex = theGraph.getVertexPtr(a); if (theVertex == 0) { opserr << "WARNING:SparseGenColLinSOE::setSize :"; opserr << " vertex " << a << " not in graph! - size set to 0\n"; size = 0; return -1; } rowA[lastLoc++] = theVertex->getTag(); // place diag in first const ID &theAdjacency = theVertex->getAdjacency(); int idSize = theAdjacency.Size(); // now we have to place the entries in the ID into order in rowA for (int i=0; i<idSize; i++) { int row = theAdjacency(i); bool foundPlace = false; // find a place in rowA for current col for (int j=startLoc; j<lastLoc; j++) if (rowA[j] > row) { // move the entries already there one further on // and place col in current location for (int k=lastLoc; k>j; k--) rowA[k] = rowA[k-1]; rowA[j] = row; foundPlace = true; j = lastLoc; } if (foundPlace == false) // put in at the end rowA[lastLoc] = row; lastLoc++; } colStartA[a+1] = lastLoc;; startLoc = lastLoc; } } ID rowAdata(rowA, nnz); ID colStartAdata(colStartA, size+1); for (int j=0; j<numChannels; j++) { Channel *theChannel = theChannels[j]; theChannel->sendID(0, 0, rowAdata); theChannel->sendID(0, 0, colStartAdata); } } if (nnz > Asize) { // we have to get more space for A and rowA if (A != 0) delete [] A; A = new double[nnz]; if (A == 0 || rowA == 0) { opserr << "WARNING SparseGenColLinSOE::SparseGenColLinSOE :"; opserr << " ran out of memory for A and rowA with nnz = "; opserr << nnz << " \n"; size = 0; Asize = 0; nnz = 0; result = -1; } Asize = nnz; } // zero the matrix for (int i=0; i<Asize; i++) A[i] = 0; factored = false; if (size > Bsize) { // we have to get space for the vectors // delete the old if (B != 0) delete [] B; if (X != 0) delete [] X; if (myB != 0) delete [] myB; // create the new B = new double[size]; X = new double[size]; myB = new double[size]; if (B == 0 || X == 0 || colStartA == 0 || myB == 0) { opserr << "WARNING SparseGenColLinSOE::SparseGenColLinSOE :"; opserr << " ran out of memory for vectors (size) ("; opserr << size << ") \n"; size = 0; Bsize = 0; result = -1; } else Bsize = size; } // zero the vectors for (int j=0; j<size; j++) { B[j] = 0; X[j] = 0; myB[j] = 0; } // create new Vectors objects if (size != oldSize) { if (vectX != 0) delete vectX; if (vectB != 0) delete vectB; vectX = new Vector(X,size); vectB = new Vector(B,size); myVectB = new Vector(myB, size); } LinearSOESolver *theSolvr = this->getSolver(); int solverOK = theSolvr->setSize(); if (solverOK < 0) { opserr << "WARNING:DistributedSparseGenColLinSOE::setSize :"; opserr << " solver failed setSize()\n"; return solverOK; } return result; }
int DistributedSparseGenColLinSOE::solve(void) { static ID result(1); // // if subprocess send B and A and receive back result X, B & result // if (processID != 0) { Channel *theChannel = theChannels[0]; // send B theChannel->sendVector(0, 0, *myVectB); // send A in packets placed in vector X if (factored == false) { Vector vectA(A, nnz); theChannel->sendVector(0, 0, vectA); } LinearSOESolver *theSoeSolver = this->getSolver(); if (theSoeSolver->getClassTag() == SOLVER_TAGS_DistributedSuperLU) this->LinearSOE::solve(); // receive X,B and result theChannel->recvVector(0, 0, *vectX); theChannel->recvVector(0, 0, *vectB); theChannel->recvID(0, 0, result); factored = true; } // // if main process, recv B & A from all, solve and send back X, B & result // else { // add P0 contribution to B *vectB = *myVectB; // receive X and A contribution from subprocess & add them in for (int j=0; j<numChannels; j++) { // get X & add Channel *theChannel = theChannels[j]; theChannel->recvVector(0, 0, *vectX); *vectB += *vectX; if (factored == false) { Vector vectA(workArea, nnz); theChannel->recvVector(0, 0, vectA); for (int i=0; i<nnz; i++) A[i] += workArea[i]; } /* // get A & add using local map const ID &localMap = *(localCol[j]); int localSize = localMap.Size() * half_band; Vector vectA(workArea, localSize); theChannel->recvVector(0, 0, vectA); int loc = 0; for (int i=0; i<localMap.Size(); i++) { int pos = localMap(i)*half_band; for (int k=0; k<half_band; k++) A[pos++] += workArea[loc++]; } */ } // solve result(0) = this->LinearSOE::solve(); // send results back for (int j=0; j<numChannels; j++) { Channel *theChannel = theChannels[j]; theChannel->sendVector(0, 0, *vectX); theChannel->sendVector(0, 0, *vectB); theChannel->sendID(0, 0, result); } } return result(0); }
int BandGenLinSOE::setSize(Graph &theGraph) { int result = 0; int oldSize = size; size = theGraph.getNumVertex(); /* * determine the number of superdiagonals and subdiagonals */ numSubD = 0; numSuperD = 0; Vertex *vertexPtr; VertexIter &theVertices = theGraph.getVertices(); while ((vertexPtr = theVertices()) != 0) { int vertexNum = vertexPtr->getTag(); const ID &theAdjacency = vertexPtr->getAdjacency(); for (int i=0; i<theAdjacency.Size(); i++) { int otherNum = theAdjacency(i); int diff = vertexNum - otherNum; if (diff > 0) { if (diff > numSuperD) numSuperD = diff; } else if (diff < numSubD) numSubD = diff; } } numSubD *= -1; int newSize = size * (2*numSubD + numSuperD +1); if (newSize > Asize) { // we have to get another space for A if (A != 0) delete [] A; A = new double[newSize]; if (A == 0) { opserr << "WARNING BandGenLinSOE::BandGenLinSOE :"; opserr << " ran out of memory for A (size,super,sub) ("; opserr << size <<", " << numSuperD << ", " << numSubD << ") \n"; Asize = 0; size = 0; numSubD = 0; numSuperD = 0; result= -1; } else Asize = newSize; } // zero the matrix for (int i=0; i<Asize; i++) A[i] = 0; factored = false; if (size > Bsize) { // we have to get space for the vectors // delete the old if (B != 0) delete [] B; if (X != 0) delete [] X; // create the new B = new double[size]; X = new double[size]; if (B == 0 || X == 0) { opserr << "WARNING BandGenLinSOE::BandGenLinSOE :"; opserr << " ran out of memory for vectors (size) ("; opserr << size << ") \n"; Bsize = 0; size = 0; numSubD = 0; numSuperD = 0; result = -1; } else Bsize = size; } // zero the vectors for (int j=0; j<size; j++) { B[j] = 0; X[j] = 0; } // get new Vector objects if size has changes if (oldSize != size) { if (vectX != 0) delete vectX; if (vectB != 0) delete vectB; vectX = new Vector(X,size); vectB = new Vector(B,size); } // invoke setSize() on the Solver LinearSOESolver *theSolvr = this->getSolver(); int solverOK = theSolvr->setSize(); if (solverOK < 0) { opserr << "WARNING:BandGenLinSOE::setSize :"; opserr << " solver failed setSize()\n"; return solverOK; } return result; }
int FullGenLinSOE::setSize(Graph &theGraph) { int result = 0; int oldSize = size; size = theGraph.getNumVertex(); if (size*size > Asize) { // we have to get another space for A if (A != 0) delete [] A; A = new (nothrow) double[size*size]; if (A == 0) { opserr << "WARNING FullGenLinSOE::FullGenLinSOE :"; opserr << " ran out of memory for A (size,size) ("; opserr << size <<", " << size << ") \n"; size = 0; Asize = 0; result = -1; } else Asize = size*size; } // zero the matrix for (int i=0; i<Asize; i++) A[i] = 0; factored = false; if (size > Bsize) { // we have to get space for the vectors // delete the old if (B != 0) delete [] B; if (X != 0) delete [] X; // create the new B = new (nothrow) double[size]; X = new (nothrow) double[size]; if (B == 0 || X == 0) { opserr << "WARNING FullGenLinSOE::FullGenLinSOE :"; opserr << " ran out of memory for vectors (size) ("; opserr << size << ") \n"; size = 0; Bsize = 0; result = -1; } else Bsize = size; } // zero the vectors for (int j=0; j<Bsize; j++) { B[j] = 0; X[j] = 0; } // create new Vectors if (size != oldSize) { if (vectX != 0) delete vectX; if (vectB != 0) delete vectB; vectX = new Vector(X,Bsize); vectB = new Vector(B,Bsize); } // invoke setSize() on the Solver LinearSOESolver *theSolvr = this->getSolver(); int solverOK = theSolvr->setSize(); if (solverOK < 0) { opserr << "WARNING:FullGenLinSOE::setSize :"; opserr << " solver failed setSize()\n"; return solverOK; } return result; }
int MumpsParallelSOE::setSize(Graph &theGraph) { int result = 0; int oldSize = size; int maxNumSubVertex = 0; // fist itearte through the vertices of the graph to get nnzLoc and n int maxVertexTag = -1; Vertex *theVertex; int newNNZ = 0; size = theGraph.getNumVertex(); int mySize = size; //opserr << "MumpsParallelSOE: size : " << size << endln; VertexIter &theVertices = theGraph.getVertices(); while ((theVertex = theVertices()) != 0) { int vertexTag = theVertex->getTag(); if (vertexTag > maxVertexTag) maxVertexTag = vertexTag; const ID &theAdjacency = theVertex->getAdjacency(); newNNZ += theAdjacency.Size() +1; // the +1 is for the diag entry } if (matType != 0) { // symmetric - allows us to reduce nnz by almost half newNNZ -= size; newNNZ /= 2; newNNZ += size; } nnz = newNNZ; if (processID != 0) { // // if subprocess, send local max vertexTag (n) // recv ax n from P0 // static ID data(1); data(0) = maxVertexTag; Channel *theChannel = theChannels[0]; theChannel->sendID(0, 0, data); theChannel->recvID(0, 0, data); size = data(0); } else { // // from each distributed soe recv it's max n and compare; return max n to all // static ID data(1); FEM_ObjectBroker theBroker; for (int j=0; j<numChannels; j++) { Channel *theChannel = theChannels[j]; theChannel->recvID(0, 0, data); if (data(0) > maxVertexTag) maxVertexTag = data(0); } data(0) = maxVertexTag; for (int j=0; j<numChannels; j++) { Channel *theChannel = theChannels[j]; theChannel->sendID(0, 0, data); } size = maxVertexTag; } size+=1; // vertices numbered 0 through n-1 if (nnz > Asize) { // we have to get more space for A and rowA and colA if (A != 0) delete [] A; if (rowA != 0) delete [] rowA; if (colA != 0) delete [] colA; A = new double[nnz]; rowA = new int[nnz]; colA = new int[nnz]; for (int i=0; i<nnz; i++) { A[i]=0; rowA[i]=0; colA[i]=0; } if (rowA == 0 || A == 0 || colA == 0) { opserr << "WARNING SparseGenColLinSOE::SparseGenColLinSOE :"; opserr << " ran out of memory for A and rowA with nnz = "; opserr << nnz << " \n"; size = 0; Asize = 0; nnz = 0; result = -1; } Asize = nnz; } if (size > Bsize) { // we have to get space for the vectors if (B != 0) delete [] B; if (X != 0) delete [] X; if (myB != 0) delete [] myB; if (workArea != 0) delete [] workArea; if (colStartA != 0) delete [] colStartA; // create the new B = new double[size]; X = new double[size]; myB = new double[size]; workArea = new double[size]; colStartA = new int[size+1]; if (B == 0 || X == 0 || colStartA == 0 || workArea == 0 || myB == 0) { opserr << "WARNING MumpsSOE::MumpsSOE :"; opserr << " ran out of memory for vectors (size) ("; opserr << size << ") \n"; size = 0; Bsize = 0; result = -1; } else Bsize = size; } // zero the vectors for (int j=0; j<size; j++) { B[j] = 0; X[j] = 0; myB[j] = 0; } // create new Vectors objects if (size != oldSize) { if (vectX != 0) delete vectX; if (vectB != 0) delete vectB; if (myVectB != 0) delete myVectB; vectX = new Vector(X,size); vectB = new Vector(B,size); myVectB = new Vector(myB, size); } // fill in colStartA and rowA if (size != 0) { colStartA[0] = 0; int startLoc = 0; int lastLoc = 0; for (int a=0; a<size; a++) { theVertex = theGraph.getVertexPtr(a); if (theVertex != 0) { int vertexTag = theVertex->getTag(); rowA[lastLoc++] = vertexTag; // place diag in first const ID &theAdjacency = theVertex->getAdjacency(); int idSize = theAdjacency.Size(); // now we have to place the entries in the ID into order in rowA if (matType != 0) { // symmetric for (int i=0; i<idSize; i++) { int row = theAdjacency(i); if (row > vertexTag) { bool foundPlace = false; // find a place in rowA for current col for (int j=startLoc; j<lastLoc; j++) if (rowA[j] > row) { // move the entries already there one further on // and place col in current location for (int k=lastLoc; k>j; k--) rowA[k] = rowA[k-1]; rowA[j] = row; foundPlace = true; j = lastLoc; } if (foundPlace == false) // put in at the end rowA[lastLoc] = row; lastLoc++; } } } else { // unsymmetric for (int i=0; i<idSize; i++) { int row = theAdjacency(i); bool foundPlace = false; // find a place in rowA for current col for (int j=startLoc; j<lastLoc; j++) if (rowA[j] > row) { // move the entries already there one further on // and place col in current location for (int k=lastLoc; k>j; k--) rowA[k] = rowA[k-1]; rowA[j] = row; foundPlace = true; j = lastLoc; } if (foundPlace == false) // put in at the end rowA[lastLoc] = row; lastLoc++; } } } colStartA[a+1] = lastLoc; startLoc = lastLoc; } } // fill in colA int count = 0; for (int i=0; i<size; i++) { for (int k=colStartA[i]; k<colStartA[i+1]; k++) colA[count++] = i; } LinearSOESolver *theSolvr = this->getSolver(); int solverOK = theSolvr->setSize(); if (solverOK < 0) { opserr << "WARNING:MumpsParallelSOE::setSize :"; opserr << " solver failed setSize()\n"; return solverOK; } return result; }
int MumpsParallelSOE::sendSelf(int commitTag, Channel &theChannel) { int sendID =0; // if P0 check if already sent. If already sent use old processID; if not allocate a new process // id for remote part of object, enlarge channel * to hold a channel * for this remote object. // if not P0, send current processID if (processID == 0) { // check if already using this object bool found = false; for (int i=0; i<numChannels; i++) if (theChannels[i] == &theChannel) { sendID = i+1; found = true; } // if new object, enlarge Channel pointers to hold new channel * & allocate new ID if (found == false) { int nextNumChannels = numChannels + 1; Channel **nextChannels = new Channel *[nextNumChannels]; if (nextNumChannels == 0) { opserr << "MumpsParallelSOE::sendSelf() - failed to allocate channel array of size: " << nextNumChannels << endln; return -1; } for (int i=0; i<numChannels; i++) nextChannels[i] = theChannels[i]; nextChannels[numChannels] = &theChannel; numChannels = nextNumChannels; if (theChannels != 0) delete [] theChannels; theChannels = nextChannels; if (localCol != 0) delete [] localCol; localCol = new ID *[numChannels]; if (localCol == 0) { opserr << "MumpsParallelSOE::sendSelf() - failed to allocate id array of size: " << nextNumChannels << endln; return -1; } for (int i=0; i<numChannels; i++) localCol[i] = 0; // allocate new processID for remote object sendID = numChannels; } } else sendID = processID; // send remotes processID ID idData(2); idData(0) = sendID; idData(1) = matType; int res = theChannel.sendID(0, commitTag, idData); if (res < 0) { opserr <<"WARNING MumpsParallelSOE::sendSelf() - failed to send data\n"; return -1; } LinearSOESolver *theSoeSolver = this->getSolver(); if (theSoeSolver != 0) { if (theSoeSolver->sendSelf(commitTag, theChannel) < 0) { opserr <<"WARNING MumpsParallelSOE::sendSelf() - failed to send solver\n"; return -1; } } else { opserr <<"WARNING MumpsParallelSOE::sendSelf() - no solver to send!\n"; return -1; } return 0; }