int
StructuralEngngModel :: exchangeRemoteElementData(int ExchangeTag)
{
int result = 1;
if ( isParallel() && nonlocalExt ) {
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT( "StructuralEngngModel :: exchangeRemoteElementData", "Packing remote element data", this->giveRank() );
#endif
result &= nonlocCommunicator->packAllData( ( StructuralEngngModel * ) this, & StructuralEngngModel :: packRemoteElementData );
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT( "StructuralEngngModel :: exchangeRemoteElementData", "Remote element data exchange started", this->giveRank() );
#endif
result &= nonlocCommunicator->initExchange(ExchangeTag);
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT( "StructuralEngngModel :: exchangeRemoteElementData", "Receiveng and Unpacking remote element data", this->giveRank() );
#endif
if ( !( result &= nonlocCommunicator->unpackAllData( ( StructuralEngngModel * ) this, & StructuralEngngModel :: unpackRemoteElementData ) ) ) {
_error("StructuralEngngModel :: exchangeRemoteElementData: Receiveng and Unpacking remote element data");
}
result &= nonlocCommunicator->finishExchange();
}
return result;
}
int
StructuralEngngModel :: updateSharedPrescribedDofManagers(FloatArray &answer, int ExchangeTag)
{
int result = 1;
if ( isParallel() ) {
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT( "StructuralEngngModel :: updateSharedPrescribedDofManagers", "Packing data", this->giveRank() );
#endif
result &= communicator->packAllData( ( StructuralEngngModel * ) this, & answer, & StructuralEngngModel :: packPrescribedDofManagers );
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT( "StructuralEngngModel :: updateSharedPrescribedDofManagers", "Exchange started", this->giveRank() );
#endif
result &= communicator->initExchange(ExchangeTag);
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT( "StructuralEngngModel :: updateSharedDofManagers", "Receiving and unpacking", this->giveRank() );
#endif
result &= communicator->unpackAllData( ( StructuralEngngModel * ) this, & answer, & StructuralEngngModel :: unpackPrescribedDofManagers );
result &= communicator->finishExchange();
}
return result;
}
void
ProblemCommunicator :: setUpCommunicationMaps(EngngModel *pm, bool excludeSelfCommFlag,
bool forceReinit)
{
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("ProblemCommunicator :: setUpCommunicationMaps", "Setting up communication maps", rank);
#endif
if ( !forceReinit && initialized ) {
return;
}
if ( this->mode == ProblemCommMode__NODE_CUT ) {
setUpCommunicationMapsForNodeCut(pm, excludeSelfCommFlag);
} else if ( this->mode == ProblemCommMode__ELEMENT_CUT ) {
setUpCommunicationMapsForElementCut(pm, excludeSelfCommFlag);
} else if ( this->mode == ProblemCommMode__REMOTE_ELEMENT_MODE ) {
setUpCommunicationMapsForRemoteElementMode(pm, excludeSelfCommFlag);
} else {
_error("setUpCommunicationMaps: unknown mode");
}
initialized = true;
}
void
FETICommunicator :: setUpCommunicationMaps(EngngModel *pm)
{
int i, j, l, maxRec;
int globaldofmannum, localNumber, ndofs;
int numberOfBoundaryDofMans;
int source, tag;
IntArray numberOfPartitionBoundaryDofMans(size);
StaticCommunicationBuffer commBuff(MPI_COMM_WORLD);
EModelDefaultEquationNumbering dn;
// FETIBoundaryDofManager *dofmanrec;
// Map containing boundary dof managers records, the key is corresponding global number
// value is corresponding local master dof manager number
map< int, int, less< int > >BoundaryDofManagerMap;
// communication maps of slaves
IntArray **commMaps = new IntArray * [ size ];
// location array
IntArray locNum;
Domain *domain = pm->giveDomain(1);
// check if receiver is master
if ( this->rank != 0 ) {
_error("FETICommunicator::setUpCommunicationMaps : rank 0 (master) expected as receiver");
}
// resize receive buffer
commBuff.resize( commBuff.givePackSize(MPI_INT, 1) );
//
// receive data
//
for ( i = 1; i < size; i++ ) {
commBuff.iRecv(MPI_ANY_SOURCE, FETICommunicator :: NumberOfBoundaryDofManagersMsg);
while ( !commBuff.testCompletion(source, tag) ) {
;
}
// unpack data
commBuff.unpackInt(j);
#ifdef __VERBOSE_PARALLEL
OOFEM_LOG_DEBUG("[process rank %3d]: %-30s: Received data from partition %3d (received %d)\n",
rank, "FETICommunicator :: setUpCommunicationMaps : received number of boundary dofMans", source, j);
#endif
numberOfPartitionBoundaryDofMans.at(source + 1) = j;
commBuff.init();
}
MPI_Barrier(MPI_COMM_WORLD);
// determine the total number of boundary dof managers at master
int nnodes = domain->giveNumberOfDofManagers();
j = 0;
for ( i = 1; i <= nnodes; i++ ) {
if ( domain->giveDofManager(i)->giveParallelMode() == DofManager_shared ) {
j++;
}
}
numberOfPartitionBoundaryDofMans.at(1) = j;
//
// receive list of bounadry dof managers with corresponding number of dofs from each partition
//
// resize the receive buffer to fit all messages
maxRec = 0;
for ( i = 0; i < size; i++ ) {
if ( numberOfPartitionBoundaryDofMans.at(i + 1) > maxRec ) {
maxRec = numberOfPartitionBoundaryDofMans.at(i + 1);
}
}
commBuff.resize( 2 * maxRec * commBuff.givePackSize(MPI_INT, 1) );
// resize communication maps acordingly
for ( i = 0; i < size; i++ ) {
j = numberOfPartitionBoundaryDofMans.at(i + 1);
commMaps [ i ] = new IntArray(j);
}
// add local master contribution first
// loop over all dofmanager data received
i = 0;
for ( j = 1; j <= numberOfPartitionBoundaryDofMans.at(1); j++ ) {
// fing next shared dofman
while ( !( domain->giveDofManager(++i)->giveParallelMode() == DofManager_shared ) ) {
;
}
globaldofmannum = domain->giveDofManager(i)->giveGlobalNumber();
domain->giveDofManager(i)->giveCompleteLocationArray(locNum, dn);
ndofs = 0;
for ( l = 1; l <= locNum.giveSize(); l++ ) {
if ( locNum.at(l) ) {
ndofs++;
}
}
// add corresponding entry to master map of boundary dof managers
if ( ( localNumber = BoundaryDofManagerMap [ globaldofmannum ] ) == 0 ) { // no local counterpart exist
// create it
boundaryDofManList.push_back( FETIBoundaryDofManager(globaldofmannum, 0, ndofs) );
// remember the local number; actual position in vector is localNumber-1
localNumber = BoundaryDofManagerMap [ globaldofmannum ] = ( boundaryDofManList.size() );
boundaryDofManList.back().addPartition(0);
} else { // update the corresponding record
boundaryDofManList [ localNumber - 1 ].addPartition(0);
if ( boundaryDofManList [ localNumber - 1 ].giveNumberOfDofs() != ndofs ) {
_error("FETICommunicator :: setUpCommunicationMaps : ndofs size mismatch");
}
}
// remember communication map for particular partition
commMaps [ 0 ]->at(j) = localNumber;
}
//
// receive data from slave partitions
//
for ( i = 1; i < size; i++ ) {
commBuff.iRecv(MPI_ANY_SOURCE, FETICommunicator :: BoundaryDofManagersRecMsg);
while ( !commBuff.testCompletion(source, tag) ) {
;
}
// unpack data
#ifdef __VERBOSE_PARALLEL
OOFEM_LOG_DEBUG("[process rank %3d]: %-30s: Received data from partition %3d\n",
rank, "FETICommunicator :: setUpCommunicationMaps : received boundary dofMans records", source);
#endif
// loop over all dofmanager data received
for ( j = 1; j <= numberOfPartitionBoundaryDofMans.at(source + 1); j++ ) {
commBuff.unpackInt(globaldofmannum);
commBuff.unpackInt(ndofs);
// add corresponding entry to master map of boundary dof managers
if ( ( localNumber = BoundaryDofManagerMap [ globaldofmannum ] ) == 0 ) { // no local counterpart exist
// create it
boundaryDofManList.push_back( FETIBoundaryDofManager(globaldofmannum, 0, ndofs) );
// remember the local number; actual position in vector is localNumber-1
localNumber = BoundaryDofManagerMap [ globaldofmannum ] = ( boundaryDofManList.size() );
boundaryDofManList.back().addPartition(source);
} else { // update the corresponding record
boundaryDofManList [ localNumber - 1 ].addPartition(source);
if ( boundaryDofManList [ localNumber - 1 ].giveNumberOfDofs() != ndofs ) {
_error("FETICommunicator :: setUpCommunicationMaps : ndofs size mismatch");
}
}
// remember communication map for particular partition
commMaps [ source ]->at(j) = localNumber;
}
commBuff.init();
}
MPI_Barrier(MPI_COMM_WORLD);
//
// assign code numbers to boundary dofs
//
numberOfEquations = 0;
numberOfBoundaryDofMans = boundaryDofManList.size();
for ( i = 1; i <= numberOfBoundaryDofMans; i++ ) {
boundaryDofManList [ i - 1 ].setCodeNumbers(numberOfEquations); // updates numberOfEquations
}
// store the commMaps
for ( i = 0; i < size; i++ ) {
if ( i != 0 ) {
this->giveProcessCommunicator(i)->setToSendArry(engngModel, * commMaps [ i ], 0);
this->giveProcessCommunicator(i)->setToRecvArry(engngModel, * commMaps [ i ], 0);
} else {
masterCommMap = * commMaps [ i ];
}
delete commMaps [ i ];
}
delete commMaps;
MPI_Barrier(MPI_COMM_WORLD);
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("FETICommunicator::setUpCommunicationMaps", "communication maps setup finished", rank);
#endif
}
void
NlDEIDynamic :: computeMassMtrx(FloatArray &massMatrix, double &maxOm, TimeStep *tStep)
{
Domain *domain = this->giveDomain(1);
int nelem = domain->giveNumberOfElements();
int neq = this->giveNumberOfDomainEquations( 1, EModelDefaultEquationNumbering() );
int i, j, jj, n;
double maxOmi, maxOmEl;
FloatMatrix charMtrx, charMtrx2, R;
IntArray loc;
Element *element;
EModelDefaultEquationNumbering en;
#ifndef LOCAL_ZERO_MASS_REPLACEMENT
FloatArray diagonalStiffMtrx;
#endif
maxOm = 0.;
massMatrix.resize(neq);
massMatrix.zero();
for ( i = 1; i <= nelem; i++ ) {
element = domain->giveElement(i);
// skip remote elements (these are used as mirrors of remote elements on other domains
// when nonlocal constitutive models are used. They introduction is necessary to
// allow local averaging on domains without fine grain communication between domains).
if ( element->giveParallelMode() == Element_remote ) {
continue;
}
element->giveLocationArray(loc, en);
element->giveCharacteristicMatrix(charMtrx, LumpedMassMatrix, tStep);
if ( charMtrx.isNotEmpty() ) {
///@todo This rotation matrix is not flexible enough.. it can only work with full size matrices and doesn't allow for flexibility in the matrixassembler.
if ( element->giveRotationMatrix(R) ) {
charMtrx.rotatedWith(R);
}
}
#ifdef LOCAL_ZERO_MASS_REPLACEMENT
element->giveCharacteristicMatrix(charMtrx2, TangentStiffnessMatrix, tStep);
if ( charMtrx2.isNotEmpty() ) {
///@todo This rotation matrix is not flexible enough.. it can only work with full size matrices and doesn't allow for flexibility in the matrixassembler.
if ( R.isNotEmpty() ) {
charMtrx2.rotatedWith(R);
}
}
#endif
#ifdef DEBUG
if ( loc.giveSize() != charMtrx.giveNumberOfRows() ) {
OOFEM_ERROR("dimension mismatch");
}
#endif
n = loc.giveSize();
#ifdef LOCAL_ZERO_MASS_REPLACEMENT
maxOmEl = 0.;
double maxElmass = -1.0;
for ( j = 1; j <= n; j++ ) {
maxElmass = max( maxElmass, charMtrx.at(j, j) );
}
if ( maxElmass <= 0.0 ) {
OOFEM_WARNING("Element (%d) with zero (or negative) lumped mass encountered\n", i);
} else {
if (charMtrx2.isNotEmpty() ) {
// in case stifness matrix defined, we can generate artificial mass
// in those DOFs without mass
for ( j = 1; j <= n; j++ ) {
if ( charMtrx.at(j, j) > maxElmass * ZERO_REL_MASS ) {
maxOmi = charMtrx2.at(j, j) / charMtrx.at(j, j);
maxOmEl = ( maxOmEl > maxOmi ) ? ( maxOmEl ) : ( maxOmi );
}
}
maxOm = ( maxOm > maxOmEl ) ? ( maxOm ) : ( maxOmEl );
for ( j = 1; j <= n; j++ ) {
jj = loc.at(j);
if ( ( jj ) && ( charMtrx.at(j, j) <= maxElmass * ZERO_REL_MASS ) ) {
charMtrx.at(j, j) = charMtrx2.at(j, j) / maxOmEl;
}
}
}
}
#endif
for ( j = 1; j <= n; j++ ) {
jj = loc.at(j);
if ( jj ) {
massMatrix.at(jj) += charMtrx.at(j, j);
}
}
}
#ifndef LOCAL_ZERO_MASS_REPLACEMENT
// If init step - find minimun period of vibration in order to
// determine maximal admisible time step
// global variant
for ( i = 1; i <= nelem; i++ ) {
element = domain->giveElement(i);
element->giveLocationArray(loc, en);
element->giveCharacteristicMatrix(charMtrx, TangentStiffnessMatrix, tStep);
if ( charMtrx.isNotEmpty() ) {
///@todo This rotation matrix is not flexible enough.. it can only work with full size matrices and doesn't allow for flexibility in the matrixassembler.
if ( element->giveRotationMatrix(R) ) {
charMtrx.rotatedWith(R);
}
}
n = loc.giveSize();
for ( j = 1; j <= n; j++ ) {
jj = loc.at(j);
if ( jj ) {
diagonalStiffMtrx.at(jj) += charMtrx.at(j, j);
}
}
}
// Find find global minimun period of vibration
double maxElmass = -1.0;
for ( j = 1; j <= n; j++ ) {
maxElmass = max( maxElmass, charMtrx.at(j, j) );
}
if ( maxElmass <= 0.0 ) {
OOFEM_ERROR("Element with zero (or negative) lumped mass encountered");
}
for ( j = 1; j <= neq; j++ ) {
if ( massMatrix.at(j) > maxElmass * ZERO_REL_MASS ) {
maxOmi = diagonalStiffMtrx.at(j) / massMatrix.at(j);
maxOm = ( maxOm > maxOmi ) ? ( maxOm ) : ( maxOmi );
}
}
// Set ZERO MASS members in massMatrix to value which corresponds to global maxOm.
for ( i = 1; i <= neq; i++ ) {
if ( massMatrix.at(i) <= maxElmass * ZERO_REL_MASS ) {
massMatrix.at(i) = diagonalStiffMtrx.at(i) / maxOm;
}
}
#endif
this->updateSharedDofManagers(massMatrix, EModelDefaultEquationNumbering(), MassExchangeTag);
#ifdef __PARALLEL_MODE
// Determine maxOm over all processes.
#ifdef __USE_MPI
double globalMaxOm;
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT( "NlDEIDynamic :: computeMassMtrx", "Reduce of maxOm started", this->giveRank() );
#endif
int result = MPI_Allreduce(& maxOm, & globalMaxOm, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT( "NlDEIDynamic :: computeMassMtrx", "Reduce of maxOm finished", this->giveRank() );
#endif
if ( result != MPI_SUCCESS ) {
OOFEM_ERROR("MPI_Allreduce failed");
}
maxOm = globalMaxOm;
#else
WARNING: NOT SUPPORTED MESSAGE PARSING LIBRARY
#endif
#endif
}
void
PetscNatural2GlobalOrdering :: init(EngngModel *emodel, EquationID ut, int di, EquationType et)
{
Domain *d = emodel->giveDomain(di);
int i, j, k, p, ndofs, ndofman = d->giveNumberOfDofManagers();
int myrank = emodel->giveRank();
DofManager *dman;
// determine number of local eqs + number of those shared DOFs which are numbered by receiver
// shared dofman is numbered on partition with lovest rank number
EModelDefaultEquationNumbering dn;
EModelDefaultPrescribedEquationNumbering dpn;
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("PetscNatural2GlobalOrdering :: init", "initializing N2G ordering", myrank);
#endif
l_neqs = 0;
for ( i = 1; i <= ndofman; i++ ) {
dman = d->giveDofManager(i);
/*
* if (dman->giveParallelMode() == DofManager_local) { // count all dofman eqs
* ndofs = dman->giveNumberOfDofs ();
* for (j=1; j<=ndofs; j++) {
* if (dman->giveDof(j)->isPrimaryDof()) {
* if (dman->giveDof(j)->giveEquationNumber()) l_neqs++;
* }
* }
* } else if (dman->giveParallelMode() == DofManager_shared) {
* // determine if problem is the lowest one sharing the dofman; if yes the receiver is responsible to
* // deliver number
* IntArray *plist = dman->givePartitionList();
* int n = plist->giveSize();
* int minrank = myrank;
* for (j=1; j<=n; j++) minrank = min (minrank, plist->at(j));
* if (minrank == myrank) { // count eqs
* ndofs = dman->giveNumberOfDofs ();
* for (j=1; j<=ndofs; j++) {
* if (dman->giveDof(j)->isPrimaryDof()) {
* if (dman->giveDof(j)->giveEquationNumber()) l_neqs++;
* }
* }
* }
* } // end shared dman
*/
if ( isLocal(dman) ) {
ndofs = dman->giveNumberOfDofs();
for ( j = 1; j <= ndofs; j++ ) {
if ( dman->giveDof(j)->isPrimaryDof() ) {
if ( et == et_standard ) {
if ( dman->giveDof(j)->giveEquationNumber(dn) ) {
l_neqs++;
}
} else {
if ( dman->giveDof(j)->giveEquationNumber(dpn) ) {
l_neqs++;
}
}
}
}
}
}
// exchange with other procs the number of eqs numbered on particular procs
int *leqs = new int [ emodel->giveNumberOfProcesses() ];
MPI_Allgather(& l_neqs, 1, MPI_INT, leqs, 1, MPI_INT, MPI_COMM_WORLD);
// compute local offset
int offset = 0;
for ( j = 0; j < myrank; j++ ) {
offset += leqs [ j ];
}
// count global number of eqs
for ( g_neqs = 0, j = 0; j < emodel->giveNumberOfProcesses(); j++ ) {
g_neqs += leqs [ j ];
}
// send numbered shared ones
if ( et == et_standard ) {
locGlobMap.resize( emodel->giveNumberOfEquations(ut) );
} else {
locGlobMap.resize( emodel->giveNumberOfPrescribedEquations(ut) );
}
// determine shared dofs
int psize, nproc = emodel->giveNumberOfProcesses();
IntArray sizeToSend(nproc), sizeToRecv(nproc), nrecToReceive(nproc);
#ifdef __VERBOSE_PARALLEL
IntArray nrecToSend(nproc);
#endif
const IntArray *plist;
for ( i = 1; i <= ndofman; i++ ) {
// if (domain->giveDofManager(i)->giveParallelMode() == DofManager_shared) {
if ( isShared( d->giveDofManager(i) ) ) {
int n = d->giveDofManager(i)->giveNumberOfDofs();
plist = d->giveDofManager(i)->givePartitionList();
psize = plist->giveSize();
int minrank = myrank;
for ( j = 1; j <= psize; j++ ) {
minrank = min( minrank, plist->at(j) );
}
if ( minrank == myrank ) { // count to send
for ( j = 1; j <= psize; j++ ) {
#ifdef __VERBOSE_PARALLEL
nrecToSend( plist->at(j) )++;
#endif
sizeToSend( plist->at(j) ) += ( 1 + n ); // ndofs+dofman number
}
} else {
nrecToReceive(minrank)++;
sizeToRecv(minrank) += ( 1 + n ); // ndofs+dofman number
}
}
}
#ifdef __VERBOSE_PARALLEL
for ( i = 0; i < nproc; i++ ) {
OOFEM_LOG_INFO("[%d] Record Statistics: Sending %d Receiving %d to %d\n",
myrank, nrecToSend(i), nrecToReceive(i), i);
}
#endif
std :: map< int, int >globloc; // global->local mapping for shared
// number local guys
int globeq = offset;
for ( i = 1; i <= ndofman; i++ ) {
dman = d->giveDofManager(i);
//if (dman->giveParallelMode() == DofManager_shared) {
if ( isShared(dman) ) {
globloc [ dman->giveGlobalNumber() ] = i; // build global->local mapping for shared
plist = dman->givePartitionList();
psize = plist->giveSize();
int minrank = myrank;
for ( j = 1; j <= psize; j++ ) {
minrank = min( minrank, plist->at(j) );
}
if ( minrank == myrank ) { // local
ndofs = dman->giveNumberOfDofs();
for ( j = 1; j <= ndofs; j++ ) {
if ( dman->giveDof(j)->isPrimaryDof() ) {
int eq;
if ( et == et_standard ) {
eq = dman->giveDof(j)->giveEquationNumber(dn);
} else {
eq = dman->giveDof(j)->giveEquationNumber(dpn);
}
if ( eq ) {
locGlobMap.at(eq) = globeq++;
}
}
}
}
//} else if (dman->giveParallelMode() == DofManager_local) {
} else {
ndofs = dman->giveNumberOfDofs();
for ( j = 1; j <= ndofs; j++ ) {
if ( dman->giveDof(j)->isPrimaryDof() ) {
int eq;
if ( et == et_standard ) {
eq = dman->giveDof(j)->giveEquationNumber(dn);
} else {
eq = dman->giveDof(j)->giveEquationNumber(dpn);
}
if ( eq ) {
locGlobMap.at(eq) = globeq++;
}
}
}
}
}
/*
* fprintf (stderr, "[%d] locGlobMap: ", myrank);
* for (i=1; i<=locGlobMap.giveSize(); i++)
* fprintf (stderr, "%d ",locGlobMap.at(i));
*/
// pack data for remote procs
CommunicationBuffer **buffs = new CommunicationBuffer * [ nproc ];
for ( p = 0; p < nproc; p++ ) {
buffs [ p ] = new StaticCommunicationBuffer(MPI_COMM_WORLD, 0);
buffs [ p ]->resize( buffs [ p ]->givePackSize(MPI_INT, 1) * sizeToSend(p) );
#if 0
OOFEM_LOG_INFO( "[%d]PetscN2G:: init: Send buffer[%d] size %d\n",
myrank, p, sizeToSend(p) );
#endif
}
for ( i = 1; i <= ndofman; i++ ) {
if ( isShared( d->giveDofManager(i) ) ) {
dman = d->giveDofManager(i);
plist = dman->givePartitionList();
psize = plist->giveSize();
int minrank = myrank;
for ( j = 1; j <= psize; j++ ) {
minrank = min( minrank, plist->at(j) );
}
if ( minrank == myrank ) { // do send
for ( j = 1; j <= psize; j++ ) {
p = plist->at(j);
if ( p == myrank ) {
continue;
}
#if 0
OOFEM_LOG_INFO("[%d]PetscN2G:: init: Sending localShared node %d[%d] to proc %d\n",
myrank, i, dman->giveGlobalNumber(), p);
#endif
buffs [ p ]->packInt( dman->giveGlobalNumber() );
ndofs = dman->giveNumberOfDofs();
for ( k = 1; k <= ndofs; k++ ) {
if ( dman->giveDof(k)->isPrimaryDof() ) {
int eq;
if ( et == et_standard ) {
eq = dman->giveDof(k)->giveEquationNumber(dn);
} else {
eq = dman->giveDof(k)->giveEquationNumber(dpn);
}
if ( eq ) {
buffs [ p ]->packInt( locGlobMap.at(eq) );
}
}
}
}
}
}
}
//fprintf (stderr, "[%d] Sending glob nums ...", myrank);
// send buffers
for ( p = 0; p < nproc; p++ ) {
if ( p != myrank ) {
buffs [ p ]->iSend(p, 999);
}
}
/****
*
* for (p=0; p<nproc; p++) {
* if (p == myrank) continue;
* for (i=1; i<= ndofman; i++) {
* //if (domain->giveDofManager(i)->giveParallelMode() == DofManager_shared) {
* if (isShared(d->giveDofManager(i))) {
* dman = d->giveDofManager(i);
* plist = dman->givePartitionList();
* psize = plist->giveSize();
* int minrank = myrank;
* for (j=1; j<=psize; j++) minrank = min (minrank, plist->at(j));
* if (minrank == myrank) { // do send
* buffs[p]->packInt(dman->giveGlobalNumber());
* ndofs = dman->giveNumberOfDofs ();
* for (j=1; j<=ndofs; j++) {
* if (dman->giveDof(j)->isPrimaryDof()) {
* buffs[p]->packInt(locGlobMap.at(dman->giveDof(j)->giveEquationNumber()));
* }
* }
* }
* }
* }
* // send buffer
* buffs[p]->iSend(p, 999);
* }
****/
// receive remote eqs and complete global numbering
CommunicationBuffer **rbuffs = new CommunicationBuffer * [ nproc ];
for ( p = 0; p < nproc; p++ ) {
rbuffs [ p ] = new StaticCommunicationBuffer(MPI_COMM_WORLD, 0);
rbuffs [ p ]->resize( rbuffs [ p ]->givePackSize(MPI_INT, 1) * sizeToRecv(p) );
#if 0
OOFEM_LOG_INFO( "[%d]PetscN2G:: init: Receive buffer[%d] size %d\n",
myrank, p, sizeToRecv(p) );
#endif
}
//fprintf (stderr, "[%d] Receiving glob nums ...", myrank);
for ( p = 0; p < nproc; p++ ) {
if ( p != myrank ) {
rbuffs [ p ]->iRecv(p, 999);
}
}
IntArray finished(nproc);
finished.zero();
int fin = 1;
finished.at(emodel->giveRank() + 1) = 1;
do {
for ( p = 0; p < nproc; p++ ) {
if ( finished.at(p + 1) == 0 ) {
if ( rbuffs [ p ]->testCompletion() ) {
// data are here
// unpack them
int nite = nrecToReceive(p);
int shdm, ldm;
for ( i = 1; i <= nite; i++ ) {
rbuffs [ p ]->unpackInt(shdm);
#if 0
OOFEM_LOG_INFO("[%d]PetscN2G:: init: Received shared node [%d] from proc %d\n",
myrank, shdm, p);
#endif
//
// find local guy coorecponding to shdm
if ( globloc.find(shdm) != globloc.end() ) {
ldm = globloc [ shdm ];
} else {
OOFEM_ERROR3("[%d] PetscNatural2GlobalOrdering :: init: invalid shared dofman received, globnum %d\n", myrank, shdm);
}
dman = d->giveDofManager(ldm);
ndofs = dman->giveNumberOfDofs();
for ( j = 1; j <= ndofs; j++ ) {
if ( dman->giveDof(j)->isPrimaryDof() ) {
int eq;
if ( et == et_standard ) {
eq = dman->giveDof(j)->giveEquationNumber(dn);
} else {
eq = dman->giveDof(j)->giveEquationNumber(dpn);
}
if ( eq ) {
int val;
rbuffs [ p ]->unpackInt(val);
locGlobMap.at(eq) = val;
}
}
}
}
finished.at(p + 1) = 1;
fin++;
}
}
}
} while ( fin < nproc );
/*
* fprintf (stderr, "[%d] Finished receiving glob nums ...", myrank);
*
* fprintf (stderr, "[%d] locGlobMap:", myrank);
* for (i=1; i<=locGlobMap.giveSize(); i++)
* fprintf (stderr, "%d ",locGlobMap.at(i));
*/
#ifdef __VERBOSE_PARALLEL
if ( et == et_standard ) {
int _eq;
char *ptr;
char *locname = "local", *shname = "shared", *unkname = "unknown";
for ( i = 1; i <= ndofman; i++ ) {
dman = d->giveDofManager(i);
if ( dman->giveParallelMode() == DofManager_local ) {
ptr = locname;
} else if ( dman->giveParallelMode() == DofManager_shared ) {
ptr = shname;
} else {
ptr = unkname;
}
ndofs = dman->giveNumberOfDofs();
for ( j = 1; j <= ndofs; j++ ) {
if ( ( _eq = dman->giveDof(j)->giveEquationNumber(dn) ) ) {
fprintf( stderr, "[%d] n:%6s %d[%d] (%d), leq = %d, geq = %d\n", emodel->giveRank(), ptr, i, dman->giveGlobalNumber(), j, _eq, locGlobMap.at(_eq) );
} else {
fprintf(stderr, "[%d] n:%6s %d[%d] (%d), leq = %d, geq = %d\n", emodel->giveRank(), ptr, i, dman->giveGlobalNumber(), j, _eq, 0);
}
}
}
}
#endif
// build reverse map
int lneq;
if ( et == et_standard ) {
lneq = emodel->giveNumberOfEquations(ut);
} else {
lneq = emodel->giveNumberOfPrescribedEquations(ut);
}
globLocMap.clear();
for ( i = 1; i <= lneq; i++ ) {
globLocMap [ locGlobMap.at(i) ] = i;
}
for ( p = 0; p < nproc; p++ ) {
delete rbuffs [ p ];
delete buffs [ p ];
}
delete[] rbuffs;
delete[] buffs;
delete[] leqs;
MPI_Barrier(MPI_COMM_WORLD);
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("PetscNatural2GlobalOrdering :: init", "done", myrank);
#endif
}
void
ProblemCommunicator :: setUpCommunicationMapsForRemoteElementMode(EngngModel *pm,
bool excludeSelfCommFlag)
{
//int nnodes = domain->giveNumberOfDofManagers();
Domain *domain = pm->giveDomain(1);
int i, j, partition;
if ( this->mode == ProblemCommMode__REMOTE_ELEMENT_MODE ) {
/*
* Initially, each partition knows for which nodes a receive
* is needed (and can therefore compute easily the recv map),
* but does not know for which nodes it should send data to which
* partition. Hence, the communication setup is performed by
* broadcasting "send request" lists of nodes for which
* a partition expects to receive data (ie. of those nodes
* which the partition uses, but does not own) to all
* collaborating processes. The "send request" list are
* converted into send maps.
*/
// receive maps can be build locally,
// but send maps should be assembled from broadcasted lists (containing
// expected receive nodes) of remote partitions.
// first build local receive map
IntArray domainNodeRecvCount(size);
const IntArray *partitionList;
//DofManager *dofMan;
Element *element;
int domainRecvListSize = 0, domainRecvListPos = 0;
int nelems;
int result = 1;
nelems = domain->giveNumberOfElements();
for ( i = 1; i <= nelems; i++ ) {
partitionList = domain->giveElement(i)->givePartitionList();
if ( domain->giveElement(i)->giveParallelMode() == Element_remote ) {
// size of partitionList should be 1 <== only ine master
for ( j = 1; j <= partitionList->giveSize(); j++ ) {
if ( !( excludeSelfCommFlag && ( this->rank == partitionList->at(j) ) ) ) {
domainRecvListSize++;
domainNodeRecvCount.at(partitionList->at(j) + 1)++;
}
}
}
}
// build maps simultaneously
IntArray pos(size);
IntArray **maps = new IntArray * [ size ];
for ( i = 0; i < size; i++ ) {
maps [ i ] = new IntArray( domainNodeRecvCount.at(i + 1) );
}
// allocate also domain receive list to be broadcasted
IntArray domainRecvList(domainRecvListSize);
if ( domainRecvListSize ) {
for ( i = 1; i <= nelems; i++ ) {
// test if element is remote one
element = domain->giveElement(i);
if ( element->giveParallelMode() == Element_remote ) {
domainRecvList.at(++domainRecvListPos) = element->giveGlobalNumber();
partitionList = domain->giveElement(i)->givePartitionList();
// size of partitionList should be 1 <== only ine master
for ( j = 1; j <= partitionList->giveSize(); j++ ) {
if ( !( excludeSelfCommFlag && ( this->rank == partitionList->at(j) ) ) ) {
partition = partitionList->at(j);
maps [ partition ]->at( ++pos.at(partition + 1) ) = i;
}
}
}
}
}
// set up domains recv communicator maps
for ( i = 0; i < size; i++ ) {
this->setProcessCommunicatorToRecvArry(this->giveProcessCommunicator(i), * maps [ i ]);
//this->giveDomainCommunicator(i)->setToRecvArry (this->engngModel, *maps[i]);
}
/*
* #ifdef __VERBOSE_PARALLEL
* for (i=0; i<size; i++) {
* fprintf (stderr, "domain %d-%d: domainCommRecvsize is %d\n",rank,i,this->giveDomainCommunicator(i)->giveRecvBuff()->giveSize() );
* printf ("domain %d-%d: reecv map:",rank,i);
* this->giveDomainCommunicator(i)->giveToRecvMap()->printYourself();
* }
*#endif
*/
// delete local maps
for ( i = 0; i < size; i++ ) {
delete maps [ i ];
}
delete [] maps;
// to assemble send maps, we must analyze broadcasted remote domain send lists
// and we must also broadcast our send list.
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("ProblemCommunicator::setUpCommunicationMaps", "Remote Element-cut broadcasting started", rank);
#endif
StaticCommunicationBuffer commBuff(MPI_COMM_WORLD);
IntArray remoteDomainRecvList;
IntArray toSendMap;
int localExpectedSize, globalRecvSize;
int sendMapPos, sendMapSize, globalDofManNum;
// determine the size of receive buffer using AllReduce operation
#ifndef IBM_MPI_IMPLEMENTATION
localExpectedSize = domainRecvList.givePackSize(commBuff);
#else
localExpectedSize = domainRecvList.givePackSize(commBuff) + 1;
#endif
#ifdef __USE_MPI
result = MPI_Allreduce(& localExpectedSize, & globalRecvSize, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
if ( result != MPI_SUCCESS ) {
_error("setUpCommunicationMaps: MPI_Allreduce failed");
}
#else
WARNING: NOT SUPPORTED MESSAGE PARSING LIBRARY
#endif
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("ProblemCommunicator::setUpCommunicationMaps", "Finished reducing receiveBufferSize", rank);
#endif
// resize to fit largest received message
commBuff.resize(globalRecvSize);
// resize toSend map to max possible size
toSendMap.resize(globalRecvSize);
for ( i = 0; i < size; i++ ) { // loop over domains
commBuff.init();
if ( i == rank ) {
//current domain has to send its receive list to all domains
// broadcast domainRecvList
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("ProblemCommunicator::setUpCommunicationMaps", "Broadcasting own send list", rank);
#endif
commBuff.packIntArray(domainRecvList);
result = commBuff.bcast(i);
if ( result != MPI_SUCCESS ) {
_error("setUpCommunicationMaps: commBuff broadcast failed");
}
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("ProblemCommunicator::setUpCommunicationMaps", "Broadcasting own send list finished", rank);
#endif
} else {
#ifdef __VERBOSE_PARALLEL
OOFEM_LOG_DEBUG("[process rank %3d]: %-30s: Receiving broadcasted send map from partition %3d\n",
rank, "ProblemCommunicator :: unpackAllData", i);
#endif
// receive broadcasted lists
result = commBuff.bcast(i);
if ( result != MPI_SUCCESS ) {
_error("setUpCommunicationMaps: commBuff broadcast failed");
}
#ifdef __VERBOSE_PARALLEL
OOFEM_LOG_DEBUG("[process rank %3d]: %-30s: Receiving broadcasted send map from partition %3d finished\n",
rank, "ProblemCommunicator :: unpackAllData", i);
#endif
// unpack remote receive list
if ( !commBuff.unpackIntArray(remoteDomainRecvList) ) {
_error("ProblemCommunicator::setUpCommunicationMaps: unpack remote receive list failed");
}
// find if remote elements are in local partition
// if yes add them into send map for correcponding i-th partition
sendMapPos = 0;
sendMapSize = 0;
// determine sendMap size
for ( j = 1; j <= nelems; j++ ) { // loop over local elements
element = domain->giveElement(j);
if ( element->giveParallelMode() == Element_local ) {
globalDofManNum = element->giveGlobalNumber();
// test id globalDofManNum is in remoteDomainRecvList
if ( remoteDomainRecvList.findFirstIndexOf(globalDofManNum) ) {
sendMapSize++;
}
}
}
toSendMap.resize(sendMapSize);
for ( j = 1; j <= nelems; j++ ) { // loop over local elements
element = domain->giveElement(j);
if ( element->giveParallelMode() == Element_local ) {
globalDofManNum = element->giveGlobalNumber();
// test id globalDofManNum is in remoteDomainRecvList
if ( remoteDomainRecvList.findFirstIndexOf(globalDofManNum) ) {
// add this local DofManager number to sed map for active partition
toSendMap.at(++sendMapPos) = j;
}
}
} // end loop over local DofManagers
// set send map to i-th process communicator
this->setProcessCommunicatorToSendArry(this->giveProcessCommunicator(i), toSendMap);
/*
* #ifdef __VERBOSE_PARALLEL
* fprintf (stderr, "domain %d-%d: domainCommSendsize is %d\n",rank,i,this->giveDomainCommunicator(i)->giveSendBuff()->giveSize() );
* printf ("domain %d-%d: send map:",rank,i);
* this->giveDomainCommunicator(i)->giveToSendMap()->printYourself();
*
*#endif
*/
//this->giveDomainCommunicator(i)->setToSendArry (this->engngModel, toSendMap);
} // end receiving broadcasted lists
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("ProblemCommunicator::setUpCommunicationMaps", "Receiving broadcasted send maps finished", rank);
#endif
} // end loop over domains
} else {
_error("setUpCommunicationMapsForRemoteElementMode: unknown mode");
}
}
/*
* should be called after basic local migration is finalized,
* when all local elements are already available
*/
void
NonlocalMaterialWTP :: migrate()
{
Domain *domain = this->lb->giveDomain();
EngngModel *emodel = domain->giveEngngModel();
int nproc = emodel->giveNumberOfProcesses();
int myrank = emodel->giveRank();
CommunicatorBuff cb(nproc, CBT_dynamic);
Communicator com(emodel, &cb, myrank, nproc, CommMode_Dynamic);
StaticCommunicationBuffer commBuff(MPI_COMM_WORLD);
/*
* build domain nonlocal element dependency list. Then exclude local elements - what remains are unsatisfied
* remote dependencies that have to be broadcasted and received from partitions owning relevant elements
*/
int _locsize, i, _i, ie, _size, _globnum, result, nelems = domain->giveNumberOfElements();
int _globsize, _val;
Element *elem;
std :: set< int >domainElementDepSet;
// loop over each element dep list to assemble domain list
for ( ie = 1; ie <= nelems; ie++ ) {
elem = domain->giveElement(ie);
if ( ( elem->giveParallelMode() == Element_local ) ) {
_globnum = elem->giveGlobalNumber();
IntArray &iedep = nonlocElementDependencyMap [ _globnum ];
_size = iedep.giveSize();
for ( _i = 1; _i <= _size; _i++ ) {
domainElementDepSet.insert( iedep.at(_i) );
}
#if NonlocalMaterialWTP_DEBUG_PRINT
fprintf(stderr, "[%d] element %d dependency:", myrank, _globnum);
for ( _i = 1; _i <= _size; _i++ ) {
fprintf( stderr, "%d ", iedep.at(_i) );
}
fprintf(stderr, "\n");
#endif
}
}
#if NonlocalMaterialWTP_DEBUG_PRINT
fprintf(stderr, "[%d] nonlocal domain dependency:", myrank);
for ( int eldep: domainElementDepSet ) {
fprintf(stderr, "%d ", eldep);
}
fprintf(stderr, "\n");
#endif
// now exclude local elements (local dependency is always satisfied)
for ( _i = 1; _i <= nelems; _i++ ) {
elem = domain->giveElement(_i);
if ( elem->giveParallelMode() == Element_local ) {
domainElementDepSet.erase( elem->giveGlobalNumber() );
}
}
#if NonlocalMaterialWTP_DEBUG_PRINT
fprintf(stderr, "[%d] remote elem wish list:", myrank);
for ( int eldep: domainElementDepSet ) {
fprintf(stderr, "%d ", eldep);
}
fprintf(stderr, "\n");
#endif
// broadcast remaining elements (unsatisfied domain nonlocal dependency) to remaining partitions
_locsize = domainElementDepSet.size() + 1;
result = MPI_Allreduce(& _locsize, & _globsize, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
if ( result != MPI_SUCCESS ) {
OOFEM_ERROR("MPI_Allreduce to determine broadcast buffer size failed");
}
commBuff.resize( commBuff.givePackSize(MPI_INT, _globsize) );
// remote domain wish list
std :: set< int >remoteWishSet;
toSendList.resize(nproc);
for ( i = 0; i < nproc; i++ ) { // loop over partitions
commBuff.init();
toSendList [ i ].clear();
if ( i == myrank ) {
// current domain has to send its receive wish list to all domains
commBuff.packInt(_locsize);
for ( int eldep: domainElementDepSet ) {
commBuff.packInt(eldep);
}
result = commBuff.bcast(i);
} else {
// unpack remote domain wish list
remoteWishSet.clear();
result = commBuff.bcast(i);
// unpack size
commBuff.unpackInt(_size);
for ( _i = 1; _i < _size; _i++ ) {
commBuff.unpackInt(_val);
remoteWishSet.insert(_val);
}
// determine which local elements are to be sent to remotepartition
for ( _i = 1; _i <= nelems; _i++ ) {
elem = domain->giveElement(_i);
if ( elem->giveParallelMode() == Element_local ) {
if ( remoteWishSet.find( elem->giveGlobalNumber() ) != remoteWishSet.end() ) {
// store local element number
toSendList [ i ].push_back(_i);
}
}
}
}
} // end loop over partitions broadcast
#if NonlocalMaterialWTP_DEBUG_PRINT
for ( i = 0; i < nproc; i++ ) { // loop over partitions
// print some info
fprintf(stderr, "[%d] elements scheduled for mirroring at [%d]:",
myrank, i);
for ( int elnum: toSendList [ i ] ) {
fprintf( stderr, "%d[%d] ", elnum, domain->giveElement(elnum)->giveGlobalNumber() );
}
fprintf(stderr, "\n");
}
#endif
com.packAllData(this, domain, & NonlocalMaterialWTP :: packRemoteElements);
com.initExchange(MIGRATE_REMOTE_ELEMENTS_TAG);
com.unpackAllData(this, domain, & NonlocalMaterialWTP :: unpackRemoteElements);
com.finishExchange();
domain->commitTransactions( domain->giveTransactionManager() );
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("NonlocalMaterialWTP::migrate", "Finished migrating remote elements", myrank);
#endif
}
int
PetscSparseMtrx :: buildInternalStructure(EngngModel *eModel, int di, EquationID ut, const UnknownNumberingScheme &s)
{
IntArray loc;
Domain *domain = eModel->giveDomain(di);
int nelem;
if ( mtrx ) {
MatDestroy(&mtrx);
}
if ( this->kspInit ) {
KSPDestroy(&ksp);
this->kspInit = false; // force ksp to be initialized
}
this->ut = ut;
this->emodel = eModel;
this->di = di;
#ifdef __PARALLEL_MODE
if ( eModel->isParallel() ) {
int rank;
PetscNatural2GlobalOrdering *n2g;
PetscNatural2LocalOrdering *n2l;
rank = eModel->giveRank();
n2g = eModel->givePetscContext(di, ut)->giveN2Gmap();
n2l = eModel->givePetscContext(di, ut)->giveN2Lmap();
n2l->init(eModel, ut, di);
n2g->init(eModel, ut, di);
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("PetscSparseMtrx:: buildInternalStructure", "", rank);
#endif
leqs = n2g->giveNumberOfLocalEqs();
geqs = n2g->giveNumberOfGlobalEqs();
//printf("%d, leqs = %d, geqs = %d\n", this->emodel->giveRank(), leqs, geqs);
#ifdef __VERBOSE_PARALLEL
OOFEM_LOG_INFO( "[%d]PetscSparseMtrx:: buildInternalStructure: l_eqs = %d, g_eqs = %d, n_eqs = %d\n", rank, leqs, geqs, eModel->giveNumberOfEquations(ut) );
#endif
// determine nonzero structure of a "local (maintained)" part of matrix, and the off-diagonal part
int i, ii, j, jj, n;
Element *elem;
// allocation map
std :: vector< std :: set< int > >d_rows(leqs); // diagonal sub-matrix allocation
std :: vector< std :: set< int > >o_rows(leqs); // off-diagonal allocation
IntArray d_nnz(leqs), o_nnz(leqs), lloc, gloc;
//fprintf (stderr,"[%d] n2l map: ",rank);
//for (n=1; n<=n2l.giveN2Lmap()->giveSize(); n++) fprintf (stderr, "%d ", n2l.giveN2Lmap()->at(n));
nelem = domain->giveNumberOfElements();
for ( n = 1; n <= nelem; n++ ) {
//fprintf (stderr, "(elem %d) ", n);
elem = domain->giveElement(n);
elem->giveLocationArray(loc, ut, s);
n2l->map2New(lloc, loc, 0); // translate natural->local numbering (remark, 1-based indexing)
n2g->map2New(gloc, loc, 0); // translate natural->global numbering (remark, 0-based indexing)
// See the petsc manual for details on how this allocation is constructed.
for ( i = 1; i <= lloc.giveSize(); i++ ) {
if ( ( ii = lloc.at(i) ) ) {
for ( j = 1; j <= lloc.giveSize(); j++ ) {
if ( ( jj = gloc.at(j) ) >= 0 ) { // if negative, then it is prescribed
if ( lloc.at(j) ) { // if true, then its the local part (the diagonal block matrix)
d_rows [ ii - 1 ].insert(jj);
} else { // Otherwise it must be off-diagonal
o_rows [ ii - 1 ].insert(jj);
}
}
}
}
}
//fprintf (stderr, "\n");
}
// Diagonal must always be allocated; this code ensures that for every local line, it adds the global column number
IntArray *n2gmap = n2g->giveN2Gmap();
IntArray *n2lmap = n2l->giveN2Lmap();
for ( int n = 1; n <= n2lmap->giveSize(); ++n ) {
if ( n2lmap->at(n) ) {
d_rows [ n2lmap->at(n)-1 ].insert( n2gmap->at(n) );
}
}
for ( i = 0; i < leqs; i++ ) {
d_nnz(i) = d_rows [ i ].size();
o_nnz(i) = o_rows [ i ].size();
}
//fprintf (stderr,"\n[%d]PetscSparseMtrx: Profile ...",rank);
//for (i=0; i<leqs; i++) fprintf(stderr, "%d ", d_nnz(i));
//fprintf (stderr,"\n[%d]PetscSparseMtrx: Creating MPIAIJ Matrix ...\n",rank);
// create PETSc mat
MatCreate(PETSC_COMM_WORLD, & mtrx);
MatSetSizes(mtrx, leqs, leqs, geqs, geqs);
MatSetType(mtrx, MATMPIAIJ);
MatSetFromOptions(mtrx);
MatSetUp(mtrx);
MatMPIAIJSetPreallocation(mtrx, 0, d_nnz.givePointer(), 0, o_nnz.givePointer());
//MatMPIBAIJSetPreallocation( mtrx, PETSC_DECIDE, 0, d_nnz.givePointer(), onz, onnz );
//MatMPISBAIJSetPreallocation( mtrx, PETSC_DECIDE, 0, d_nnz_sym.givePointer(), onz, onnz );
MatSetOption(mtrx, MAT_ROW_ORIENTED, PETSC_FALSE); // To allow the insertion of values using MatSetValues in column major order
MatSetOption(mtrx, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("PetscSparseMtrx:: buildInternalStructure", "done", rank);
#endif
} else {
#endif
leqs = geqs = eModel->giveNumberOfEquations(ut);
int i, ii, j, jj, n;
Element *elem;
// allocation map
std :: vector< std :: set< int > >rows(leqs);
std :: vector< std :: set< int > >rows_sym(leqs);
nelem = domain->giveNumberOfElements();
for ( n = 1; n <= nelem; n++ ) {
elem = domain->giveElement(n);
elem->giveLocationArray(loc, ut, s);
for ( i = 1; i <= loc.giveSize(); i++ ) {
if ( ( ii = loc.at(i) ) ) {
for ( j = 1; j <= loc.giveSize(); j++ ) {
jj = loc.at(j);
if ( jj ) {
rows [ ii - 1 ].insert(jj - 1);
if ( jj >= ii ) {
rows_sym [ ii - 1 ].insert(jj - 1);
}
}
}
}
}
}
// Structure from active boundary conditions.
AList<IntArray> locs, temp;
for ( n = 1; n <= domain->giveNumberOfBoundaryConditions(); n++ ) {
ActiveBoundaryCondition *activebc = dynamic_cast<ActiveBoundaryCondition*>(domain->giveBc(n));
if (activebc) {
///@todo Deal with the CharType here.
activebc->giveLocationArrays(locs, temp, ut, TangentStiffnessMatrix, s, s, domain);
for (int k = 1; k < locs.giveSize(); k++) {
IntArray *kloc = locs.at(k);
for ( i = 1; i <= kloc->giveSize(); i++ ) {
if ( ( ii = kloc->at(i) ) ) {
for ( j = 1; j <= kloc->giveSize(); j++ ) {
jj = kloc->at(j);
if ( jj ) {
rows [ ii - 1 ].insert(jj - 1);
if ( jj >= ii ) {
rows_sym [ ii - 1 ].insert(jj - 1);
}
}
}
}
}
}
}
}
IntArray d_nnz(leqs);
IntArray d_nnz_sym(leqs);
for ( i = 0; i < leqs; i++ ) {
d_nnz(i) = rows [ i ].size();
d_nnz_sym(i) = rows_sym [ i ].size();
}
MatCreate(PETSC_COMM_SELF, & mtrx);
MatSetSizes(mtrx, leqs, leqs, geqs, geqs);
MatSetType(mtrx, MATSEQAIJ);
//MatSetType(mtrx, MATSBAIJ);
//MatSetType(mtrx, MATDENSE);
MatSetFromOptions(mtrx);
MatSetUp(mtrx);
MatSeqAIJSetPreallocation( mtrx, 0, d_nnz.givePointer() );
MatSeqBAIJSetPreallocation( mtrx, PETSC_DECIDE, 0, d_nnz.givePointer() );
MatSeqSBAIJSetPreallocation( mtrx, PETSC_DECIDE, 0, d_nnz_sym.givePointer() );
MatSetOption(mtrx, MAT_ROW_ORIENTED, PETSC_FALSE); // To allow the insertion of values using MatSetValues in column major order
MatSetOption(mtrx, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
#ifdef __PARALLEL_MODE
}
#endif
nRows = nColumns = geqs;
this->newValues = true;
return true;
}