bool
NodeErrorCheckingRule :: check(Domain *domain, TimeStep *tStep)
{
// Rule doesn't apply yet.
if ( tStep->giveNumber() != tstep ) {
return true;
}
DofManager *dman = domain->giveGlobalDofManager(number);
if ( !dman ) {
if ( domain->giveEngngModel()->isParallel() ) {
return true;
} else {
OOFEM_WARNING("Dof manager %d not found.", number);
return false;
}
}
if ( dman->giveParallelMode() == DofManager_remote || dman->giveParallelMode() == DofManager_null ) {
return true;
}
Dof *dof = dman->giveDofWithID(dofid);
double dmanValue = dof->giveUnknown(mode, tStep);
bool check = checkValue(dmanValue);
if ( !check ) {
OOFEM_WARNING("Check failed in: tstep %d, node %d, dof %d, mode %d:\n"
"value is %.8e, but should be %.8e ( error is %e but tolerance is %e )",
tstep, number, dofid, mode,
dmanValue, value, fabs(dmanValue-value), tolerance );
}
return check;
}
int
EIPrimaryUnknownMapper :: mapAndUpdate(FloatArray &answer, ValueModeType mode,
Domain *oldd, Domain *newd, TimeStep *tStep)
{
int inode, nd_nnodes = newd->giveNumberOfDofManagers();
int nsize = newd->giveEngngModel()->giveNumberOfDomainEquations( newd->giveNumber(), EModelDefaultEquationNumbering() );
FloatArray unknownValues;
IntArray dofidMask, locationArray;
IntArray reglist;
#ifdef OOFEM_MAPPING_CHECK_REGIONS
ConnectivityTable *conTable = newd->giveConnectivityTable();
const IntArray *nodeConnectivity;
#endif
answer.resize(nsize);
answer.zero();
for ( inode = 1; inode <= nd_nnodes; inode++ ) {
DofManager *node = newd->giveNode(inode);
/* HUHU CHEATING */
#ifdef __PARALLEL_MODE
if ( ( node->giveParallelMode() == DofManager_null ) ||
( node->giveParallelMode() == DofManager_remote ) ) {
continue;
}
#endif
#ifdef OOFEM_MAPPING_CHECK_REGIONS
// build up region list for node
nodeConnectivity = conTable->giveDofManConnectivityArray(inode);
reglist.resize( nodeConnectivity->giveSize() );
reglist.clear();
for ( int indx = 1; indx <= nodeConnectivity->giveSize(); indx++ ) {
reglist.insertSortedOnce( newd->giveElement( nodeConnectivity->at(indx) )->giveRegionNumber() );
}
#endif
///@todo Shouldn't we pass a primary field or something to this function?
if ( this->evaluateAt(unknownValues, dofidMask, mode, oldd, * node->giveCoordinates(), reglist, tStep) ) {
///@todo This doesn't respect local coordinate systems in nodes. Supporting that would require major reworking.
for ( int ii = 1; ii <= dofidMask.giveSize(); ii++ ) {
// exclude slaves; they are determined from masters
auto it = node->findDofWithDofId((DofIDItem)dofidMask.at(ii));
if ( it != node->end() ) {
Dof *dof = *it;
if ( dof->isPrimaryDof() ) {
int eq = dof->giveEquationNumber(EModelDefaultEquationNumbering());
answer.at( eq ) += unknownValues.at(ii);
}
}
}
} else {
OOFEM_ERROR("evaluateAt service failed for node %d", inode);
}
}
return 1;
}
int
ParmetisLoadBalancer :: packSharedDmanPartitions(ProcessCommunicator &pc)
{
int myrank = domain->giveEngngModel()->giveRank();
int iproc = pc.giveRank();
int ndofman, idofman;
DofManager *dofman;
if ( iproc == myrank ) {
return 1; // skip local partition
}
// query process communicator to use
ProcessCommunicatorBuff *pcbuff = pc.giveProcessCommunicatorBuff();
// loop over dofManagers and pack shared dofMan data
ndofman = domain->giveNumberOfDofManagers();
for ( idofman = 1; idofman <= ndofman; idofman++ ) {
dofman = domain->giveDofManager(idofman);
// test if iproc is in list of existing shared partitions
if ( ( dofman->giveParallelMode() == DofManager_shared ) &&
( dofman->givePartitionList()->findFirstIndexOf(iproc) ) ) {
// send new partitions to remote representation
// fprintf (stderr, "[%d] sending shared plist of %d to [%d]\n", myrank, dofman->giveGlobalNumber(), iproc);
pcbuff->write( dofman->giveGlobalNumber() );
this->giveDofManPartitions(idofman)->storeYourself(*pcbuff);
}
}
pcbuff->write((int)PARMETISLB_END_DATA);
return 1;
}
int
StructuralEngngModel :: unpackDofManagers(FloatArray *dest, ProcessCommunicator &processComm, bool prescribedEquations)
{
int result = 1;
int i, size;
int j, ndofs, eqNum;
Domain *domain = this->giveDomain(1);
dofManagerParallelMode dofmanmode;
IntArray const *toRecvMap = processComm.giveToRecvMap();
ProcessCommunicatorBuff *pcbuff = processComm.giveProcessCommunicatorBuff();
DofManager *dman;
Dof *jdof;
double value;
size = toRecvMap->giveSize();
for ( i = 1; i <= size; i++ ) {
dman = domain->giveDofManager( toRecvMap->at(i) );
ndofs = dman->giveNumberOfDofs();
dofmanmode = dman->giveParallelMode();
for ( j = 1; j <= ndofs; j++ ) {
jdof = dman->giveDof(j);
if ( prescribedEquations ) {
eqNum = jdof->__givePrescribedEquationNumber();
} else {
eqNum = jdof->__giveEquationNumber();
}
if ( jdof->isPrimaryDof() && eqNum ) {
result &= pcbuff->unpackDouble(value);
if ( dofmanmode == DofManager_shared ) {
dest->at(eqNum) += value;
} else if ( dofmanmode == DofManager_remote ) {
dest->at(eqNum) = value;
} else {
_error("unpackReactions: unknown dof namager parallel mode");
}
}
}
}
return result;
}
void
ParmetisLoadBalancer :: labelDofManagers()
{
int idofman, ndofman = domain->giveNumberOfDofManagers();
ConnectivityTable *ct = domain->giveConnectivityTable();
const IntArray *dofmanconntable;
DofManager *dofman;
Element *ielem;
dofManagerParallelMode dmode;
std :: set< int, std :: less< int > >__dmanpartitions;
int myrank = domain->giveEngngModel()->giveRank();
int nproc = domain->giveEngngModel()->giveNumberOfProcesses();
int ie, npart;
// resize label array
dofManState.resize(ndofman);
dofManState.zero();
// resize dof man partitions
dofManPartitions.clear();
dofManPartitions.resize(ndofman);
#ifdef ParmetisLoadBalancer_DEBUG_PRINT
int _cols = 0;
fprintf(stderr, "[%d] DofManager labels:\n", myrank);
#endif
// loop over local dof managers
for ( idofman = 1; idofman <= ndofman; idofman++ ) {
dofman = domain->giveDofManager(idofman);
dmode = dofman->giveParallelMode();
if ( ( dmode == DofManager_local ) || ( dmode == DofManager_shared ) ) {
dofmanconntable = ct->giveDofManConnectivityArray(idofman);
__dmanpartitions.clear();
for ( ie = 1; ie <= dofmanconntable->giveSize(); ie++ ) {
ielem = domain->giveElement( dofmanconntable->at(ie) );
// assemble list of partitions sharing idofman dofmanager
// set is used to include possibly repeated partition only once
if ( ielem->giveParallelMode() == Element_local ) {
__dmanpartitions.insert( giveElementPartition( dofmanconntable->at(ie) ) );
}
}
npart = __dmanpartitions.size();
dofManPartitions [ idofman - 1 ].resize( __dmanpartitions.size() );
int i = 1;
for ( auto &dm: __dmanpartitions ) {
dofManPartitions [ idofman - 1 ].at(i++) = dm;
}
}
}
// handle master slave links between dofmans (master and slave required on same partition)
this->handleMasterSlaveDofManLinks();
/* Exchange new partitions for shared nodes */
CommunicatorBuff cb(nproc, CBT_dynamic);
Communicator com(domain->giveEngngModel(), &cb, myrank, nproc, CommMode_Dynamic);
com.packAllData(this, & ParmetisLoadBalancer :: packSharedDmanPartitions);
com.initExchange(SHARED_DOFMAN_PARTITIONS_TAG);
com.unpackAllData(this, & ParmetisLoadBalancer :: unpackSharedDmanPartitions);
com.finishExchange();
/* label dof managers */
for ( idofman = 1; idofman <= ndofman; idofman++ ) {
dofman = domain->giveDofManager(idofman);
dmode = dofman->giveParallelMode();
npart = dofManPartitions [ idofman - 1 ].giveSize();
if ( ( dmode == DofManager_local ) || ( dmode == DofManager_shared ) ) {
// determine its state after balancing -> label
dofManState.at(idofman) = this->determineDofManState(idofman, myrank, npart, & dofManPartitions [ idofman - 1 ]);
} else {
dofManState.at(idofman) = DM_NULL;
}
}
#ifdef ParmetisLoadBalancer_DEBUG_PRINT
for ( idofman = 1; idofman <= ndofman; idofman++ ) {
fprintf(stderr, " | %d: ", idofman);
if ( dofManState.at(idofman) == DM_NULL ) {
fprintf(stderr, "NULL ");
} else if ( dofManState.at(idofman) == DM_Local ) {
fprintf(stderr, "Local ");
} else if ( dofManState.at(idofman) == DM_Shared ) {
fprintf(stderr, "Shared");
} else if ( dofManState.at(idofman) == DM_Remote ) {
fprintf(stderr, "Remote");
} else {
fprintf(stderr, "Unknown");
}
//else if (dofManState.at(idofman) == DM_SharedExclude)fprintf (stderr, "ShdExc");
//else if (dofManState.at(idofman) == DM_SharedNew) fprintf (stderr, "ShdNew");
//else if (dofManState.at(idofman) == DM_SharedUpdate) fprintf (stderr, "ShdUpd");
if ( ( ( ++_cols % 4 ) == 0 ) || ( idofman == ndofman ) ) {
fprintf(stderr, "\n");
}
}
#endif
}
void NlDEIDynamic :: solveYourselfAt(TimeStep *tStep)
{
//
// Creates system of governing eq's and solves them at given time step.
//
Domain *domain = this->giveDomain(1);
int neq = this->giveNumberOfDomainEquations( 1, EModelDefaultEquationNumbering() );
int nman = domain->giveNumberOfDofManagers();
DofManager *node;
int i, k, j, jj;
double coeff, maxDt, maxOm = 0.;
double prevIncrOfDisplacement, incrOfDisplacement;
if ( initFlag ) {
#ifdef VERBOSE
OOFEM_LOG_DEBUG("Assembling mass matrix\n");
#endif
//
// Assemble mass matrix.
//
this->computeMassMtrx(massMatrix, maxOm, tStep);
if ( drFlag ) {
// If dynamic relaxation: Assemble amplitude load vector.
loadRefVector.resize(neq);
loadRefVector.zero();
this->computeLoadVector(loadRefVector, VM_Total, tStep);
#ifdef __PARALLEL_MODE
// Compute the processor part of load vector norm pMp
this->pMp = 0.0;
double my_pMp = 0.0, coeff = 1.0;
int eqNum, ndofman = domain->giveNumberOfDofManagers();
dofManagerParallelMode dofmanmode;
DofManager *dman;
for ( int dm = 1; dm <= ndofman; dm++ ) {
dman = domain->giveDofManager(dm);
dofmanmode = dman->giveParallelMode();
// Skip all remote and null dofmanagers
coeff = 1.0;
if ( ( dofmanmode == DofManager_remote ) || ( ( dofmanmode == DofManager_null ) ) ) {
continue;
} else if ( dofmanmode == DofManager_shared ) {
coeff = 1. / dman->givePartitionsConnectivitySize();
}
// For shared nodes we add locally an average = 1/givePartitionsConnectivitySize()*contribution,
for ( Dof *dof: *dman ) {
if ( dof->isPrimaryDof() && ( eqNum = dof->__giveEquationNumber() ) ) {
my_pMp += coeff * loadRefVector.at(eqNum) * loadRefVector.at(eqNum) / massMatrix.at(eqNum);
}
}
}
// Sum up the contributions from processors.
MPI_Allreduce(& my_pMp, & pMp, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
#else
this->pMp = 0.0;
for ( i = 1; i <= neq; i++ ) {
pMp += loadRefVector.at(i) * loadRefVector.at(i) / massMatrix.at(i);
}
#endif
// Solve for rate of loading process (parameter "c") (undamped system assumed),
if ( dumpingCoef < 1.e-3 ) {
c = 3.0 * this->pyEstimate / pMp / Tau / Tau;
} else {
c = this->pyEstimate * Tau * dumpingCoef * dumpingCoef * dumpingCoef / pMp /
( -3.0 / 2.0 + dumpingCoef * Tau + 2.0 * exp(-dumpingCoef * Tau) - 0.5 * exp(-2.0 * dumpingCoef * Tau) );
}
}
initFlag = 0;
}
if ( tStep->isTheFirstStep() ) {
//
// Special init step - Compute displacements at tstep 0.
//
displacementVector.resize(neq);
displacementVector.zero();
previousIncrementOfDisplacementVector.resize(neq);
previousIncrementOfDisplacementVector.zero();
velocityVector.resize(neq);
velocityVector.zero();
accelerationVector.resize(neq);
accelerationVector.zero();
for ( j = 1; j <= nman; j++ ) {
node = domain->giveDofManager(j);
for ( Dof *dof: *node ) {
// Ask for initial values obtained from
// bc (boundary conditions) and ic (initial conditions)
// all dofs are expected to be DisplacementVector type.
if ( !dof->isPrimaryDof() ) {
continue;
}
jj = dof->__giveEquationNumber();
if ( jj ) {
displacementVector.at(jj) = dof->giveUnknown(VM_Total, tStep);
velocityVector.at(jj) = dof->giveUnknown(VM_Velocity, tStep);
accelerationVector.at(jj) = dof->giveUnknown(VM_Acceleration, tStep);
}
}
}
//
// Set-up numerical model.
//
// Try to determine the best deltaT,
maxDt = 2.0 / sqrt(maxOm);
if ( deltaT > maxDt ) {
// Print reduced time step increment and minimum period Tmin
OOFEM_LOG_RELEVANT("deltaT reduced to %e, Tmin is %e\n", maxDt, maxDt * M_PI);
deltaT = maxDt;
tStep->setTimeIncrement(deltaT);
}
for ( j = 1; j <= neq; j++ ) {
previousIncrementOfDisplacementVector.at(j) = velocityVector.at(j) * ( deltaT );
displacementVector.at(j) -= previousIncrementOfDisplacementVector.at(j);
}
#ifdef VERBOSE
OOFEM_LOG_RELEVANT( "\n\nSolving [Step number %8d, Time %15e]\n", tStep->giveNumber(), tStep->giveTargetTime() );
#endif
return;
} // end of init step
#ifdef VERBOSE
OOFEM_LOG_DEBUG("Assembling right hand side\n");
#endif
displacementVector.add(previousIncrementOfDisplacementVector);
// Update solution state counter
tStep->incrementStateCounter();
// Compute internal forces.
this->giveInternalForces(internalForces, false, 1, tStep);
if ( !drFlag ) {
//
// Assembling the element part of load vector.
//
this->computeLoadVector(loadVector, VM_Total, tStep);
//
// Assembling additional parts of right hand side.
//
loadVector.subtract(internalForces);
} else {
// Dynamic relaxation
// compute load factor
pt = 0.0;
#ifdef __PARALLEL_MODE
double my_pt = 0.0, coeff = 1.0;
int eqNum, ndofman = domain->giveNumberOfDofManagers();
dofManagerParallelMode dofmanmode;
DofManager *dman;
for ( int dm = 1; dm <= ndofman; dm++ ) {
dman = domain->giveDofManager(dm);
dofmanmode = dman->giveParallelMode();
// skip all remote and null dofmanagers
coeff = 1.0;
if ( ( dofmanmode == DofManager_remote ) || ( dofmanmode == DofManager_null ) ) {
continue;
} else if ( dofmanmode == DofManager_shared ) {
coeff = 1. / dman->givePartitionsConnectivitySize();
}
// For shared nodes we add locally an average= 1/givePartitionsConnectivitySize()*contribution.
for ( Dof *dof: *dman ) {
if ( dof->isPrimaryDof() && ( eqNum = dof->__giveEquationNumber() ) ) {
my_pt += coeff * internalForces.at(eqNum) * loadRefVector.at(eqNum) / massMatrix.at(eqNum);
}
}
}
// Sum up the contributions from processors.
MPI_Allreduce(& my_pt, & pt, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
#else
for ( k = 1; k <= neq; k++ ) {
pt += internalForces.at(k) * loadRefVector.at(k) / massMatrix.at(k);
}
#endif
pt = pt / pMp;
if ( dumpingCoef < 1.e-3 ) {
pt += c * ( Tau - tStep->giveTargetTime() ) / Tau;
} else {
pt += c * ( 1.0 - exp( dumpingCoef * ( tStep->giveTargetTime() - Tau ) ) ) / dumpingCoef / Tau;
}
loadVector.resize( this->giveNumberOfDomainEquations( 1, EModelDefaultEquationNumbering() ) );
for ( k = 1; k <= neq; k++ ) {
loadVector.at(k) = pt * loadRefVector.at(k) - internalForces.at(k);
}
// Compute relative error.
double err = 0.0;
#ifdef __PARALLEL_MODE
double my_err = 0.0;
for ( int dm = 1; dm <= ndofman; dm++ ) {
dman = domain->giveDofManager(dm);
dofmanmode = dman->giveParallelMode();
// Skip all remote and null dofmanagers.
coeff = 1.0;
if ( ( dofmanmode == DofManager_remote ) || ( dofmanmode == DofManager_null ) ) {
continue;
} else if ( dofmanmode == DofManager_shared ) {
coeff = 1. / dman->givePartitionsConnectivitySize();
}
// For shared nodes we add locally an average= 1/givePartitionsConnectivitySize()*contribution.
for ( Dof *dof: *dman ) {
if ( dof->isPrimaryDof() && ( eqNum = dof->__giveEquationNumber() ) ) {
my_err += coeff * loadVector.at(eqNum) * loadVector.at(eqNum) / massMatrix.at(eqNum);
}
}
}
// Sum up the contributions from processors.
MPI_Allreduce(& my_err, & err, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
#else
for ( k = 1; k <= neq; k++ ) {
err = loadVector.at(k) * loadVector.at(k) / massMatrix.at(k);
}
#endif
err = err / ( pMp * pt * pt );
OOFEM_LOG_RELEVANT("Relative error is %e, loadlevel is %e\n", err, pt);
}
for ( j = 1; j <= neq; j++ ) {
coeff = massMatrix.at(j);
loadVector.at(j) +=
coeff * ( ( 1. / ( deltaT * deltaT ) ) - dumpingCoef * 1. / ( 2. * deltaT ) ) *
previousIncrementOfDisplacementVector.at(j);
}
//
// Set-up numerical model
//
/* it is not necesary to call numerical method
* approach used here is not good, but effective enough
* inverse of diagonal mass matrix is done here
*/
//
// call numerical model to solve arised problem - done localy here
//
#ifdef VERBOSE
OOFEM_LOG_RELEVANT( "\n\nSolving [Step number %8d, Time %15e]\n", tStep->giveNumber(), tStep->giveTargetTime() );
#endif
// NM_Status s = nMethod->solve(*massMatrix, loadVector, displacementVector);
// if ( !(s & NM_Success) ) {
// OOFEM_ERROR("No success in solving system. Ma=f");
// }
for ( i = 1; i <= neq; i++ ) {
prevIncrOfDisplacement = previousIncrementOfDisplacementVector.at(i);
incrOfDisplacement = loadVector.at(i) /
( massMatrix.at(i) * ( 1. / ( deltaT * deltaT ) + dumpingCoef / ( 2. * deltaT ) ) );
accelerationVector.at(i) = ( incrOfDisplacement - prevIncrOfDisplacement ) / ( deltaT * deltaT );
velocityVector.at(i) = ( incrOfDisplacement + prevIncrOfDisplacement ) / ( 2. * deltaT );
previousIncrementOfDisplacementVector.at(i) = incrOfDisplacement;
}
}
void
PetscNatural2GlobalOrdering :: init(EngngModel *emodel, EquationID ut, int di, EquationType et)
{
Domain *d = emodel->giveDomain(di);
int i, j, k, p, ndofs, ndofman = d->giveNumberOfDofManagers();
int myrank = emodel->giveRank();
DofManager *dman;
// determine number of local eqs + number of those shared DOFs which are numbered by receiver
// shared dofman is numbered on partition with lovest rank number
EModelDefaultEquationNumbering dn;
EModelDefaultPrescribedEquationNumbering dpn;
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("PetscNatural2GlobalOrdering :: init", "initializing N2G ordering", myrank);
#endif
l_neqs = 0;
for ( i = 1; i <= ndofman; i++ ) {
dman = d->giveDofManager(i);
/*
* if (dman->giveParallelMode() == DofManager_local) { // count all dofman eqs
* ndofs = dman->giveNumberOfDofs ();
* for (j=1; j<=ndofs; j++) {
* if (dman->giveDof(j)->isPrimaryDof()) {
* if (dman->giveDof(j)->giveEquationNumber()) l_neqs++;
* }
* }
* } else if (dman->giveParallelMode() == DofManager_shared) {
* // determine if problem is the lowest one sharing the dofman; if yes the receiver is responsible to
* // deliver number
* IntArray *plist = dman->givePartitionList();
* int n = plist->giveSize();
* int minrank = myrank;
* for (j=1; j<=n; j++) minrank = min (minrank, plist->at(j));
* if (minrank == myrank) { // count eqs
* ndofs = dman->giveNumberOfDofs ();
* for (j=1; j<=ndofs; j++) {
* if (dman->giveDof(j)->isPrimaryDof()) {
* if (dman->giveDof(j)->giveEquationNumber()) l_neqs++;
* }
* }
* }
* } // end shared dman
*/
if ( isLocal(dman) ) {
ndofs = dman->giveNumberOfDofs();
for ( j = 1; j <= ndofs; j++ ) {
if ( dman->giveDof(j)->isPrimaryDof() ) {
if ( et == et_standard ) {
if ( dman->giveDof(j)->giveEquationNumber(dn) ) {
l_neqs++;
}
} else {
if ( dman->giveDof(j)->giveEquationNumber(dpn) ) {
l_neqs++;
}
}
}
}
}
}
// exchange with other procs the number of eqs numbered on particular procs
int *leqs = new int [ emodel->giveNumberOfProcesses() ];
MPI_Allgather(& l_neqs, 1, MPI_INT, leqs, 1, MPI_INT, MPI_COMM_WORLD);
// compute local offset
int offset = 0;
for ( j = 0; j < myrank; j++ ) {
offset += leqs [ j ];
}
// count global number of eqs
for ( g_neqs = 0, j = 0; j < emodel->giveNumberOfProcesses(); j++ ) {
g_neqs += leqs [ j ];
}
// send numbered shared ones
if ( et == et_standard ) {
locGlobMap.resize( emodel->giveNumberOfEquations(ut) );
} else {
locGlobMap.resize( emodel->giveNumberOfPrescribedEquations(ut) );
}
// determine shared dofs
int psize, nproc = emodel->giveNumberOfProcesses();
IntArray sizeToSend(nproc), sizeToRecv(nproc), nrecToReceive(nproc);
#ifdef __VERBOSE_PARALLEL
IntArray nrecToSend(nproc);
#endif
const IntArray *plist;
for ( i = 1; i <= ndofman; i++ ) {
// if (domain->giveDofManager(i)->giveParallelMode() == DofManager_shared) {
if ( isShared( d->giveDofManager(i) ) ) {
int n = d->giveDofManager(i)->giveNumberOfDofs();
plist = d->giveDofManager(i)->givePartitionList();
psize = plist->giveSize();
int minrank = myrank;
for ( j = 1; j <= psize; j++ ) {
minrank = min( minrank, plist->at(j) );
}
if ( minrank == myrank ) { // count to send
for ( j = 1; j <= psize; j++ ) {
#ifdef __VERBOSE_PARALLEL
nrecToSend( plist->at(j) )++;
#endif
sizeToSend( plist->at(j) ) += ( 1 + n ); // ndofs+dofman number
}
} else {
nrecToReceive(minrank)++;
sizeToRecv(minrank) += ( 1 + n ); // ndofs+dofman number
}
}
}
#ifdef __VERBOSE_PARALLEL
for ( i = 0; i < nproc; i++ ) {
OOFEM_LOG_INFO("[%d] Record Statistics: Sending %d Receiving %d to %d\n",
myrank, nrecToSend(i), nrecToReceive(i), i);
}
#endif
std :: map< int, int >globloc; // global->local mapping for shared
// number local guys
int globeq = offset;
for ( i = 1; i <= ndofman; i++ ) {
dman = d->giveDofManager(i);
//if (dman->giveParallelMode() == DofManager_shared) {
if ( isShared(dman) ) {
globloc [ dman->giveGlobalNumber() ] = i; // build global->local mapping for shared
plist = dman->givePartitionList();
psize = plist->giveSize();
int minrank = myrank;
for ( j = 1; j <= psize; j++ ) {
minrank = min( minrank, plist->at(j) );
}
if ( minrank == myrank ) { // local
ndofs = dman->giveNumberOfDofs();
for ( j = 1; j <= ndofs; j++ ) {
if ( dman->giveDof(j)->isPrimaryDof() ) {
int eq;
if ( et == et_standard ) {
eq = dman->giveDof(j)->giveEquationNumber(dn);
} else {
eq = dman->giveDof(j)->giveEquationNumber(dpn);
}
if ( eq ) {
locGlobMap.at(eq) = globeq++;
}
}
}
}
//} else if (dman->giveParallelMode() == DofManager_local) {
} else {
ndofs = dman->giveNumberOfDofs();
for ( j = 1; j <= ndofs; j++ ) {
if ( dman->giveDof(j)->isPrimaryDof() ) {
int eq;
if ( et == et_standard ) {
eq = dman->giveDof(j)->giveEquationNumber(dn);
} else {
eq = dman->giveDof(j)->giveEquationNumber(dpn);
}
if ( eq ) {
locGlobMap.at(eq) = globeq++;
}
}
}
}
}
/*
* fprintf (stderr, "[%d] locGlobMap: ", myrank);
* for (i=1; i<=locGlobMap.giveSize(); i++)
* fprintf (stderr, "%d ",locGlobMap.at(i));
*/
// pack data for remote procs
CommunicationBuffer **buffs = new CommunicationBuffer * [ nproc ];
for ( p = 0; p < nproc; p++ ) {
buffs [ p ] = new StaticCommunicationBuffer(MPI_COMM_WORLD, 0);
buffs [ p ]->resize( buffs [ p ]->givePackSize(MPI_INT, 1) * sizeToSend(p) );
#if 0
OOFEM_LOG_INFO( "[%d]PetscN2G:: init: Send buffer[%d] size %d\n",
myrank, p, sizeToSend(p) );
#endif
}
for ( i = 1; i <= ndofman; i++ ) {
if ( isShared( d->giveDofManager(i) ) ) {
dman = d->giveDofManager(i);
plist = dman->givePartitionList();
psize = plist->giveSize();
int minrank = myrank;
for ( j = 1; j <= psize; j++ ) {
minrank = min( minrank, plist->at(j) );
}
if ( minrank == myrank ) { // do send
for ( j = 1; j <= psize; j++ ) {
p = plist->at(j);
if ( p == myrank ) {
continue;
}
#if 0
OOFEM_LOG_INFO("[%d]PetscN2G:: init: Sending localShared node %d[%d] to proc %d\n",
myrank, i, dman->giveGlobalNumber(), p);
#endif
buffs [ p ]->packInt( dman->giveGlobalNumber() );
ndofs = dman->giveNumberOfDofs();
for ( k = 1; k <= ndofs; k++ ) {
if ( dman->giveDof(k)->isPrimaryDof() ) {
int eq;
if ( et == et_standard ) {
eq = dman->giveDof(k)->giveEquationNumber(dn);
} else {
eq = dman->giveDof(k)->giveEquationNumber(dpn);
}
if ( eq ) {
buffs [ p ]->packInt( locGlobMap.at(eq) );
}
}
}
}
}
}
}
//fprintf (stderr, "[%d] Sending glob nums ...", myrank);
// send buffers
for ( p = 0; p < nproc; p++ ) {
if ( p != myrank ) {
buffs [ p ]->iSend(p, 999);
}
}
/****
*
* for (p=0; p<nproc; p++) {
* if (p == myrank) continue;
* for (i=1; i<= ndofman; i++) {
* //if (domain->giveDofManager(i)->giveParallelMode() == DofManager_shared) {
* if (isShared(d->giveDofManager(i))) {
* dman = d->giveDofManager(i);
* plist = dman->givePartitionList();
* psize = plist->giveSize();
* int minrank = myrank;
* for (j=1; j<=psize; j++) minrank = min (minrank, plist->at(j));
* if (minrank == myrank) { // do send
* buffs[p]->packInt(dman->giveGlobalNumber());
* ndofs = dman->giveNumberOfDofs ();
* for (j=1; j<=ndofs; j++) {
* if (dman->giveDof(j)->isPrimaryDof()) {
* buffs[p]->packInt(locGlobMap.at(dman->giveDof(j)->giveEquationNumber()));
* }
* }
* }
* }
* }
* // send buffer
* buffs[p]->iSend(p, 999);
* }
****/
// receive remote eqs and complete global numbering
CommunicationBuffer **rbuffs = new CommunicationBuffer * [ nproc ];
for ( p = 0; p < nproc; p++ ) {
rbuffs [ p ] = new StaticCommunicationBuffer(MPI_COMM_WORLD, 0);
rbuffs [ p ]->resize( rbuffs [ p ]->givePackSize(MPI_INT, 1) * sizeToRecv(p) );
#if 0
OOFEM_LOG_INFO( "[%d]PetscN2G:: init: Receive buffer[%d] size %d\n",
myrank, p, sizeToRecv(p) );
#endif
}
//fprintf (stderr, "[%d] Receiving glob nums ...", myrank);
for ( p = 0; p < nproc; p++ ) {
if ( p != myrank ) {
rbuffs [ p ]->iRecv(p, 999);
}
}
IntArray finished(nproc);
finished.zero();
int fin = 1;
finished.at(emodel->giveRank() + 1) = 1;
do {
for ( p = 0; p < nproc; p++ ) {
if ( finished.at(p + 1) == 0 ) {
if ( rbuffs [ p ]->testCompletion() ) {
// data are here
// unpack them
int nite = nrecToReceive(p);
int shdm, ldm;
for ( i = 1; i <= nite; i++ ) {
rbuffs [ p ]->unpackInt(shdm);
#if 0
OOFEM_LOG_INFO("[%d]PetscN2G:: init: Received shared node [%d] from proc %d\n",
myrank, shdm, p);
#endif
//
// find local guy coorecponding to shdm
if ( globloc.find(shdm) != globloc.end() ) {
ldm = globloc [ shdm ];
} else {
OOFEM_ERROR3("[%d] PetscNatural2GlobalOrdering :: init: invalid shared dofman received, globnum %d\n", myrank, shdm);
}
dman = d->giveDofManager(ldm);
ndofs = dman->giveNumberOfDofs();
for ( j = 1; j <= ndofs; j++ ) {
if ( dman->giveDof(j)->isPrimaryDof() ) {
int eq;
if ( et == et_standard ) {
eq = dman->giveDof(j)->giveEquationNumber(dn);
} else {
eq = dman->giveDof(j)->giveEquationNumber(dpn);
}
if ( eq ) {
int val;
rbuffs [ p ]->unpackInt(val);
locGlobMap.at(eq) = val;
}
}
}
}
finished.at(p + 1) = 1;
fin++;
}
}
}
} while ( fin < nproc );
/*
* fprintf (stderr, "[%d] Finished receiving glob nums ...", myrank);
*
* fprintf (stderr, "[%d] locGlobMap:", myrank);
* for (i=1; i<=locGlobMap.giveSize(); i++)
* fprintf (stderr, "%d ",locGlobMap.at(i));
*/
#ifdef __VERBOSE_PARALLEL
if ( et == et_standard ) {
int _eq;
char *ptr;
char *locname = "local", *shname = "shared", *unkname = "unknown";
for ( i = 1; i <= ndofman; i++ ) {
dman = d->giveDofManager(i);
if ( dman->giveParallelMode() == DofManager_local ) {
ptr = locname;
} else if ( dman->giveParallelMode() == DofManager_shared ) {
ptr = shname;
} else {
ptr = unkname;
}
ndofs = dman->giveNumberOfDofs();
for ( j = 1; j <= ndofs; j++ ) {
if ( ( _eq = dman->giveDof(j)->giveEquationNumber(dn) ) ) {
fprintf( stderr, "[%d] n:%6s %d[%d] (%d), leq = %d, geq = %d\n", emodel->giveRank(), ptr, i, dman->giveGlobalNumber(), j, _eq, locGlobMap.at(_eq) );
} else {
fprintf(stderr, "[%d] n:%6s %d[%d] (%d), leq = %d, geq = %d\n", emodel->giveRank(), ptr, i, dman->giveGlobalNumber(), j, _eq, 0);
}
}
}
}
#endif
// build reverse map
int lneq;
if ( et == et_standard ) {
lneq = emodel->giveNumberOfEquations(ut);
} else {
lneq = emodel->giveNumberOfPrescribedEquations(ut);
}
globLocMap.clear();
for ( i = 1; i <= lneq; i++ ) {
globLocMap [ locGlobMap.at(i) ] = i;
}
for ( p = 0; p < nproc; p++ ) {
delete rbuffs [ p ];
delete buffs [ p ];
}
delete[] rbuffs;
delete[] buffs;
delete[] leqs;
MPI_Barrier(MPI_COMM_WORLD);
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("PetscNatural2GlobalOrdering :: init", "done", myrank);
#endif
}
void
ProblemCommunicator :: setUpCommunicationMapsForElementCut(EngngModel *pm,
bool excludeSelfCommFlag)
{
Domain *domain = pm->giveDomain(1);
int nnodes = domain->giveNumberOfDofManagers();
int i, j, partition;
if ( this->mode == ProblemCommMode__ELEMENT_CUT ) {
/*
* Initially, each partition knows for which nodes a receive
* is needed (and can therefore compute easily the recv map),
* but does not know for which nodes it should send data to which
* partition. Hence, the communication setup is performed by
* broadcasting "send request" lists of nodes for which
* a partition expects to receive data (ie. of those nodes
* which the partition uses, but does not own) to all
* collaborating processes. The "send request" list are
* converted into send maps.
*/
// receive maps can be build locally,
// but send maps should be assembled from broadcasted lists (containing
// expected receive nodes) of remote partitions.
// first build local receive map
IntArray domainNodeRecvCount(size);
const IntArray *partitionList;
DofManager *dofMan;
//Element *element;
int domainRecvListSize = 0, domainRecvListPos = 0;
//int nelems;
int result = 1;
for ( i = 1; i <= nnodes; i++ ) {
partitionList = domain->giveDofManager(i)->givePartitionList();
if ( domain->giveDofManager(i)->giveParallelMode() == DofManager_remote ) {
// size of partitionList should be 1 <== only ine master
for ( j = 1; j <= partitionList->giveSize(); j++ ) {
if ( !( excludeSelfCommFlag && ( this->rank == partitionList->at(j) ) ) ) {
domainRecvListSize++;
domainNodeRecvCount.at(partitionList->at(j) + 1)++;
}
}
}
}
// build maps simultaneously
IntArray pos(size);
IntArray **maps = new IntArray * [ size ];
for ( i = 0; i < size; i++ ) {
maps [ i ] = new IntArray( domainNodeRecvCount.at(i + 1) );
}
// allocate also domain receive list to be broadcasted
IntArray domainRecvList(domainRecvListSize);
if ( domainRecvListSize ) {
for ( i = 1; i <= nnodes; i++ ) {
// test if node is remote DofMan
dofMan = domain->giveDofManager(i);
if ( dofMan->giveParallelMode() == DofManager_remote ) {
domainRecvList.at(++domainRecvListPos) = dofMan->giveGlobalNumber();
partitionList = domain->giveDofManager(i)->givePartitionList();
// size of partitionList should be 1 <== only ine master
for ( j = 1; j <= partitionList->giveSize(); j++ ) {
if ( !( excludeSelfCommFlag && ( this->rank == partitionList->at(j) ) ) ) {
partition = partitionList->at(j);
maps [ partition ]->at( ++pos.at(partition + 1) ) = i;
}
}
}
}
}
// set up process recv communicator maps
for ( i = 0; i < size; i++ ) {
this->setProcessCommunicatorToRecvArry(this->giveProcessCommunicator(i), * maps [ i ]);
//this->giveDomainCommunicator(i)->setToRecvArry (this->engngModel, *maps[i]);
}
// delete local maps
for ( i = 0; i < size; i++ ) {
delete maps [ i ];
}
delete maps;
// to assemble send maps, we must analyze broadcasted remote domain send lists
// and we must also broadcast our send list.
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("ProblemCommunicator::setUpCommunicationMaps", "Element-cut broadcasting started", rank);
#endif
StaticCommunicationBuffer commBuff(MPI_COMM_WORLD);
IntArray remoteDomainRecvList;
IntArray toSendMap;
int localExpectedSize, globalRecvSize;
int sendMapPos, sendMapSize, globalDofManNum;
// determine the size of receive buffer using AllReduce operation
#ifndef IBM_MPI_IMPLEMENTATION
localExpectedSize = domainRecvList.givePackSize(commBuff);
#else
localExpectedSize = domainRecvList.givePackSize(commBuff) + 1;
#endif
#ifdef __USE_MPI
result = MPI_Allreduce(& localExpectedSize, & globalRecvSize, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
if ( result != MPI_SUCCESS ) {
_error("setUpCommunicationMaps: MPI_Allreduce failed");
}
#else
WARNING: NOT SUPPORTED MESSAGE PARSING LIBRARY
#endif
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("ProblemCommunicator::setUpCommunicationMaps", "Finished reducing receiveBufferSize", rank);
#endif
// resize to fit largest received message
commBuff.resize(globalRecvSize);
// resize toSend map to max possible size
toSendMap.resize(globalRecvSize);
for ( i = 0; i < size; i++ ) { // loop over domains
commBuff.init();
if ( i == rank ) {
//current domain has to send its receive list to all domains
// broadcast domainRecvList
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("ProblemCommunicator::setUpCommunicationMaps", "Broadcasting own send list", rank);
#endif
commBuff.packIntArray(domainRecvList);
result = commBuff.bcast(i);
if ( result != MPI_SUCCESS ) {
_error("setUpCommunicationMaps: commBuff broadcast failed");
}
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("ProblemCommunicator::setUpCommunicationMaps", "Broadcasting own send list finished", rank);
#endif
} else {
#ifdef __VERBOSE_PARALLEL
OOFEM_LOG_DEBUG("[process rank %3d]: %-30s: Receiving broadcasted send map from partition %3d\n",
rank, "ProblemCommunicator :: unpackAllData", i);
#endif
// receive broadcasted lists
result = commBuff.bcast(i);
if ( result != MPI_SUCCESS ) {
_error("setUpCommunicationMaps: commBuff broadcast failed");
}
#ifdef __VERBOSE_PARALLEL
OOFEM_LOG_DEBUG("[process rank %3d]: %-30s: Receiving broadcasted send map from partition %3d finished\n",
rank, "ProblemCommunicator :: unpackAllData", i);
#endif
// unpack remote receive list
if ( !commBuff.unpackIntArray(remoteDomainRecvList) ) {
_error("ProblemCommunicator::setUpCommunicationMaps: unpack remote receive list failed");
}
// find if remote nodes are in local partition
// if yes add them into send map for correcponding i-th partition
sendMapPos = 0;
sendMapSize = 0;
// determine sendMap size
for ( j = 1; j <= nnodes; j++ ) { // loop over local DofManagers
dofMan = domain->giveDofManager(j);
globalDofManNum = dofMan->giveGlobalNumber();
// test id globalDofManNum is in remoteDomainRecvList
if ( remoteDomainRecvList.findFirstIndexOf(globalDofManNum) ) {
sendMapSize++;
}
}
toSendMap.resize(sendMapSize);
for ( j = 1; j <= nnodes; j++ ) { // loop over local DofManagers
dofMan = domain->giveDofManager(j);
globalDofManNum = dofMan->giveGlobalNumber();
// test id globalDofManNum is in remoteDomainRecvList
if ( remoteDomainRecvList.findFirstIndexOf(globalDofManNum) ) {
// add this local DofManager number to sed map for active partition
toSendMap.at(++sendMapPos) = j;
}
} // end loop over local DofManagers
// set send map to i-th process communicator
this->setProcessCommunicatorToSendArry(this->giveProcessCommunicator(i), toSendMap);
//this->giveDomainCommunicator(i)->setToSendArry (this->engngModel, toSendMap);
} // end receiving broadcasted lists
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("ProblemCommunicator::setUpCommunicationMaps", "Receiving broadcasted send maps finished", rank);
#endif
} // end loop over domains
} else {
_error("setUpCommunicationMapsForElementCut: unknown mode");
}
}