void Cartographer::fixupResourceMaps(const ResourceType *rt, const Vec2i &pos) {
PF_TRACE();
const Map &map = *world->getMap();
Vec2i junk;
for (set<ResourceMapKey>::iterator it = resourceMapKeys.begin(); it != resourceMapKeys.end(); ++it) {
if (it->resourceType == rt) {
PatchMap<1> *pMap = resourceMaps[*it];
const int &size = it->workerSize;
const Field &field = it->workerField;
Vec2i tl = pos + OrdinalOffsets[odNorthWest] * size;
Vec2i br(tl.x + size + 2, tl.y + size + 2);
Util::RectIterator iter(tl, br);
while (iter.more()) {
Vec2i cur = iter.next();
if (map.isInside(cur) && masterMap->canOccupy(cur, size, field)
&& map.isResourceNear(cur, size, rt, junk)) {
pMap->setInfluence(cur, 1);
} else {
pMap->setInfluence(cur, 0);
}
}
}
}
}
foreach (KeyList, it, storeMaps) {
PatchMap<1> *pMap = g_world.getCartographer()->getStoreMap(*it, false);
if (pMap && pMap->getInfluence(cell)) {
colour = Vec4f(0.f, 1.f, 0.3f, 0.7f);
return true;
}
}
void Sync::releaseComputes()
{
PatchMap *patchMap = PatchMap::Object();
traceUserEvent(eventReleaseComputes);
for (int i= 0; i<cnum; i++) {
int &pid = clist[i].pid;
if (pid == -1) continue;
if (clist[i].step != step) {
continue;
}
// CkPrintf(" %d-%d-%d ",
// clist[i].pid, clist[i].step,
// patchMap->patch(pid)->flags.sequence);
ComputePtrListIter cid = clist[i].cid;
int compute_count = 0;
for(cid = cid.begin(); cid != cid.end(); cid++) {
compute_count++;
(*cid)->patchReady(pid,clist[i].doneMigration,step);
}
if (compute_count == 0 && patchMap->node(pid) != CkMyPe()) {
iout << iINFO << "PATCH_COUNT-Sync step " << step
<< "]: Patch " << pid << " on PE "
<< CkMyPe() <<" home patch "
<< patchMap->node(pid) << " does not have any computes\n"
<< endi;
}
pid = -1;
}
// CkPrintf("\n");
}
bool ResourceMapOverlay::operator()(const Vec2i &cell, Vec4f &colour) {
ResourceMapKey mapKey(rt, Field::LAND, 1);
PatchMap<1> *pMap = g_world.getCartographer()->getResourceMap(mapKey);
if (pMap && pMap->getInfluence(cell)) {
colour = Vec4f(1.f, 1.f, 0.f, 0.7f);
return true;
}
return false;
}
//! \brief Free up the memory allocated in the PatchMap, making sure to avoid double free's
//! \param map The map to free up
void freePatchMap(PatchMap& map)
{
std::vector<RealArray*> free;
for (PatchMap::iterator it = map.begin(); it != map.end(); ++it) {
for (size_t i = 0; i < it->second.Patch.size(); ++i)
delete it->second.Patch[i];
free.insert(free.end(), it->second.FakeModel.begin(), it->second.FakeModel.end());
}
std::vector<RealArray*>::iterator it = std::unique(free.begin(), free.end());
for (std::vector<RealArray*>::iterator it2 = free.begin(); it2 != it; ++it2)
delete *it2;
map.clear();
}
PatchMapIt _add(const int eTag)
{
PatchMapIns insPatch = patch.insert(PatchMapVal(eTag, PatchDataListIt()));
if(insPatch.second) {
insPatch.first->second =
patchData.insert(patchData.end(), PatchData(eTag));
}
return insPatch.first;
}
PatchMap<1>* Cartographer::buildAdjacencyMap(const UnitType *uType, const Vec2i &pos,
CardinalDir facing, Field f, int size) {
PF_TRACE();
const Vec2i mapPos = pos + (OrdinalOffsets[odNorthWest] * size);
const int sx = pos.x;
const int sy = pos.y;
Rectangle rect(mapPos.x, mapPos.y, uType->getSize() + 2 + size, uType->getSize() + 2 + size);
PatchMap<1> *pMap = new PatchMap<1>(rect, 0);
pMap->zeroMap();
PatchMap<1> tmpMap(rect, 0);
tmpMap.zeroMap();
// mark cells occupied by unitType at pos (on tmpMap)
Util::RectIterator rIter(pos, pos + Vec2i(uType->getSize() - 1));
while (rIter.more()) {
Vec2i gpos = rIter.next();
if (!uType->hasCellMap() || uType->getCellMapCell(gpos.x - sx, gpos.y - sy, facing)) {
tmpMap.setInfluence(gpos, 1);
}
}
// mark goal cells on result map
rIter = Util::RectIterator(mapPos, pos + Vec2i(uType->getSize()));
while (rIter.more()) {
Vec2i gpos = rIter.next();
if (tmpMap.getInfluence(gpos) || !masterMap->canOccupy(gpos, size, f)) {
continue; // building or obstacle
}
Util::PerimeterIterator pIter(gpos - Vec2i(1), gpos + Vec2i(size));
while (pIter.more()) {
if (tmpMap.getInfluence(pIter.next())) {
pMap->setInfluence(gpos, 1);
break;
}
}
}
return pMap;
}
int NamdHybridLB::requiredProxies(PatchID id, int neighborNodes[])
{
PatchMap* patchMap = PatchMap::Object();
int myNode = patchMap->node(id);
int nProxyNodes = 0;
#define IF_NEW_NODE \
int j; \
for ( j=0; j<nProxyNodes && neighborNodes[j] != proxyNode; ++j ); \
if ( j == nProxyNodes )
PatchID neighbors[1 + PatchMap::MaxOneAway + PatchMap::MaxTwoAway];
neighbors[0] = id;
int numNeighbors = 1 + patchMap->downstreamNeighbors(id,neighbors+1);
for ( int i = 0; i < numNeighbors; ++i ) {
const int proxyNode = patchMap->basenode(neighbors[i]);
if ( proxyNode != myNode ) {
IF_NEW_NODE {
neighborNodes[nProxyNodes] = proxyNode;
nProxyNodes++;
}
}
}
void Sync::triggerCompute()
{
PatchMap *patchMap = PatchMap::Object();
const int numHomePatches = patchMap->numHomePatches();
if (numPatches == -1)
numPatches = ProxyMgr::Object()->numProxies() + numHomePatches;
// if (CkMyPe()<=8) CkPrintf("SYNC[%d]: PATCHREADY:%d %d patches:%d %d\n", CkMyPe(), counter, numHomePatches, nPatcheReady, numPatches);
// CkPrintf("SYNC[%d]: PATCHREADY:%d %d patches:%d %d\n", CkMyPe(), counter, PatchMap::Object()->numHomePatches(), nPatcheReady, numPatches);
if (homeReady == 0 && counter >= numHomePatches) {
homeReady = 1;
// if (CkMyPe()<=8) CkPrintf("HOMEREADY[%d]\n", CkMyPe());
if (!useProxySync) releaseComputes();
}
if (homeReady && nPatcheReady == numPatches)
{
// if (CkMyPe()<=8) CkPrintf("TRIGGERED[%d]\n", CkMyPe());
// CkPrintf("TRIGGERED[%d]\n", CkMyPe());
if (useProxySync) releaseComputes();
// reset counter
numPatches = -1;
step++;
nPatcheReady = 0;
for (int i= 0; i<cnum; i++) {
if (clist[i].pid != -1 && clist[i].step == step) ++nPatcheReady;
}
homeReady = 0;
if ( numHomePatches ) {
counter -= numHomePatches;
if (counter >= numHomePatches) triggerCompute();
}
}
}
/**
* @brief Builds the data structures required for the load balancing strategies in NAMD.
*/
int NamdHybridLB::buildData(LDStats* stats) {
int n_pes = stats->nprocs();
PatchMap* patchMap = PatchMap::Object();
ComputeMap* computeMap = ComputeMap::Object();
const SimParameters* simParams = Node::Object()->simParameters;
BigReal bgfactor = simParams->ldbBackgroundScaling;
BigReal pmebgfactor = simParams->ldbPMEBackgroundScaling;
BigReal homebgfactor = simParams->ldbHomeBackgroundScaling;
int pmeOn = simParams->PMEOn;
int unLoadPme = simParams->ldbUnloadPME;
int pmeBarrier = simParams->PMEBarrier;
int unLoadZero = simParams->ldbUnloadZero;
int unLoadOne = simParams->ldbUnloadOne;
int unLoadIO= simParams->ldbUnloadOutputPEs;
// traversing the list of processors and getting their load information
int i, pe_no;
for (i=0; i<n_pes; ++i) {
pe_no = stats->procs[i].pe;
// BACKUP processorArray[i].Id = i;
processorArray[i].Id = pe_no; // absolute pe number
processorArray[i].available = true;
// BACKUP if ( pmeOn && isPmeProcessor(i) )
if ( pmeOn && isPmeProcessor(pe_no) ) {
processorArray[i].backgroundLoad = pmebgfactor * stats->procs[i].bg_walltime;
// BACKUP } else if (patchMap->numPatchesOnNode(i) > 0) {
} else if (patchMap->numPatchesOnNode(pe_no) > 0) {
processorArray[i].backgroundLoad = homebgfactor * stats->procs[i].bg_walltime;
} else {
processorArray[i].backgroundLoad = bgfactor * stats->procs[i].bg_walltime;
}
processorArray[i].idleTime = stats->procs[i].idletime;
processorArray[i].load = processorArray[i].computeLoad = 0.0;
}
// If I am group zero, then offload processor 0 and 1 in my group
if(stats->procs[0].pe == 0) {
if(unLoadZero) processorArray[0].available = false;
if(unLoadOne) processorArray[1].available = false;
}
// if all pes are Pme, disable this flag
if (pmeOn && unLoadPme) {
for (i=0; i<n_pes; i++) {
if(!isPmeProcessor(stats->procs[i].pe)) break;
}
if (i == n_pes) {
iout << iINFO << "Turned off unLoadPme flag!\n" << endi;
unLoadPme = 0;
}
}
if (pmeOn && unLoadPme) {
for (i=0; i<n_pes; i++) {
if ((pmeBarrier && i==0) || isPmeProcessor(stats->procs[i].pe))
processorArray[i].available = false;
}
}
// if all pes are output, disable this flag
#ifdef MEM_OPT_VERSION
if (unLoadIO) {
if (simParams->numoutputprocs == n_pes) {
iout << iINFO << "Turned off unLoadIO flag!\n" << endi;
unLoadIO = 0;
}
}
if (unLoadIO){
for (i=0; i<n_pes; i++) {
if (isOutputProcessor(stats->procs[i].pe))
processorArray[i].available = false;
}
}
#endif
// need to go over all patches to get all required proxies
int numPatches = patchMap->numPatches();
int totalLocalProxies = 0;
int totalProxies = 0;
for ( int pid=0; pid<numPatches; ++pid ) {
int neighborNodes[PatchMap::MaxOneAway + PatchMap::MaxTwoAway];
patchArray[pid].Id = pid;
patchArray[pid].numAtoms = 0;
patchArray[pid].processor = patchMap->node(pid);
const int numProxies =
#if 0 // USE_TOPOMAP - this function needs to be there for the hybrid case
requiredProxiesOnProcGrid(pid,neighborNodes);
#else
requiredProxies(pid, neighborNodes);
#endif
int numLocalProxies = 0;
for (int k=0; k<numProxies; k++) {
if( (neighborNodes[k] >= stats->procs[0].pe) && (neighborNodes[k] <= stats->procs[n_pes-1].pe) ){
++numLocalProxies;
int index = neighborNodes[k] - stats->procs[0].pe;
processorArray[index].proxies.unchecked_insert(&patchArray[pid]);
patchArray[pid].proxiesOn.unchecked_insert(&processorArray[index]);
}
}
#if 0
if ( numLocalProxies ) {
CkPrintf("LDB Pe %d patch %d has %d local of %d total proxies\n",
CkMyPe(), pid, numLocalProxies, numProxies);
}
#endif
totalLocalProxies += numLocalProxies;
totalProxies += numProxies;
}
#if 0
CkPrintf("LDB Pe %d has %d local of %d total proxies\n",
CkMyPe(), totalLocalProxies, totalProxies);
#endif
int nMoveableComputes=0;
int index;
int j;
// this loop goes over only the objects in this group
for(j=0; j < stats->n_objs; j++) {
const LDObjData &this_obj = stats->objData[j];
int frompe = stats->from_proc[j];
// filter out non-NAMD managed objects (like PME array)
if (this_obj.omID().id.idx != 1) {
// CmiAssert(frompe>=0 && frompe<n_pes);
// CkPrintf("non-NAMD object %d on pe %d with walltime %lf\n",
// this_obj.id().id[0], frompe + stats->procs[0].pe, this_obj.wallTime);
processorArray[frompe].backgroundLoad += this_obj.wallTime;
continue;
}
if (this_obj.id().id[1] == -2) { // Its a patch
// handled above to get required proxies from all patches
processorArray[frompe].backgroundLoad += this_obj.wallTime;
} else if (this_obj.id().id[1] == -3) { // Its a bonded compute
processorArray[frompe].backgroundLoad += this_obj.wallTime;
} else if (this_obj.migratable && this_obj.wallTime != 0.) { // Its a compute
const int cid = this_obj.id().id[0];
const int p0 = computeMap->pid(cid,0);
// For self-interactions, just return the same pid twice
int p1;
if (computeMap->numPids(cid) > 1)
p1 = computeMap->pid(cid,1);
else p1 = p0;
computeArray[nMoveableComputes].Id = cid;
//BACKUP computeArray[nMoveableComputes].oldProcessor = stats->from_proc[j];
if (frompe >= n_pes) { // from outside
CkPrintf("assigning random old processor...this looks broken\n");
computeArray[nMoveableComputes].oldProcessor = CrnRand()%n_pes + stats->procs[0].pe; // random
}
else {
computeArray[nMoveableComputes].oldProcessor = frompe + stats->procs[0].pe;
}
from_procs[nMoveableComputes] = frompe;
//BACKUP2 index = stats->from_proc[j] - stats->procs[0].pe;
//BACKUP processorArray[stats->from_proc[j]].computeLoad += this_obj.wallTime;
int index = computeArray[nMoveableComputes].oldProcessor - stats->procs[0].pe;
processorArray[index].computeLoad += this_obj.wallTime;
computeArray[nMoveableComputes].processor = -1;
computeArray[nMoveableComputes].patch1 = p0;
computeArray[nMoveableComputes].patch2 = p1;
computeArray[nMoveableComputes].handle = this_obj.handle;
computeArray[nMoveableComputes].load = this_obj.wallTime;
nMoveableComputes++;
}
}
for (i=0; i<n_pes; i++) {
processorArray[i].load = processorArray[i].backgroundLoad + processorArray[i].computeLoad;
}
stats->clear();
return nMoveableComputes;
}
void OrbLB::mapPartitionsToNodes()
{
int i,j;
#if 1
if (!_lb_args.ignoreBgLoad()) {
// processor mapping has already been determined by the background load pe
for (i=0; i<npartition; i++) partitions[i].node = partitions[i].bkpes[0];
}
else {
int n = 0;
for (i=0; i<P; i++) {
if (!statsData->procs[i].available) continue;
partitions[n++].node = i;
}
}
#else
PatchMap *patchMap = PatchMap::Object();
int **pool = new int *[P];
for (i=0; i<P; i++) pool[i] = new int[P];
for (i=0; i<P; i++) for (j=0; j<P; j++) pool[i][j] = 0;
// sum up the number of nodes that patches of computes are on
for (i=0; i<numComputes; i++)
{
for (j=0; j<P; j++)
if (computeLoad[i].refno == partitions[j].refno)
{
int node1 = patchMap->node(computes[i].patch1);
int node2 = patchMap->node(computes[i].patch2);
pool[j][node1]++;
pool[j][node2]++;
}
}
#ifdef DEBUG
for (i=0; i<P; i++) {
for (j=0; j<P; j++) CmiPrintf("%d ", pool[i][j]);
CmiPrintf("\n");
}
#endif
while (1)
{
int index=-1, node=0, eager=-1;
for (j=0; j<npartition; j++) {
if (partitions[j].node != -1) continue;
int wantmost=-1, maxnodes=-1;
for (k=0; k<P; k++) if (pool[j][k] > maxnodes && !partitions[k].mapped) {wantmost=k; maxnodes = pool[j][k];}
if (maxnodes > eager) {
index = j; node = wantmost; eager = maxnodes;
}
}
if (eager == -1) break;
partitions[index].node = node;
partitions[node].mapped = 1;
}
for (i=0; i<P; i++) delete [] pool[i];
delete [] pool;
#endif
/*
if (_lb_args.debug()) {
CmiPrintf("partition load: ");
for (i=0; i<npartition; i++) CmiPrintf("%f ", partitions[i].load);
CmiPrintf("\n");
CmiPrintf("partitions to nodes mapping: ");
for (i=0; i<npartition; i++) CmiPrintf("%d ", partitions[i].node);
CmiPrintf("\n");
}
*/
if (_lb_args.debug()) {
CmiPrintf("After partitioning: \n");
for (i=0; i<npartition; i++) {
double bgload = 0.0;
if (!_lb_args.ignoreBgLoad())
bgload = statsData->procs[partitions[i].bkpes[0]].bg_walltime;
CmiPrintf("[%d=>%d] (%d,%d,%d) (%d,%d,%d) load:%f count:%d objload:%f\n", i, partitions[i].node, partitions[i].origin[0], partitions[i].origin[1], partitions[i].origin[2], partitions[i].corner[0], partitions[i].corner[1], partitions[i].corner[2], partitions[i].load, partitions[i].count, partitions[i].load-bgload);
}
for (i=npartition; i<P; i++) CmiPrintf("[%d] --------- \n", i);
}
}
int main (int argc, char** argv)
{
int format = 1;
int n[3] = { 2, 2, 2 };
int dims = 3;
int skip=1;
int start=0;
int end=-1;
bool last=false;
char* infile = 0;
char* vtffile = 0;
float starttime = -1, endtime = -1;
for (int i = 1; i < argc; i++)
if (!strcmp(argv[i],"-format") && i < argc-1) {
if (!strcasecmp(argv[++i],"ascii"))
format = 0;
else if (!strcasecmp(argv[i],"binary"))
format = 1;
else
format = atoi(argv[i]);
}
else if (!strcmp(argv[i],"-nviz") && i < argc-1)
n[0] = n[1] = n[2] = atoi(argv[++i]);
else if (!strcmp(argv[i],"-1D"))
dims = 1;
else if (!strcmp(argv[i],"-2D"))
dims = 2;
else if (!strcmp(argv[i],"-last"))
last = true;
else if (!strcmp(argv[i],"-start") && i < argc-1)
start = atoi(argv[++i]);
else if (!strcmp(argv[i],"-starttime") && i < argc-1)
starttime = atof(argv[++i]);
else if (!strcmp(argv[i],"-end") && i < argc-1)
end = atoi(argv[++i]);
else if (!strcmp(argv[i],"-endtime") && i < argc-1)
endtime = atof(argv[++i]);
else if (!strcmp(argv[i],"-ndump") && i < argc-1)
skip = atoi(argv[++i]);
else if (!infile)
infile = argv[i];
else if (!vtffile)
vtffile = argv[i];
else
std::cerr <<" ** Unknown option ignored: "<< argv[i] << std::endl;
if (!infile) {
std::cout <<"usage: "<< argv[0]
<<" <inputfile> [<vtffile>|<vtufile>] [-nviz <nviz>] \n"
<< "[-ndump <ndump>] [-last] [-start <level>] [-end <level>]\n"
<< "[-starttime <time>] [-endtime <time>] [-1D|-2D]\n"
<< "[-format <0|1|ASCII|BINARY>]\n";
return 0;
}
else if (!vtffile)
vtffile = infile;
std::cout <<"\n >>> IFEM HDF5 to VT[F|U] converter <<<"
<<"\n ==================================\n"
<<"\nInput file: " << infile;
std::cout <<"\nOutput file: "<< vtffile
<<"\nNumber of visualization points: "
<< n[0] <<" "<< n[1] << " " << n[2] << std::endl;
VTF* myVtf;
if (strstr(vtffile,".vtf"))
myVtf = new VTF(vtffile,format);
else
myVtf = new VTU(vtffile,last?1:0);
// Process XML - establish fields and collapse bases
PatchMap patches;
HDF5Writer hdf(strtok(infile,"."),ProcessAdm(),true,true);
XMLWriter xml(infile,ProcessAdm());
xml.readInfo();
int levels = xml.getLastTimeLevel();
std::cout <<"Reading "<< infile <<": Time levels = "<< levels << std::endl;
const std::vector<XMLWriter::Entry>& entry = xml.getEntries();
std::vector<XMLWriter::Entry>::const_iterator it;
ProcessList processlist;
for (it = entry.begin(); it != entry.end(); ++it) {
if (!it->basis.empty() && it->type != "restart") {
processlist[it->basis].push_back(*it);
std::cout << it->name <<"\t"<< it->description <<"\tnc="<< it->components
<<"\t"<< it->basis << std::endl;
}
if (it->type == "eigenmodes") {
levels = it->components-1;
processlist[it->basis].back().components = 1;
}
if (it->type == "nodalforces")
processlist["nodalforces"].push_back(*it);
}
if (processlist.empty()) {
std::cout << "No fields to process, bailing" << std::endl;
exit(1);
}
ProcessList::const_iterator pit = processlist.begin();
double time = 0.0;
// setup step boundaries and initial time
if (starttime > 0)
start = (int)(floor(starttime/pit->second.begin()->timestep));
if (endtime > 0)
end = int(endtime/pit->second.begin()->timestep+0.5f);
if (end == -1)
end = levels;
time=last?end *pit->second.begin()->timestep:
start*pit->second.begin()->timestep;
bool ok = true;
int block = 0;
VTFFieldBlocks fieldBlocks;
int k = 1;
for (int i = last?end:start; i <= end && ok; i += skip) {
if (levels > 0) {
if (processlist.begin()->second.begin()->timestep > 0) {
hdf.readDouble(i,"timeinfo","SIMbase-1",time);
std::cout <<"Time level "<< i;
std::cout << " (t=" << time << ")";
} else
std::cout << "Step " << i+1;
std::cout << std::endl;
}
VTFList vlist, slist;
bool geomWritten=false;
if ((isLR && hdf.hasGeometries(i)) || patches.empty()) {
patches = setupPatchMap(processlist, hdf.hasGeometries(i)?i:0, hdf, dims, n, *myVtf, block, k);
geomWritten = true;
}
for (pit = processlist.begin(); pit != processlist.end(); ++pit) {
for (it = pit->second.begin(); it != pit->second.end() && ok; ++it) {
if (it->once && k > 1)
continue;
if (pit->first != "nodalforces" && patches[pit->first].Patch.empty()) {
if (k == 1)
std::cerr << "Ignoring \"" << it->name << "\", basis not loaded" << std::endl;
continue;
}
std::cout <<"Reading \""<< it->name <<"\""<< std::endl;
if (pit->first == "nodalforces") {
Vector vec;
hdf.readVector(i, it->name, -1, vec);
std::vector<Vec3Pair> pts(vec.size()/6);
for (size_t j=0;j<vec.size()/6;++j) {
for (int l=0;l<3;++l) {
pts[j].first[l] = vec[6*j+l];
pts[j].second[l] = vec[6*j+l+3];
}
}
int geoBlck=-1;
ok = myVtf->writeVectors(pts,geoBlck,++block,it->name.c_str(),k);
continue;
}
for( int j=0;j<pit->second[0].patches;++j) {
Vector vec;
ok = hdf.readVector(it->once?0:i,it->name,j+1,vec);
if (it->name.find('+') != std::string::npos) {
/*
Temporary hack to split a vector into scalar fields.
The big assumption here is that the individual scalar names
are separated by '+'-characters in the vector field name
*/
Matrix tmp(it->components,vec.size()/it->components);
tmp.fill(vec.ptr());
size_t pos = 0;
size_t fp = it->name.find('+');
std::string prefix;
if (fp != std::string::npos) {
size_t fs = it->name.find(' ');
if (fs < fp) {
prefix = it->name.substr(0,fs+1);
pos = fs+1;
}
}
for (size_t r = 1; r <= tmp.rows() && pos < it->name.size(); r++) {
size_t end = it->name.find('+',pos);
ok &= writeFieldPatch(tmp.getRow(r),1, *patches[pit->first].Patch[j],
patches[pit->first].FakeModel[j],
patches[pit->first].StartPart+j,
block, prefix+it->name.substr(pos,end-pos),
vlist, slist, *myVtf, it->description, it->type);
pos = end+1;
}
}
else {
if (it->type == "knotspan") {
ok &= writeElmPatch(vec,*patches[pit->first].Patch[j],myVtf->getBlock(j+1),
patches[pit->first].StartPart+j,block,
it->description, it->name, slist, *myVtf);
} else if (it->type == "eigenmodes") {
ok &= writeFieldPatch(vec,it->components,
*patches[pit->first].Patch[j],
patches[pit->first].FakeModel[j],
patches[pit->first].StartPart+j,
block,it->name,vlist,slist,*myVtf, it->description, it->type);
} else {
ok &= writeFieldPatch(vec,it->components,
*patches[pit->first].Patch[j],
patches[pit->first].FakeModel[j],
patches[pit->first].StartPart+j,
block,it->name,vlist,slist,*myVtf, it->description, it->type);
}
}
}
}
}
if (geomWritten)
myVtf->writeGeometryBlocks(k);
writeFieldBlocks(vlist,slist,*myVtf,k,fieldBlocks);
if (!ok)
return 3;
bool res;
if (processlist.begin()->second.begin()->type == "eigenmodes") {
double val;
bool freq=false;
if (!hdf.readDouble(i, "1", "eigenval", val)) {
freq = true;
hdf.readDouble(i, "1", "eigenfrequency", val);
}
res=myVtf->writeState(k++, freq?"Frequency %g" : "Eigenvalue %g", val, 1);
} else if (processlist.begin()->second.begin()->timestep > 0)
res=myVtf->writeState(k++,"Time %g",time,0);
else {
double foo = k;
res=myVtf->writeState(k++,"Step %g", foo, 0);
}
if (!res) {
std::cerr << "Error writing state" << std::endl;
return 4;
}
pit = processlist.begin();
time += pit->second.begin()->timestep*skip;
}
hdf.closeFile(levels,true);
delete myVtf;
return 0;
}
void ComputeGlobal::recvResults(ComputeGlobalResultsMsg *msg) {
DebugM(3,"Receiving results (" << msg->aid.size() << " forces, "
<< msg->newgdef.size() << " new group atoms) on client\n");
// set the forces only if we aren't going to resend the data
int setForces = !msg->resendCoordinates;
if(setForces) { // we are requested to
// Store forces to patches
PatchMap *patchMap = PatchMap::Object();
int numPatches = patchMap->numPatches();
AtomMap *atomMap = AtomMap::Object();
const Lattice & lattice = patchList[0].p->lattice;
ResizeArrayIter<PatchElem> ap(patchList);
Force **f = new Force*[numPatches];
FullAtom **t = new FullAtom*[numPatches];
for ( int i = 0; i < numPatches; ++i ) { f[i] = 0; t[i] = 0; }
Force extForce = 0.;
Tensor extVirial;
for (ap = ap.begin(); ap != ap.end(); ap++) {
(*ap).r = (*ap).forceBox->open();
f[(*ap).patchID] = (*ap).r->f[Results::normal];
t[(*ap).patchID] = (*ap).p->getAtomList().begin();
}
AtomIDList::iterator a = msg->aid.begin();
AtomIDList::iterator a_e = msg->aid.end();
ForceList::iterator f2 = msg->f.begin();
for ( ; a != a_e; ++a, ++f2 ) {
DebugM(1,"processing atom "<<(*a)<<", F="<<(*f2)<<"...\n");
/* XXX if (*a) is out of bounds here we get a segfault */
LocalID localID = atomMap->localID(*a);
if ( localID.pid == notUsed || ! f[localID.pid] ) continue;
Force f_atom = (*f2);
f[localID.pid][localID.index] += f_atom;
Position x_orig = t[localID.pid][localID.index].position;
Transform trans = t[localID.pid][localID.index].transform;
Position x_atom = lattice.reverse_transform(x_orig,trans);
extForce += f_atom;
extVirial += outer(f_atom,x_atom);
}
DebugM(1,"done with the loop\n");
// calculate forces for atoms in groups
Molecule *mol = Node::Object()->molecule;
AtomIDList::iterator g_i, g_e;
g_i = gdef.begin(); g_e = gdef.end();
ResizeArray<BigReal>::iterator gm_i = gmass.begin();
ForceList::iterator gf_i = msg->gforce.begin();
//iout << iDEBUG << "recvResults\n" << endi;
for ( ; g_i != g_e; ++g_i, ++gm_i, ++gf_i ) {
//iout << iDEBUG << *gf_i << '\n' << endi;
Vector accel = (*gf_i) / (*gm_i);
for ( ; *g_i != -1; ++g_i ) {
//iout << iDEBUG << *g_i << '\n' << endi;
LocalID localID = atomMap->localID(*g_i);
if ( localID.pid == notUsed || ! f[localID.pid] ) continue;
Force f_atom = accel * mol->atommass(*g_i);
f[localID.pid][localID.index] += f_atom;
Position x_orig = t[localID.pid][localID.index].position;
Transform trans = t[localID.pid][localID.index].transform;
Position x_atom = lattice.reverse_transform(x_orig,trans);
extForce += f_atom;
extVirial += outer(f_atom,x_atom);
}
}
DebugM(1,"done with the groups\n");
for (ap = ap.begin(); ap != ap.end(); ap++) {
(*ap).forceBox->close(&((*ap).r));
}
delete [] f;
delete [] t;
ADD_VECTOR_OBJECT(reduction,REDUCTION_EXT_FORCE_NORMAL,extForce);
ADD_TENSOR_OBJECT(reduction,REDUCTION_VIRIAL_NORMAL,extVirial);
reduction->submit();
}
// done setting the forces
// Get reconfiguration if present
if ( msg->reconfig ) configure(msg->newaid, msg->newgdef);
// send another round of data if requested
if(msg->resendCoordinates) {
DebugM(3,"Sending requested data right away\n");
sendData();
}
delete msg;
DebugM(3,"Done processing results\n");
}
void ComputeGlobal::sendData()
{
DebugM(2,"sendData\n");
// Get positions from patches
PatchMap *patchMap = PatchMap::Object();
int numPatches = patchMap->numPatches();
AtomMap *atomMap = AtomMap::Object();
const Lattice & lattice = patchList[0].p->lattice;
ResizeArrayIter<PatchElem> ap(patchList);
CompAtom **x = new CompAtom*[numPatches];
FullAtom **t = new FullAtom*[numPatches];
for ( int i = 0; i < numPatches; ++i ) { x[i] = 0; t[i] = 0; }
int step = -1;
for (ap = ap.begin(); ap != ap.end(); ap++) {
x[(*ap).patchID] = (*ap).positionBox->open();
t[(*ap).patchID] = (*ap).p->getAtomList().begin();
step = (*ap).p->flags.step;
}
ComputeGlobalDataMsg *msg = new ComputeGlobalDataMsg;
msg->step = step;
AtomIDList::iterator a = aid.begin();
AtomIDList::iterator a_e = aid.end();
for ( ; a != a_e; ++a ) {
LocalID localID = atomMap->localID(*a);
if ( localID.pid == notUsed || ! x[localID.pid] ) continue;
msg->aid.add(*a);
Position x_orig = x[localID.pid][localID.index].position;
Transform trans = t[localID.pid][localID.index].transform;
msg->p.add(lattice.reverse_transform(x_orig,trans));
}
// calculate group centers of mass
Molecule *mol = Node::Object()->molecule;
AtomIDList::iterator g_i, g_e;
g_i = gdef.begin(); g_e = gdef.end();
ResizeArray<BigReal>::iterator gm_i = gmass.begin();
for ( ; g_i != g_e; ++g_i, ++gm_i ) {
Vector com(0,0,0);
for ( ; *g_i != -1; ++g_i ) {
LocalID localID = atomMap->localID(*g_i);
if ( localID.pid == notUsed || ! x[localID.pid] ) continue;
Position x_orig = x[localID.pid][localID.index].position;
Transform trans = t[localID.pid][localID.index].transform;
com += lattice.reverse_transform(x_orig,trans) * mol->atommass(*g_i);
}
com /= *gm_i;
DebugM(1,"Adding center of mass "<<com<<"\n");
msg->gcom.add(com);
}
for (ap = ap.begin(); ap != ap.end(); ap++) {
(*ap).positionBox->close(&(x[(*ap).patchID]));
}
msg->fid.swap(fid);
msg->tf.swap(totalForce);
fid.resize(0);
totalForce.resize(0);
delete [] x;
delete [] t;
DebugM(3,"Sending data (" << msg->aid.size() << " positions) on client\n");
comm->sendComputeGlobalData(msg);
}
void registerUserEventsForAllComputeObjs()
{
#ifdef TRACE_COMPUTE_OBJECTS
ComputeMap *map = ComputeMap::Object();
PatchMap *pmap = PatchMap::Object();
char user_des[50];
int p1, p2;
int adim, bdim, cdim;
int t1, t2;
int x1, y1, z1, x2, y2, z2;
int dx, dy, dz;
for (int i=0; i<map->numComputes(); i++)
{
memset(user_des, 0, 50);
switch ( map->type(i) )
{
case computeNonbondedSelfType:
sprintf(user_des, "computeNonBondedSelfType_%d_pid_%d", i, map->pid(i,0));
break;
case computeLCPOType:
sprintf(user_des, "computeLCPOType_%d_pid_%d", i, map->pid(i,0));
break;
case computeNonbondedPairType:
adim = pmap->gridsize_a();
bdim = pmap->gridsize_b();
cdim = pmap->gridsize_c();
p1 = map->pid(i, 0);
t1 = map->trans(i, 0);
x1 = pmap->index_a(p1) + adim * Lattice::offset_a(t1);
y1 = pmap->index_b(p1) + bdim * Lattice::offset_b(t1);
z1 = pmap->index_c(p1) + cdim * Lattice::offset_c(t1);
p2 = map->pid(i, 1);
t2 = map->trans(i, 1);
x2 = pmap->index_a(p2) + adim * Lattice::offset_a(t2);
y2 = pmap->index_b(p2) + bdim * Lattice::offset_b(t2);
z2 = pmap->index_c(p2) + cdim * Lattice::offset_c(t2);
dx = abs(x1-x2);
dy = abs(y1-y2);
dz = abs(z1-z2);
sprintf(user_des, "computeNonBondedPairType_%d(%d,%d,%d)", i, dx,dy,dz);
break;
case computeExclsType:
sprintf(user_des, "computeExclsType_%d", i);
break;
case computeBondsType:
sprintf(user_des, "computeBondsType_%d", i);
break;
case computeAnglesType:
sprintf(user_des, "computeAnglesType_%d", i);
break;
case computeDihedralsType:
sprintf(user_des, "computeDihedralsType_%d", i);
break;
case computeImpropersType:
sprintf(user_des, "computeImpropersType_%d", i);
break;
case computeTholeType:
sprintf(user_des, "computeTholeType_%d", i);
break;
case computeAnisoType:
sprintf(user_des, "computeAnisoType_%d", i);
break;
case computeCrosstermsType:
sprintf(user_des, "computeCrosstermsType_%d", i);
break;
case computeSelfExclsType:
sprintf(user_des, "computeSelfExclsType_%d", i);
break;
case computeSelfBondsType:
sprintf(user_des, "computeSelfBondsType_%d", i);
break;
case computeSelfAnglesType:
sprintf(user_des, "computeSelfAnglesType_%d", i);
break;
case computeSelfDihedralsType:
sprintf(user_des, "computeSelfDihedralsType_%d", i);
break;
case computeSelfImpropersType:
sprintf(user_des, "computeSelfImpropersType_%d", i);
break;
case computeSelfTholeType:
sprintf(user_des, "computeSelfTholeType_%d", i);
break;
case computeSelfAnisoType:
sprintf(user_des, "computeSelfAnisoType_%d", i);
break;
case computeSelfCrosstermsType:
sprintf(user_des, "computeSelfCrosstermsType_%d", i);
break;
#ifdef DPMTA
case computeDPMTAType:
sprintf(user_des, "computeDPMTAType_%d", i);
break;
#endif
#ifdef DPME
case computeDPMEType:
sprintf(user_des, "computeDPMEType_%d", i);
break;
#endif
case computePmeType:
sprintf(user_des, "computePMEType_%d", i);
break;
case computeEwaldType:
sprintf(user_des, "computeEwaldType_%d", i);
break;
case computeFullDirectType:
sprintf(user_des, "computeFullDirectType_%d", i);
break;
case computeGlobalType:
sprintf(user_des, "computeGlobalType_%d", i);
break;
case computeStirType:
sprintf(user_des, "computeStirType_%d", i);
break;
case computeExtType:
sprintf(user_des, "computeExtType_%d", i);
break;
case computeEFieldType:
sprintf(user_des, "computeEFieldType_%d", i);
break;
/* BEGIN gf */
case computeGridForceType:
sprintf(user_des, "computeGridForceType_%d", i);
break;
/* END gf */
case computeSphericalBCType:
sprintf(user_des, "computeSphericalBCType_%d", i);
break;
case computeCylindricalBCType:
sprintf(user_des, "computeCylindricalBCType_%d", i);
break;
case computeTclBCType:
sprintf(user_des, "computeTclBCType_%d", i);
break;
case computeRestraintsType:
sprintf(user_des, "computeRestraintsType_%d", i);
break;
case computeConsForceType:
sprintf(user_des, "computeConsForceType_%d", i);
break;
case computeConsTorqueType:
sprintf(user_des, "computeConsTorqueType_%d", i);
break;
default:
NAMD_bug("Unknown compute type in ComputeMgr::registerUserEventForAllComputeObjs().");
break;
}
int user_des_len = strlen(user_des);
char *user_des_cst = new char[user_des_len+1];
memcpy(user_des_cst, user_des, user_des_len);
user_des_cst[user_des_len] = 0;
//Since the argument in traceRegisterUserEvent is supposed
//to be a const string which will not be copied inside the
//function when a new user event is created, user_des_cst
//has to be allocated in heap.
int reEvenId = traceRegisterUserEvent(user_des_cst, TRACE_COMPOBJ_IDOFFSET+i);
//printf("Register user event (%s) with id (%d)\n", user_des, reEvenId);
}
#else
return;
#endif
}
int ComputeNonbondedPair::noWork() {
if (patch[0]->flags.doGBIS) {
gbisPhase = 1 + (gbisPhase % 3);//1->2->3->1...
}
#ifndef NAMD_CUDA
if ( patch[0]->flags.doNonbonded && (numAtoms[0] && numAtoms[1]) ) {
return 0; // work to do, enqueue as usual
} else {
#else
{
#endif
if (patch[0]->flags.doGBIS) {
if (gbisPhase == 1) {
for (int i=0; i<2; i++) {
psiSumBox[i]->skip();
intRadBox[i]->skip();
}
if (patch[0]->flags.doNonbonded) return 1;
else gbisPhase = 2;
}
if (gbisPhase == 2) {
for (int i=0; i<2; i++) {
bornRadBox[i]->skip();
dEdaSumBox[i]->skip();
}
if (patch[0]->flags.doNonbonded) return 1;
else gbisPhase = 3;
}
if (gbisPhase == 3) {
for (int i=0; i<2; i++) {
dHdrPrefixBox[i]->skip();
}
}
}
// skip all boxes
for (int i=0; i<2; i++) {
positionBox[i]->skip();
forceBox[i]->skip();
if ( patch[0]->flags.doMolly ) avgPositionBox[i]->skip();
// BEGIN LA
if (patch[0]->flags.doLoweAndersen) velocityBox[i]->skip();
// END LA
}
reduction->item(REDUCTION_COMPUTE_CHECKSUM) += 1.;
reduction->submit();
if (accelMDOn) amd_reduction->submit();
if (pressureProfileOn)
pressureProfileReduction->submit();
#ifndef NAMD_CUDA
// Inform load balancer
LdbCoordinator::Object()->skipWork(ldObjHandle);
#endif
return 1; // no work to do, do not enqueue
}
}
void ComputeNonbondedPair::doForce(CompAtom* p[2], CompAtomExt* pExt[2], Results* r[2])
{
// Inform load balancer.
// I assume no threads will suspend until endWork is called
//single phase declarations
int doEnergy = patch[0]->flags.doEnergy;
int a = 0; int b = 1;
// swap to place more atoms in inner loop (second patch)
if ( numAtoms[0] > numAtoms[1] ) { a = 1; b = 0; }
CompAtom* v[2];
/*******************************************************************************
* Prepare Parameters
*******************************************************************************/
if (!patch[0]->flags.doGBIS || gbisPhase == 1) {
#ifdef TRACE_COMPUTE_OBJECTS
double traceObjStartTime = CmiWallTimer();
#endif
DebugM(2,"doForce() called.\n");
DebugM(2, numAtoms[0] << " patch #1 atoms and " <<
numAtoms[1] << " patch #2 atoms\n");
for ( int i = 0; i < reductionDataSize; ++i )
reductionData[i] = 0;
if (pressureProfileOn) {
int n = pressureProfileAtomTypes;
memset(pressureProfileData, 0, 3*n*n*pressureProfileSlabs*sizeof(BigReal));
// adjust lattice dimensions to allow constant pressure
const Lattice &lattice = patch[0]->lattice;
pressureProfileThickness = lattice.c().z / pressureProfileSlabs;
pressureProfileMin = lattice.origin().z - 0.5*lattice.c().z;
}
params.reduction = reductionData;
params.pressureProfileReduction = pressureProfileData;
params.minPart = minPart;
params.maxPart = maxPart;
params.numParts = numParts;
params.workArrays = workArrays;
params.pairlists = &pairlists;
params.savePairlists = 0;
params.usePairlists = 0;
if ( patch[0]->flags.savePairlists ) {
params.savePairlists = 1;
params.usePairlists = 1;
} else if ( patch[0]->flags.usePairlists && patch[1]->flags.usePairlists ) {
if ( ! pairlistsValid ||
( patch[0]->flags.maxAtomMovement +
patch[1]->flags.maxAtomMovement > pairlistTolerance ) ) {
reductionData[pairlistWarningIndex] += 1;
} else {
params.usePairlists = 1;
}
}
if ( ! params.usePairlists ) {
pairlistsValid = 0;
}
params.plcutoff = cutoff;
params.groupplcutoff = cutoff +
patch[0]->flags.maxGroupRadius + patch[1]->flags.maxGroupRadius;
if ( params.savePairlists ) {
pairlistsValid = 1;
pairlistTolerance = patch[0]->flags.pairlistTolerance +
patch[1]->flags.pairlistTolerance;
params.plcutoff += pairlistTolerance;
params.groupplcutoff += pairlistTolerance;
}
const Lattice &lattice = patch[0]->lattice;
params.offset = lattice.offset(trans[a]) - lattice.offset(trans[b]);
// Atom Sorting : If we are sorting the atoms along the line connecting
// the patch centers, then calculate a normalized vector pointing from
// patch a to patch b (i.e. outer loop patch to inner loop patch).
#if NAMD_ComputeNonbonded_SortAtoms != 0
// Center of patch a (outer-loop; i-loop) and patch b (inner-loop; j/k-loop)
PatchMap* patchMap = PatchMap::Object();
ScaledPosition p_a_center, p_b_center;
p_a_center.x = patchMap->min_a(patchID[a]);
p_a_center.y = patchMap->min_b(patchID[a]);
p_a_center.z = patchMap->min_c(patchID[a]);
p_b_center.x = patchMap->min_a(patchID[b]);
p_b_center.y = patchMap->min_b(patchID[b]);
p_b_center.z = patchMap->min_c(patchID[b]);
p_a_center.x += patchMap->max_a(patchID[a]);
p_a_center.y += patchMap->max_b(patchID[a]);
p_a_center.z += patchMap->max_c(patchID[a]);
p_b_center.x += patchMap->max_a(patchID[b]);
p_b_center.y += patchMap->max_b(patchID[b]);
p_b_center.z += patchMap->max_c(patchID[b]);
p_a_center *= (BigReal)0.5;
p_b_center *= (BigReal)0.5;
p_a_center = lattice.unscale(p_a_center);
p_b_center = lattice.unscale(p_b_center);
// Adjust patch a's center by the offset
p_a_center.x += params.offset.x;
p_a_center.y += params.offset.y;
p_a_center.z += params.offset.z;
// Calculate and fill in the projected line vector
params.projLineVec = p_b_center - p_a_center;
params.projLineVec /= params.projLineVec.length(); // Normalize the vector
#endif
params.p[0] = p[a];
params.p[1] = p[b];
params.pExt[0] = pExt[a];
params.pExt[1] = pExt[b];
// BEGIN LA
params.doLoweAndersen = patch[0]->flags.doLoweAndersen;
if (params.doLoweAndersen) {
DebugM(4, "opening velocity boxes\n");
v[0] = velocityBox[0]->open();
v[1] = velocityBox[1]->open();
params.v[0] = v[a];
params.v[1] = v[b];
}
// END LA
params.ff[0] = r[a]->f[Results::nbond];
params.ff[1] = r[b]->f[Results::nbond];
params.numAtoms[0] = numAtoms[a];
params.numAtoms[1] = numAtoms[b];
// DMK - Atom Separation (water vs. non-water)
#if NAMD_SeparateWaters != 0
params.numWaterAtoms[0] = numWaterAtoms[a];
params.numWaterAtoms[1] = numWaterAtoms[b];
#endif
/*******************************************************************************
* Call Nonbonded Functions
*******************************************************************************/
if (numAtoms[0] && numAtoms[1]) {//only do if has atoms since gbis noWork doesn't account for no atoms
//force calculation calls
if ( patch[0]->flags.doFullElectrostatics )
{
params.fullf[0] = r[a]->f[Results::slow];
params.fullf[1] = r[b]->f[Results::slow];
if ( patch[0]->flags.doMolly ) {
if ( doEnergy )
calcPairEnergy(¶ms);
else calcPair(¶ms);
CompAtom *p_avg[2];
p_avg[0] = avgPositionBox[0]->open();
p_avg[1] = avgPositionBox[1]->open();
params.p[0] = p_avg[a];
params.p[1] = p_avg[b];
if ( doEnergy ) calcSlowPairEnergy(¶ms);
else calcSlowPair(¶ms);
avgPositionBox[0]->close(&p_avg[0]);
avgPositionBox[1]->close(&p_avg[1]);
} else if ( patch[0]->flags.maxForceMerged == Results::slow ) {
if ( doEnergy ) calcMergePairEnergy(¶ms);
else calcMergePair(¶ms);
} else {
if ( doEnergy ) calcFullPairEnergy(¶ms);
else calcFullPair(¶ms);
}
}
else
if ( doEnergy ) calcPairEnergy(¶ms);
else calcPair(¶ms);
}//end if has atoms
// BEGIN LA
if (params.doLoweAndersen) {
DebugM(4, "closing velocity boxes\n");
velocityBox[0]->close(&v[0]);
velocityBox[1]->close(&v[1]);
}
// END LA
}// end not gbis
/*******************************************************************************
* gbis Loop
*******************************************************************************/
if (patch[0]->flags.doGBIS) {
SimParameters *simParams = Node::Object()->simParameters;
gbisParams.sequence = sequence();
gbisParams.doGBIS = patch[0]->flags.doGBIS;
gbisParams.numPatches = 2;//pair
gbisParams.gbisPhase = gbisPhase;
gbisParams.doFullElectrostatics = patch[0]->flags.doFullElectrostatics;
gbisParams.epsilon_s = simParams->solvent_dielectric;
gbisParams.epsilon_p = simParams->dielectric;
gbisParams.rho_0 = simParams->coulomb_radius_offset;
gbisParams.kappa = simParams->kappa;
gbisParams.cutoff = simParams->cutoff;
gbisParams.doSmoothing = simParams->switchingActive;
gbisParams.a_cut = simParams->alpha_cutoff;
gbisParams.delta = simParams->gbis_delta;
gbisParams.beta = simParams->gbis_beta;
gbisParams.gamma = simParams->gbis_gamma;
gbisParams.alpha_max = simParams->alpha_max;
gbisParams.cid = cid;
gbisParams.patchID[0] = patch[a]->getPatchID();
gbisParams.patchID[1] = patch[b]->getPatchID();
gbisParams.maxGroupRadius = patch[0]->flags.maxGroupRadius;
if (patch[1]->flags.maxGroupRadius > gbisParams.maxGroupRadius)
gbisParams.maxGroupRadius = patch[1]->flags.maxGroupRadius;
gbisParams.doEnergy = doEnergy;
gbisParams.fsMax = simParams->fsMax;
for (int i = 0; i < numGBISPairlists; i++)
gbisParams.gbisStepPairlists[i] = &gbisStepPairlists[i];
//open boxes
if (gbisPhase == 1) {
gbisParams.intRad[0] = intRadBox[a]->open();
gbisParams.intRad[1] = intRadBox[b]->open();
gbisParams.psiSum[0] = psiSumBox[a]->open();
gbisParams.psiSum[1] = psiSumBox[b]->open();
gbisParams.gbInterEnergy=0;
gbisParams.gbSelfEnergy=0;
} else if (gbisPhase == 2) {
gbisParams.bornRad[0] = bornRadBox[a]->open();
gbisParams.bornRad[1] = bornRadBox[b]->open();
gbisParams.dEdaSum[0] = dEdaSumBox[a]->open();
gbisParams.dEdaSum[1] = dEdaSumBox[b]->open();
} else if (gbisPhase == 3) {
gbisParams.dHdrPrefix[0] = dHdrPrefixBox[a]->open();
gbisParams.dHdrPrefix[1] = dHdrPrefixBox[b]->open();
}
//make call to calculate GBIS
if ( !ComputeNonbondedUtil::commOnly ) {
calcGBIS(¶ms,&gbisParams);
}
//close boxes
if (gbisPhase == 1) {
psiSumBox[0]->close(&(gbisParams.psiSum[a]));
psiSumBox[1]->close(&(gbisParams.psiSum[b]));
} else if (gbisPhase == 2) {
dEdaSumBox[0]->close(&(gbisParams.dEdaSum[a]));
dEdaSumBox[1]->close(&(gbisParams.dEdaSum[b]));
} else if (gbisPhase == 3) {
bornRadBox[0]->close(&(gbisParams.bornRad[a]));
bornRadBox[1]->close(&(gbisParams.bornRad[b]));
reduction->item(REDUCTION_ELECT_ENERGY) += gbisParams.gbInterEnergy;
reduction->item(REDUCTION_ELECT_ENERGY) += gbisParams.gbSelfEnergy;
intRadBox[0]->close(&(gbisParams.intRad[a]));
intRadBox[1]->close(&(gbisParams.intRad[b]));
dHdrPrefixBox[0]->close(&(gbisParams.dHdrPrefix[a]));
dHdrPrefixBox[1]->close(&(gbisParams.dHdrPrefix[b]));
}
}//end if doGBIS
if (!patch[0]->flags.doGBIS || gbisPhase == 3) {
submitReductionData(reductionData,reduction);
if (accelMDOn) submitReductionData(reductionData,amd_reduction);
if (pressureProfileOn)
submitPressureProfileData(pressureProfileData, pressureProfileReduction);
#ifdef TRACE_COMPUTE_OBJECTS
traceUserBracketEvent(TRACE_COMPOBJ_IDOFFSET+cid, traceObjStartTime, CmiWallTimer());
#endif
reduction->submit();
if (accelMDOn) amd_reduction->submit();
if (pressureProfileOn)
pressureProfileReduction->submit();
}//end gbis end phase
}//end do Force