void LBNSCommunicator::gatherVelocity(
std::vector<int>& keys,
std::vector<int>& offset,
std::vector<int>& flips,
std::vector<double>& velocities)
{
//_logComm<<"start gather com:"<<_index<<",on rank:"<<_parameters.parallel.rank<<" keys:"<<keys.size()<<std::endl;
gatherArray(_velocity_keys_count_sum,_velocity_keys_count,_velocity_keys_displ,keys);
//_logComm<<"start gather com:"<<_index<<",on rank:"<<_parameters.parallel.rank<<" flips:"<<flips.size()<<std::endl;
gatherArray(_velocity_flips_count_sum,_velocity_flips_count,_velocity_flips_displ,flips);
//_logComm<<"start gather com:"<<_index<<",on rank:"<<_parameters.parallel.rank<<" offset:"<<offset.size()<<std::endl;
gatherArray(_velocity_offset_count_sum,_velocity_offset_count,_velocity_offset_displ,offset);
//_logComm<<"start gather com:"<<_index<<",on rank:"<<_parameters.parallel.rank<<" data:"<<velocities.size()<<std::endl;
gatherArray(_velocity_count_sum,_velocity_count,_velocity_displ,velocities);
//_logComm<<"end gather com:"<<_index<<",on rank:"<<_parameters.parallel.rank<<" data:"<<_velocities.size()<<std::endl;
}
static void
writeGribStream(struct winHeaderEntry *winDict, struct streamMapping *mapping,
double **data_, int *currentDataBufSize, int root,
int nProcsModel)
{
int streamID = mapping->streamID;
int headerIdx, lastHeaderIdx = mapping->lastHeaderIdx;
int vlistID = streamInqVlist(streamID);
if (lastHeaderIdx < 0)
{
/* write zero bytes to trigger synchronization code in fileWrite */
cdiPioFileWrite(streamInqFileID(streamID), NULL, 0,
streamInqCurTimestepID(streamID));
}
else
for (headerIdx = mapping->firstHeaderIdx;
headerIdx <= lastHeaderIdx;
headerIdx += 2)
if (streamID == winDict[headerIdx].id)
{
int varID = winDict[headerIdx].specific.dataRecord.varID;
int size = vlistInqVarSize(vlistID, varID);
int nmiss;
resizeVarGatherBuf(vlistID, varID, data_, currentDataBufSize);
double *data = *data_;
gatherArray(root, nProcsModel, headerIdx,
vlistID, data, &nmiss);
streamWriteVar(streamID, varID, data, nmiss);
if ( ddebug > 2 )
{
char text[1024];
sprintf(text, "streamID=%d, var[%d], size=%d",
streamID, varID, size);
xprintArray(text, data, size, DATATYPE_FLT);
}
}
}
static void readGetBuffers()
{
int nProcsModel = commInqNProcsModel ();
int root = commInqRootGlob ();
#ifdef HAVE_NETCDF4
int myCollRank = commInqRankColl();
MPI_Comm collComm = commInqCommColl();
#endif
xdebug("%s", "START");
struct winHeaderEntry *winDict
= (struct winHeaderEntry *)rxWin[root].buffer;
xassert(winDict[0].id == HEADERSIZEMARKER);
{
int dictSize = rxWin[root].dictSize,
firstNonRPCEntry = dictSize - winDict[0].specific.headerSize.numRPCEntries - 1,
headerIdx,
numFuncCalls = 0;
for (headerIdx = dictSize - 1;
headerIdx > firstNonRPCEntry;
--headerIdx)
{
xassert(winDict[headerIdx].id >= MINFUNCID
&& winDict[headerIdx].id <= MAXFUNCID);
++numFuncCalls;
readFuncCall(winDict + headerIdx);
}
xassert(numFuncCalls == winDict[0].specific.headerSize.numRPCEntries);
}
/* build list of streams, data was transferred for */
{
struct streamMap map = buildStreamMap(winDict);
double *data = NULL;
#ifdef HAVE_NETCDF4
int *varIsWritten = NULL;
#endif
#if defined (HAVE_PARALLEL_NC4)
double *writeBuf = NULL;
#endif
int currentDataBufSize = 0;
for (int streamIdx = 0; streamIdx < map.numEntries; ++streamIdx)
{
int streamID = map.entries[streamIdx].streamID;
int vlistID = streamInqVlist(streamID);
int filetype = map.entries[streamIdx].filetype;
switch (filetype)
{
case FILETYPE_GRB:
case FILETYPE_GRB2:
writeGribStream(winDict, map.entries + streamIdx,
&data, ¤tDataBufSize,
root, nProcsModel);
break;
#ifdef HAVE_NETCDF4
case FILETYPE_NC:
case FILETYPE_NC2:
case FILETYPE_NC4:
#ifdef HAVE_PARALLEL_NC4
/* HAVE_PARALLE_NC4 implies having ScalES-PPM and yaxt */
{
int nvars = map.entries[streamIdx].numVars;
int *varMap = map.entries[streamIdx].varMap;
buildWrittenVars(map.entries + streamIdx, &varIsWritten,
myCollRank, collComm);
for (int varID = 0; varID < nvars; ++varID)
if (varIsWritten[varID])
{
struct PPM_extent varShape[3];
queryVarBounds(varShape, vlistID, varID);
struct xyzDims collGrid = varDimsCollGridMatch(varShape);
xdebug("writing varID %d with dimensions: "
"x=%d, y=%d, z=%d,\n"
"found distribution with dimensions:"
" x=%d, y=%d, z=%d.", varID,
varShape[0].size, varShape[1].size, varShape[2].size,
collGrid.sizes[0], collGrid.sizes[1],
collGrid.sizes[2]);
struct PPM_extent varChunk[3];
myVarPart(varShape, collGrid, varChunk);
int myChunk[3][2];
for (int i = 0; i < 3; ++i)
{
myChunk[i][0] = PPM_extent_start(varChunk[i]);
myChunk[i][1] = PPM_extent_end(varChunk[i]);
}
xdebug("Writing chunk { { %d, %d }, { %d, %d },"
" { %d, %d } }", myChunk[0][0], myChunk[0][1],
myChunk[1][0], myChunk[1][1], myChunk[2][0],
myChunk[2][1]);
Xt_int varSize[3];
for (int i = 0; i < 3; ++i)
varSize[2 - i] = varShape[i].size;
Xt_idxlist preRedistChunk, preWriteChunk;
/* prepare yaxt descriptor for current data
distribution after collect */
int nmiss;
if (varMap[varID] == -1)
{
preRedistChunk = xt_idxempty_new();
xdebug("%s", "I got none\n");
}
else
{
Xt_int preRedistStart[3] = { 0, 0, 0 };
preRedistChunk
= xt_idxsection_new(0, 3, varSize, varSize,
preRedistStart);
resizeVarGatherBuf(vlistID, varID, &data,
¤tDataBufSize);
int headerIdx = varMap[varID];
gatherArray(root, nProcsModel, headerIdx,
vlistID, data, &nmiss);
xdebug("%s", "I got all\n");
}
MPI_Bcast(&nmiss, 1, MPI_INT, varIsWritten[varID] - 1,
collComm);
/* prepare yaxt descriptor for write chunk */
{
Xt_int preWriteChunkStart[3], preWriteChunkSize[3];
for (int i = 0; i < 3; ++i)
{
preWriteChunkStart[2 - i] = varChunk[i].first;
preWriteChunkSize[2 - i] = varChunk[i].size;
}
preWriteChunk = xt_idxsection_new(0, 3, varSize,
preWriteChunkSize,
preWriteChunkStart);
}
/* prepare redistribution */
{
Xt_xmap xmap = xt_xmap_all2all_new(preRedistChunk,
preWriteChunk,
collComm);
Xt_redist redist = xt_redist_p2p_new(xmap, MPI_DOUBLE);
xt_idxlist_delete(preRedistChunk);
xt_idxlist_delete(preWriteChunk);
xt_xmap_delete(xmap);
writeBuf = (double*) xrealloc(writeBuf,
sizeof (double)
* PPM_extents_size(3, varChunk));
xt_redist_s_exchange1(redist, data, writeBuf);
xt_redist_delete(redist);
}
/* write chunk */
streamWriteVarChunk(streamID, varID,
(const int (*)[2])myChunk, writeBuf,
nmiss);
}
}
#else
/* determine process which has stream open (writer) and
* which has data for which variable (var owner)
* three cases need to be distinguished */
{
int nvars = map.entries[streamIdx].numVars;
int *varMap = map.entries[streamIdx].varMap;
buildWrittenVars(map.entries + streamIdx, &varIsWritten,
myCollRank, collComm);
int writerRank;
if ((writerRank = cdiPioSerialOpenFileMap(streamID))
== myCollRank)
{
for (int varID = 0; varID < nvars; ++varID)
if (varIsWritten[varID])
{
int nmiss;
int size = vlistInqVarSize(vlistID, varID);
resizeVarGatherBuf(vlistID, varID, &data,
¤tDataBufSize);
int headerIdx = varMap[varID];
if (varIsWritten[varID] == myCollRank + 1)
{
/* this process has the full array and will
* write it */
xdebug("gathering varID=%d for direct writing",
varID);
gatherArray(root, nProcsModel, headerIdx,
vlistID, data, &nmiss);
}
else
{
/* another process has the array and will
* send it over */
MPI_Status stat;
xdebug("receiving varID=%d for writing from"
" process %d",
varID, varIsWritten[varID] - 1);
xmpiStat(MPI_Recv(&nmiss, 1, MPI_INT,
varIsWritten[varID] - 1,
COLLBUFNMISS,
collComm, &stat), &stat);
xmpiStat(MPI_Recv(data, size, MPI_DOUBLE,
varIsWritten[varID] - 1,
COLLBUFTX,
collComm, &stat), &stat);
}
streamWriteVar(streamID, varID, data, nmiss);
}
}
else
for (int varID = 0; varID < nvars; ++varID)
if (varIsWritten[varID] == myCollRank + 1)
{
/* this process has the full array and another
* will write it */
int nmiss;
int size = vlistInqVarSize(vlistID, varID);
resizeVarGatherBuf(vlistID, varID, &data,
¤tDataBufSize);
int headerIdx = varMap[varID];
gatherArray(root, nProcsModel, headerIdx,
vlistID, data, &nmiss);
MPI_Request req;
MPI_Status stat;
xdebug("sending varID=%d for writing to"
" process %d",
varID, writerRank);
xmpi(MPI_Isend(&nmiss, 1, MPI_INT,
writerRank, COLLBUFNMISS,
collComm, &req));
xmpi(MPI_Send(data, size, MPI_DOUBLE,
writerRank, COLLBUFTX,
collComm));
xmpiStat(MPI_Wait(&req, &stat), &stat);
}
}
#endif
break;
#endif
default:
xabort("unhandled filetype in parallel I/O.");
}
}
#ifdef HAVE_NETCDF4
free(varIsWritten);
#ifdef HAVE_PARALLEL_NC4
free(writeBuf);
#endif
#endif
free(map.entries);
free(data);
}
xdebug("%s", "RETURN");
}
