/*
* Sends a token to all peer Motion Nodes, indicating that this motion
* node has no more tuples to send out.
*/
void
SendEndOfStream(MotionLayerState *mlStates,
ChunkTransportState *transportStates,
int motNodeID)
{
MotionNodeEntry *pMNEntry;
/*
* Pull up the motion node entry with the node's details. This includes
* details that affect sending, such as whether the motion node needs to
* include backup segment-dbs.
*/
pMNEntry = getMotionNodeEntry(mlStates, motNodeID, "SendEndOfStream");
transportStates->SendEos(mlStates, transportStates, motNodeID, s_eos_chunk_data);
/*
* We increment our own "stream-ends received" count when we send our own,
* as well as when we receive one.
*/
pMNEntry->num_stream_ends_recvd++;
/* We record EOS as if a tuple were sent. */
statSendEOS(mlStates, pMNEntry);
}
void
setExpectedReceivers(MotionLayerState *mlStates, int16 motNodeID, int expectedReceivers)
{
MotionNodeEntry *pEntry = getMotionNodeEntry(mlStates, motNodeID, "setExpectedReceivers");
pEntry->num_senders = expectedReceivers;
}
/*
* Set the statistic info in gpmon packet.
*/
static void
setMotionStatsForGpmon(MotionState *node)
{
MotionLayerState *mlStates = (MotionLayerState *)node->ps.state->motionlayer_context;
ChunkTransportState *transportStates = node->ps.state->interconnect_context;
int motionId = ((Motion *) node->ps.plan)->motionID;
MotionNodeEntry *mlEntry = getMotionNodeEntry(mlStates, motionId, "setMotionStatsForGpmon");
ChunkTransportStateEntry *transportEntry = NULL;
getChunkTransportState(transportStates, motionId, &transportEntry);
uint64 avgAckTime = 0;
if (transportEntry->stat_count_acks > 0)
avgAckTime = transportEntry->stat_total_ack_time / transportEntry->stat_count_acks;
Gpmon_M_Set(GpmonPktFromMotionState(node), GPMON_MOTION_BYTES_SENT,
mlEntry->stat_total_bytes_sent);
Gpmon_M_Set(GpmonPktFromMotionState(node), GPMON_MOTION_TOTAL_ACK_TIME,
transportEntry->stat_total_ack_time);
Gpmon_M_Set(GpmonPktFromMotionState(node), GPMON_MOTION_AVG_ACK_TIME,
avgAckTime);
Gpmon_M_Set(GpmonPktFromMotionState(node), GPMON_MOTION_MAX_ACK_TIME,
transportEntry->stat_max_ack_time);
Gpmon_M_Set(GpmonPktFromMotionState(node), GPMON_MOTION_MIN_ACK_TIME,
transportEntry->stat_min_ack_time);
Gpmon_M_Set(GpmonPktFromMotionState(node), GPMON_MOTION_COUNT_RESENT,
transportEntry->stat_count_resent);
Gpmon_M_Set(GpmonPktFromMotionState(node), GPMON_MOTION_MAX_RESENT,
transportEntry->stat_max_resent);
Gpmon_M_Set(GpmonPktFromMotionState(node), GPMON_MOTION_BYTES_RECEIVED,
mlEntry->stat_total_bytes_recvd);
Gpmon_M_Set(GpmonPktFromMotionState(node), GPMON_MOTION_COUNT_DROPPED,
transportEntry->stat_count_dropped);
}
void
SendStopMessage(MotionLayerState *mlStates,
ChunkTransportState *transportStates,
int16 motNodeID)
{
MotionNodeEntry *pEntry = getMotionNodeEntry(mlStates, motNodeID, "SendStopMessage");
pEntry->stopped = true;
if (transportStates != NULL && transportStates->doSendStopMessage != NULL)
transportStates->doSendStopMessage(transportStates, motNodeID);
}
void
EndMotionLayerNode(MotionLayerState *mlStates, int16 motNodeID, bool flushCommLayer)
{
MotionNodeEntry *pMNEntry;
ChunkSorterEntry *pCSEntry;
int i;
pMNEntry = getMotionNodeEntry(mlStates, motNodeID, "EndMotionLayerNode");
#ifdef AMS_VERBOSE_LOGGING
elog(DEBUG5, "Cleaning up Motion Layer details for motion node %d.",
motNodeID);
#endif
/*
* Iterate through all entries in the motion layer's chunk-sort map, to
* see if we have gotten end-of-stream from all senders.
*/
if (pMNEntry->preserve_order && pMNEntry->ready_tuple_lists != NULL)
{
for (i=0; i < GpIdentity.numsegments; i++)
{
pCSEntry = &pMNEntry->ready_tuple_lists[i];
/*
* QD should not expect end-of-stream comes from QEs who is not members of
* direct dispatch
*/
if (!pCSEntry->init)
continue;
if (pMNEntry->preserve_order &&
gp_log_interconnect >= GPVARS_VERBOSITY_DEBUG)
{
/* Print chunk-sorter entry statistics. */
elog(DEBUG4, "Chunk-sorter entry [route=%d,node=%d] statistics:\n"
"\tAvailable Tuples High-Watermark: " UINT64_FORMAT,
i, pMNEntry->motion_node_id,
pMNEntry->stat_tuples_available_hwm);
}
if (!pMNEntry->stopped && !pCSEntry->end_of_stream)
{
if (flushCommLayer)
{
elog(FATAL, "Motion layer node %d cleanup - did not receive"
" end-of-stream from sender %d.", motNodeID, i);
/*** TODO - get chunks until end-of-stream comes in. ***/
}
else
{
elog(LOG, "Motion layer node %d cleanup - did not receive"
" end-of-stream from sender %d.", motNodeID, i);
}
}
else
{
/* End-of-stream is marked for this entry. */
/*** TODO - do more than just complain! ***/
if (pCSEntry->chunk_list.num_chunks > 0)
{
elog(LOG, "Motion layer node %d cleanup - there are still"
" %d chunks enqueued from sender %d.", motNodeID,
pCSEntry->chunk_list.num_chunks, i );
}
/***
TODO - Make sure there are no outstanding tuples in the
tuple-store.
***/
}
/*
* Clean up the chunk-sorter entry, then remove it from the hash
* table.
*/
clearTCList(&pMNEntry->ser_tup_info.chunkCache, &pCSEntry->chunk_list);
if (pMNEntry->preserve_order) /* Clean up the tuple-store. */
htfifo_destroy(pCSEntry->ready_tuples);
}
}
pMNEntry->cleanedUp = true;
/* Clean up the motion-node entry, then remove it from the hash table. */
if (gp_log_interconnect >= GPVARS_VERBOSITY_VERBOSE)
{
if (pMNEntry->stat_total_bytes_sent > 0 ||
pMNEntry->sel_wr_wait > 0)
{
elog(LOG, "Interconnect seg%d slice%d sent " UINT64_FORMAT " tuples, "
UINT64_FORMAT " total bytes, " UINT64_FORMAT " tuple bytes, "
UINT64_FORMAT " chunks; waited " UINT64_FORMAT " usec.",
Gp_segment,
currentSliceId,
pMNEntry->stat_total_sends,
pMNEntry->stat_total_bytes_sent,
pMNEntry->stat_tuple_bytes_sent,
pMNEntry->stat_total_chunks_sent,
pMNEntry->sel_wr_wait
);
}
if (pMNEntry->stat_total_bytes_recvd > 0 ||
pMNEntry->sel_rd_wait > 0)
{
elog(LOG, "Interconnect seg%d slice%d received from slice%d: " UINT64_FORMAT " tuples, "
UINT64_FORMAT " total bytes, " UINT64_FORMAT " tuple bytes, "
UINT64_FORMAT " chunks; waited " UINT64_FORMAT " usec.",
Gp_segment,
currentSliceId,
motNodeID,
pMNEntry->stat_total_recvs,
pMNEntry->stat_total_bytes_recvd,
pMNEntry->stat_tuple_bytes_recvd,
pMNEntry->stat_total_chunks_recvd,
pMNEntry->sel_rd_wait
);
}
}
CleanupSerTupInfo(&pMNEntry->ser_tup_info);
FreeTupleDesc(pMNEntry->tuple_desc);
if (!pMNEntry->preserve_order)
htfifo_destroy(pMNEntry->ready_tuples);
pMNEntry->valid = false;
}
/* An unordered receiver will call this with srcRoute == ANY_ROUTE */
ReceiveReturnCode
RecvTupleFrom(MotionLayerState *mlStates,
ChunkTransportState *transportStates,
int16 motNodeID,
HeapTuple *tup_i,
int16 srcRoute)
{
MotionNodeEntry *pMNEntry;
ChunkSorterEntry *pCSEntry;
ReceiveReturnCode recvRC = END_OF_STREAM;
htup_fifo ReadyList;
#ifdef AMS_VERBOSE_LOGGING
elog(DEBUG5, "RecvTupleFrom( motNodeID = %d, srcRoute = %d )", motNodeID, srcRoute);
#endif
pMNEntry = getMotionNodeEntry(mlStates, motNodeID, "RecvTupleFrom");
if (srcRoute == ANY_ROUTE)
{
Assert(pMNEntry->preserve_order == 0);
pCSEntry = NULL;
ReadyList = pMNEntry->ready_tuples;
}
else
{
Assert(pMNEntry->preserve_order != 0);
/*
* Pull up the chunk-sorter entry for the specified sender, and get the
* tuple-store we should use.
*/
pCSEntry = getChunkSorterEntry(mlStates, pMNEntry, srcRoute);
ReadyList = pCSEntry->ready_tuples;
}
/* Get the next HeapTuple, if one is available! */
*tup_i = htfifo_gettuple(ReadyList);
if (*tup_i != NULL)
{
recvRC = GOT_TUPLE;
statRecvTuple(pMNEntry, pCSEntry, recvRC);
return recvRC;
}
/* We need to get more chunks before we have a full tuple to return */
do
{
if (srcRoute == ANY_ROUTE)
{
if (!pMNEntry->moreNetWork)
{
recvRC = END_OF_STREAM;
statRecvTuple(pMNEntry, pCSEntry, recvRC);
return recvRC;
}
}
else
{
if (pCSEntry->end_of_stream)
{
recvRC = END_OF_STREAM;
statRecvTuple(pMNEntry, pCSEntry, recvRC);
return recvRC;
}
}
processIncomingChunks(mlStates, transportStates, pMNEntry, motNodeID, srcRoute);
if (srcRoute == ANY_ROUTE)
*tup_i = htfifo_gettuple(pMNEntry->ready_tuples);
else
*tup_i = htfifo_gettuple(pCSEntry->ready_tuples);
if (*tup_i != NULL)
{
/* We got a tuple. */
recvRC = GOT_TUPLE;
}
else if ((srcRoute == ANY_ROUTE && !pMNEntry->moreNetWork) ||
(srcRoute != ANY_ROUTE && pCSEntry->end_of_stream))
{
/*
* No tuple was available (tuple-store was at EOF), and end-of-stream
* has been marked. No more tuples are going to show up.
*/
recvRC = END_OF_STREAM;
}
else
{
/*
* No tuple was available (tuple-store was at EOF), but we
* have not been sent an end-of-stream. We need to loop so
* that we process more chunks to assemble a complete
* tuple.
*/
recvRC = NO_TUPLE;
}
}
while (recvRC == NO_TUPLE);
/* Stats */
statRecvTuple(pMNEntry, pCSEntry, recvRC);
return recvRC;
}
/*
* Function: SendTuple - Sends a portion or whole tuple to the AMS layer.
*/
SendReturnCode
SendTuple(MotionLayerState *mlStates,
ChunkTransportState *transportStates,
int16 motNodeID,
HeapTuple tuple,
int16 targetRoute)
{
MotionNodeEntry *pMNEntry;
TupleChunkListData tcList;
MemoryContext oldCtxt;
SendReturnCode rc;
AssertArg(tuple != NULL);
/*
* Analyze tools. Do not send any thing if this slice is in the bit mask
*/
if (gp_motion_slice_noop != 0 && (gp_motion_slice_noop & (1 << currentSliceId)) != 0)
return SEND_COMPLETE;
/*
* Pull up the motion node entry with the node's details. This includes
* details that affect sending, such as whether the motion node needs to
* include backup segment-dbs.
*/
pMNEntry = getMotionNodeEntry(mlStates, motNodeID, "SendTuple");
#ifdef AMS_VERBOSE_LOGGING
elog(DEBUG5, "Serializing HeapTuple for sending.");
#endif
if (targetRoute != BROADCAST_SEGIDX)
{
struct directTransportBuffer b;
getTransportDirectBuffer(transportStates, motNodeID, targetRoute, &b);
if (b.pri != NULL && b.prilen > TUPLE_CHUNK_HEADER_SIZE)
{
int sent = 0;
sent = SerializeTupleDirect(tuple, &pMNEntry->ser_tup_info, &b);
if (sent > 0)
{
putTransportDirectBuffer(transportStates, motNodeID, targetRoute, sent);
/* fill-in tcList fields to update stats */
tcList.num_chunks = 1;
tcList.serialized_data_length = sent;
/* update stats */
statSendTuple(mlStates, pMNEntry, &tcList);
return SEND_COMPLETE;
}
}
/* Otherwise fall-through */
}
/* Create and store the serialized form, and some stats about it. */
oldCtxt = MemoryContextSwitchTo(mlStates->motion_layer_mctx);
SerializeTupleIntoChunks(tuple, &pMNEntry->ser_tup_info, &tcList);
MemoryContextSwitchTo(oldCtxt);
#ifdef AMS_VERBOSE_LOGGING
elog(DEBUG5, "Serialized HeapTuple for sending:\n"
"\ttarget-route %d \n"
"\t%d bytes in serial form\n"
"\tbroken into %d chunks",
targetRoute,
tcList.serialized_data_length,
tcList.num_chunks);
#endif
/* do the send. */
if (!SendTupleChunkToAMS(mlStates, transportStates, motNodeID, targetRoute, tcList.p_first))
{
pMNEntry->stopped = true;
rc = STOP_SENDING;
}
else
{
/* update stats */
statSendTuple(mlStates, pMNEntry, &tcList);
rc = SEND_COMPLETE;
}
/* cleanup */
clearTCList(&pMNEntry->ser_tup_info.chunkCache, &tcList);
return rc;
}
void
CheckAndSendRecordCache(MotionLayerState *mlStates,
ChunkTransportState *transportStates,
int16 motNodeID,
int16 targetRoute)
{
MotionNodeEntry *pMNEntry;
TupleChunkListData tcList;
MemoryContext oldCtxt;
ChunkTransportStateEntry *pEntry = NULL;
MotionConn *conn;
getChunkTransportState(transportStates, motNodeID, &pEntry);
/*
* for broadcast we only mark sent_record_typmod for connection 0 for
* efficiency and convenience
*/
if (targetRoute == BROADCAST_SEGIDX)
conn = &pEntry->conns[0];
else
conn = &pEntry->conns[targetRoute];
/*
* Analyze tools. Do not send any thing if this slice is in the bit mask
*/
if (gp_motion_slice_noop != 0 && (gp_motion_slice_noop & (1 << currentSliceId)) != 0)
return;
/*
* Pull up the motion node entry with the node's details. This includes
* details that affect sending, such as whether the motion node needs to
* include backup segment-dbs.
*/
pMNEntry = getMotionNodeEntry(mlStates, motNodeID, "SendRecordCache");
if (!ShouldSendRecordCache(conn, &pMNEntry->ser_tup_info))
return;
#ifdef AMS_VERBOSE_LOGGING
elog(DEBUG5, "Serializing RecordCache for sending.");
#endif
/* Create and store the serialized form, and some stats about it. */
oldCtxt = MemoryContextSwitchTo(mlStates->motion_layer_mctx);
SerializeRecordCacheIntoChunks(&pMNEntry->ser_tup_info, &tcList, conn);
MemoryContextSwitchTo(oldCtxt);
#ifdef AMS_VERBOSE_LOGGING
elog(DEBUG5, "Serialized RecordCache for sending:\n"
"\ttarget-route %d \n"
"\t%d bytes in serial form\n"
"\tbroken into %d chunks",
targetRoute,
tcList.serialized_data_length,
tcList.num_chunks);
#endif
/* do the send. */
if (!SendTupleChunkToAMS(mlStates, transportStates, motNodeID, targetRoute, tcList.p_first))
{
pMNEntry->stopped = true;
}
else
{
/* update stats */
statSendTuple(mlStates, pMNEntry, &tcList);
}
/* cleanup */
clearTCList(&pMNEntry->ser_tup_info.chunkCache, &tcList);
UpdateSentRecordCache(conn);
}