NAMESPACE_UPP
#define LTIMING(x) // RTIMING(x)
VectorMap<EscValue, EscValue>& EscValue::CloneMap()
{
LTIMING("CloneMap");
ASSERT(IsMap());
if(AtomicRead(map->refcount) != 1) {
EscMap *c = new EscMap;
c->map <<= map->map;
map->Release();
map = c;
}
hash = 0;
return map->map;
}
void WorkerThread::Idle()
{
// decrease the active worker count
AtomicDecrement(&Pool_->ActiveWorkers_);
// were we the last active worker?
if(AtomicRead(&Pool_->ActiveWorkers_) == 1)
// notify the pool that everyone is now idle
ConditionWakeAll(Pool_->AllWorkersIdle_);
// wait until we get woken up by the pool, immediately unlock our locked local mutex
MutexLock(TaskMutex_);
ConditionWait(Pool_->WakeupNotifiction_, TaskMutex_);
MutexUnlock(TaskMutex_);
// increase the number of active workers
AtomicIncrement(&Pool_->ActiveWorkers_);
}
void TimerTest::Close0()
{
while(AtomicRead(demo) > 0) Sleep(1);
ASSERT(AtomicRead(demo)==0);
TopWindow::Close();
}
// called in a thread context, so can preempt
void
GsaCompletionCallback(
IN void *CaContext,
IN IB_HANDLE CqHandle
)
{
FSTATUS status;
IBT_DGRM_ELEMENT *pDgrm = NULL;
boolean bRearmed = FALSE;
IBT_DGRM_ELEMENT_PRIV *pDgrmPrivate;
GSA_POST_SEND_LIST *pPostList;
MAD_COMMON *pMadHdr;
GSA_SERVICE_CLASS_INFO *classInfo;
IB_WORK_COMPLETION workCompletion;
uint8 clientId;
uint64 portGuid;
_DBG_ENTER_LVL(_DBG_LVL_CALLBACK, GsaCompletionCallback);
//
// Processing completions on the CQ is done as follows:
// 1. Repeatedly Poll till all entries are done.
// 2. Rearm CQ and Poll again. If there are no more completions bail out.
// 3. If there are completions go back to step 1.
// 4. Skip step 2 and loop through 3.
//
// We follow these steps to optimize on hardware events and race
// conditions.
//
while (1)
{
#ifdef SA_ALLOC_DEBUG
if (iba_gsi_dgrm_pool_count(sa_poolhandle) < 50) MsgOut("GSA Callback\n");
#endif
//
// Loop and pull out all sends and receives
//
while ( FSUCCESS == (status = iba_poll_cq( CqHandle, &workCompletion )) )
{
// fill in common values
pDgrm = (IBT_DGRM_ELEMENT *)((uintn)MASK_WORKREQID(workCompletion.WorkReqId));
// This should never happen. test for bad hardware
ASSERT ( pDgrm );
if ( NULL == pDgrm )
{
//
// This is for debug only. We should remove this conditional
// once the problem is debugged
//
_DBG_ERROR (("Spurious CQ Event!!!\n"
"\tCompleted WQE.....:(%s)\n"
"\tSTATUS............:x%x\n"
"\tLength............:x%x\n",
_DBG_PTR(((WCTypeSend == workCompletion.WcType) ? \
"Send":"Recv")),
workCompletion.Status,
workCompletion.Length));
if ( WCTypeRecv == workCompletion.WcType )
{
_DBG_ERROR (("\n"
"\tQP Number.........:x%x\n"
"\tSourceLID.........:x%x\n",
workCompletion.Req.RecvUD.SrcQPNumber,
workCompletion.Req.RecvUD.SrcLID));
}
_DBG_BREAK;
break;
}
// process work completion
pDgrm->Element.Status = workCompletion.Status;
pDgrmPrivate = (IBT_DGRM_ELEMENT_PRIV *)pDgrm;
_DBG_PRINT (_DBG_LVL_CALLBACK,(
"Completed %s WQE (%p):\n"
"\tSTATUS.......:x%X (%s)\n",
_DBG_PTR(((WCTypeSend == workCompletion.WcType) ? "Send":"Recv")),
_DBG_PTR(pDgrm),
workCompletion.Status,
_DBG_PTR(iba_wc_status_msg(workCompletion.Status)) ));
if (IS_SQ_WORKREQID(workCompletion.WorkReqId))
{
#ifdef SA_ALLOC_DEBUG
if (iba_gsi_dgrm_pool_count(sa_poolhandle) < 50) MsgOut("GSA Send Compl\n");
#endif
if (! pDgrm->Allocated) {
_DBG_ERROR(("GSI ERROR: Send Completion for non-allocated Dgrm, discard: pDgrm=%p completion type=%d OnrecvQ=%d\n", _DBG_PTR(pDgrm), workCompletion.WcType, pDgrm->OnRecvQ));
if (Gsa_debug_params.debug_level & _DBG_LVL_STORE)
{
ASSERT(pDgrm->Allocated);
}
DEBUG_ASSERT(pDgrm->Allocated);
continue;
}
if (pDgrm->OnRecvQ) {
_DBG_ERROR(("GSI ERROR: Send Completion for Recv Q Dgrm, discard: pDgrm=%p completion type=%d\n", _DBG_PTR(pDgrm), workCompletion.WcType));
if (Gsa_debug_params.debug_level & _DBG_LVL_STORE)
{
ASSERT(! pDgrm->OnRecvQ);
}
DEBUG_ASSERT(! pDgrm->OnRecvQ);
continue;
}
if (! pDgrm->OnSendQ) {
_DBG_ERROR(("GSI ERROR: Send Completion for non-OnSendQ Dgrm, discard: pDgrm=%p completion type=%d allocated=%d\n", _DBG_PTR(pDgrm), workCompletion.WcType, pDgrm->Allocated));
if (Gsa_debug_params.debug_level & _DBG_LVL_STORE)
{
ASSERT(pDgrm->OnSendQ);
}
DEBUG_ASSERT(pDgrm->OnSendQ);
continue;
}
pDgrm->OnSendQ = FALSE;
// Handle common send completion. Only for SendInvalid
// will a Recv Pool Dgrm be used, we get correct pMadHdr
// mainly so log messages below are correct
if (((IBT_DGRM_ELEMENT_PRIV *)pDgrm)->MemPoolHandle->ReceivePool )
pMadHdr = (MAD_COMMON *)GsiDgrmGetRecvMad(pDgrm);
else
pMadHdr = (MAD_COMMON *)GsiDgrmGetSendMad(pDgrm);
BSWAP_MAD_HEADER ((MAD*)pMadHdr);
pPostList = pDgrmPrivate->Base;
AtomicIncrementVoid( &pPostList->DgrmOut);
#ifdef ICS_LOGGING
_DBG_PRINT (_DBG_LVL_CALLBACK,(
"Send Info:\n"
"\tClass........:x%x\n"
"\tResponseBit..:x%x\n"
"\tTID..........:x%"PRIx64"\n"
"Local\n"
"\tPathBits.....:x%X\n"
"Remote:\n"
"\tLID..........:x%X\n"
"\tQP...........:x%X\n",
(uint8)pMadHdr->MgmtClass,
pMadHdr->mr.s.R,
pMadHdr->TransactionID,
pDgrmPrivate->DgrmElement.PathBits,
pDgrmPrivate->DgrmElement.RemoteLID,
pDgrmPrivate->DgrmElement.RemoteQP
));
_DBG_PRINT (_DBG_LVL_CALLBACK,(
"Send Info:\n"
"\tClass........:x%x\n"
"\tResponseBit..:x%x\n"
"\tTID..........:x%"PRIx64"\n"
"Local\n"
"\tQKey.........:x%X\n"
"\tServiceLvl...:x%X\n"
"\tGlobalRoute..:%s\n",
(uint8)pMadHdr->MgmtClass,
pMadHdr->mr.s.R,
pMadHdr->TransactionID,
pDgrmPrivate->DgrmElement.RemoteQKey,
pDgrmPrivate->DgrmElement.ServiceLevel,
_DBG_PTR((pDgrmPrivate->DgrmElement.GlobalRoute ? "TRUE":"FALSE") )
));
#else
_DBG_PRINT (_DBG_LVL_CALLBACK,(
"Send Info:\n"
"\tClass........:x%x\n"
"\tResponseBit..:x%x\n"
"\tTID..........:x%"PRIx64"\n"
"Local\n"
"\tPathBits.....:x%X\n"
"Remote:\n"
"\tLID..........:x%X\n"
"\tQP...........:x%X\n"
"\tQKey.........:x%X\n"
"\tServiceLvl...:x%X\n"
"\tRate.........:x%d\n"
"\tGlobalRoute..:%s\n",
(uint8)pMadHdr->MgmtClass,
pMadHdr->mr.s.R,
pMadHdr->TransactionID,
pDgrmPrivate->DgrmElement.PathBits,
pDgrmPrivate->DgrmElement.RemoteLID,
pDgrmPrivate->DgrmElement.RemoteQP,
pDgrmPrivate->DgrmElement.RemoteQKey,
pDgrmPrivate->DgrmElement.ServiceLevel,
pDgrmPrivate->DgrmElement.StaticRate,
(pDgrmPrivate->DgrmElement.GlobalRoute ? "TRUE":"FALSE")
));
#endif
// Send completion for Send initiated by RMPP protocol
if( pPostList->SARRequest)
{
#ifdef SA_ALLOC_DEBUG
if (iba_gsi_dgrm_pool_count(sa_poolhandle) < 50) MsgOut("GSA Send Compl: SAR\n");
#endif
// send was posted by RMPP protocol handler
// let it process the completion
ProcessSARSendCompletion((MAD*)pMadHdr, pDgrm);
continue;
}
// The datagram has been successfully sent. Reset the
// transaction ID to that specified by the user if this is
// not a response. Responses use the transaction ID from
// the original request.
if (MAD_IS_REQUEST((MAD*)pMadHdr))
{
// Restore the original TID from the user. We cannot
// use the context manager here, since we may have already
// completed the request. (The associated receive may
// be processed before the send is completed.)
pMadHdr->TransactionID = pDgrmPrivate->SavedSendTid;
}
// Free resources
ASSERT ( pDgrmPrivate->AvHandle );
status = PutAV( NULL, pDgrmPrivate->AvHandle );
classInfo = (GSA_SERVICE_CLASS_INFO*)pDgrmPrivate->MemPoolHandle->ServiceHandle;
if ( NULL == classInfo )
{
// no associated class, must be a GSI initiated MAD
DEBUG_ASSERT(pPostList->DgrmIn == AtomicRead(&pPostList->DgrmOut));
status = DgrmPoolPut( pPostList->DgrmList );
continue;
}
// Match all sends before callback
if ( pPostList->DgrmIn != AtomicRead(&pPostList->DgrmOut) )
{
#ifdef SA_ALLOC_DEBUG
if (iba_gsi_dgrm_pool_count(sa_poolhandle) < 50) MsgOut("GSA Send Compl: more to do\n");
#endif
_DBG_PRINT (_DBG_LVL_CALLBACK,(
"All sends not completed in send list:\n"
"\tTotal.....:%d\n"
"\tDone......:%d\n",
pPostList->DgrmIn,
AtomicRead(&pPostList->DgrmOut) ));
continue;
}
// Call user's send completion
#ifdef SA_ALLOC_DEBUG
if (iba_gsi_dgrm_pool_count(sa_poolhandle) < 50) MsgOut("GSA Send Compl: class callback: %p\n", classInfo->SendCompletionCallback);
#endif
if ( classInfo->SendCompletionCallback )
{
(classInfo->SendCompletionCallback)(
classInfo->ServiceClassContext,
pPostList->DgrmList );
} else {
status = DgrmPoolPut( pPostList->DgrmList );
} // Class Callback
} else {
// Receive Completion processing
uint32 post_count;
IBT_BUFFER *Current = NULL;
uint32 Length = 0;
#ifdef SA_ALLOC_DEBUG
if (iba_gsi_dgrm_pool_count(sa_poolhandle) < 50) MsgOut("GSA Recv Compl\n");
#endif
if (pDgrm->OnSendQ) {
_DBG_ERROR(("GSI ERROR: Recv Completion for Send Q Dgrm, discard: pDgrm=%p completion type=%d\n", _DBG_PTR(pDgrm), workCompletion.WcType));
if (Gsa_debug_params.debug_level & _DBG_LVL_STORE)
{
ASSERT(! pDgrm->OnSendQ);
}
DEBUG_ASSERT(! pDgrm->OnSendQ);
continue;
}
if (! pDgrm->OnRecvQ) {
_DBG_ERROR(("GSI ERROR: Recv Completion for non-OnRecvQ Dgrm, discard: pDgrm=%p completion type=%d allocated=%d\n", _DBG_PTR(pDgrm), workCompletion.WcType, pDgrm->Allocated));
if (Gsa_debug_params.debug_level & _DBG_LVL_STORE)
{
ASSERT(pDgrm->OnRecvQ);
}
DEBUG_ASSERT(pDgrm->OnRecvQ);
continue;
}
if (! pDgrm->Allocated) {
_DBG_ERROR(("GSI ERROR: Completion for non-allocated Dgrm, discard: pDgrm=%p completion type=%d\n", _DBG_PTR(pDgrm), workCompletion.WcType));
if (Gsa_debug_params.debug_level & _DBG_LVL_STORE)
{
ASSERT(pDgrm->Allocated);
}
DEBUG_ASSERT(pDgrm->Allocated);
pDgrm->OnRecvQ = FALSE;
continue;
}
pDgrm->OnRecvQ = FALSE;
status = DecrementGsiRecvQPostedForPortGuid(
pDgrm->PortGuid, &post_count);
_DBG_PRINT (_DBG_LVL_CALLBACK,("0x%"PRIx64": got post count of %u: %d\n",pDgrm->PortGuid, post_count, status));
if (FSUCCESS == status
&& post_count < g_GsaSettings.MinBuffersToPostPerPort
)
{
// replenish to desired min
post_count = g_GsaSettings.MinBuffersToPostPerPort - post_count;
} else {
// can't be sure, just repost 1
post_count = 1;
}
_DBG_PRINT (_DBG_LVL_CALLBACK,("computed post count of %u\n", post_count));
// Do not callback for QpFlush
if ( WRStatusWRFlushed == workCompletion.Status )
{
#ifdef SA_ALLOC_DEBUG
if (iba_gsi_dgrm_pool_count(sa_poolhandle) < 50) MsgOut("GSA Recv Compl: Flush\n");
#endif
_DBG_PRINT (_DBG_LVL_CALLBACK,("WRFlushed: skip users.\n"));
status = DgrmPoolPut( pDgrm );
continue;
}
/* MADs can be of variable length */
if ( workCompletion.Length > (sizeof(IB_GRH) + sizeof(MAD)))
{
_DBG_ERROR ((
"Invalid MAD receive length (%d bytes)! <Discard>\n",
workCompletion.Length ));
status = DgrmPoolPut( pDgrm );
continue;
}
// Overload base with SAR field
pDgrmPrivate->Base = 0; // not a SAR Receive Dgrm
// Fill in bytes rcvd
pDgrm->TotalRecvSize = workCompletion.Length;
pDgrm->Element.RecvByteCount = workCompletion.Length;
// Save GUID for a post since the user may FPENDING the Dgrm
portGuid = pDgrm->PortGuid;
// GRH is first buffer, MAD is second
pDgrm->Element.pBufferList->ByteCount = sizeof(IB_GRH);
Current = pDgrm->Element.pBufferList->pNextBuffer;
Length = workCompletion.Length - sizeof(IB_GRH);
pMadHdr = (MAD_COMMON *)GsiDgrmGetRecvMad(pDgrm);
while (Length && Current)
{
Current->ByteCount = (pMadHdr->BaseVersion == IB_BASE_VERSION)
? MIN(Length, IB_MAD_BLOCK_SIZE)
: MIN(Length, STL_MAD_BLOCK_SIZE);
if (__DBG_LEVEL__ & _DBG_LVL_PKTDUMP)
{
_DBG_DUMP_MEMORY(_DBG_LVL_PKTDUMP, ("Recv GSI MAD Packet"),
Current->pData,
MIN(Current->ByteCount, Length));
}
Length -= Current->ByteCount;
Current = Current->pNextBuffer;
}
BSWAP_MAD_HEADER ((MAD*)pMadHdr);
#ifdef ICS_LOGGING
_DBG_PRINT (_DBG_LVL_CALLBACK,(
"Receive Info:\n"
"\tClass........:x%x\n"
"\tResponseBit..:x%x\n"
"\tTID..........:x%"PRIx64"\n"
"Local\n"
"\tPathBits.....:x%X\n"
"Remote:\n"
"\tLID..........:x%X\n"
"\tQP...........:x%X\n",
(uint8)pMadHdr->MgmtClass,
pMadHdr->mr.s.R,
pMadHdr->TransactionID,
workCompletion.Req.RecvUD.DestPathBits,
workCompletion.Req.RecvUD.SrcLID,
workCompletion.Req.RecvUD.SrcQPNumber
));
_DBG_PRINT (_DBG_LVL_CALLBACK,(
"Receive Info:\n"
"\tClass........:x%x\n"
"\tResponseBit..:x%x\n"
"\tTID..........:x%"PRIx64"\n"
"Remote:\n"
"\tPKeyIndex....:x%X\n"
"\tServiceLvl...:x%X\n"
"\tGlobalRoute..:%s\n",
(uint8)pMadHdr->MgmtClass,
pMadHdr->mr.s.R,
pMadHdr->TransactionID,
workCompletion.Req.RecvUD.PkeyIndex,
workCompletion.Req.RecvUD.ServiceLevel,
_DBG_PTR((workCompletion.Req.RecvUD.Flags.s.GlobalRoute ? "TRUE":"FALSE"))
));
#else
_DBG_PRINT (_DBG_LVL_CALLBACK,(
"Receive Info:\n"
"\tClass........:x%x\n"
"\tResponseBit..:x%x\n"
"\tTID..........:x%"PRIx64"\n"
"Local\n"
"\tPathBits.....:x%X\n"
"Remote:\n"
"\tLID..........:x%X\n"
"\tQP...........:x%X\n"
"\tPKeyIndex....:x%X\n"
"\tServiceLvl...:x%X\n"
"\tGlobalRoute..:%s\n",
(uint8)pMadHdr->MgmtClass,
pMadHdr->mr.s.R,
pMadHdr->TransactionID,
workCompletion.Req.RecvUD.DestPathBits,
workCompletion.Req.RecvUD.SrcLID,
workCompletion.Req.RecvUD.SrcQPNumber,
workCompletion.Req.RecvUD.PkeyIndex,
workCompletion.Req.RecvUD.ServiceLevel,
(workCompletion.Req.RecvUD.Flags.s.GlobalRoute ? "TRUE":"FALSE")
));
#endif
// Port Number and Q pair are filled-up during
// Gsa's PostRecv
pDgrm->PathBits = workCompletion.Req.RecvUD.DestPathBits;
pDgrm->RemoteLID = workCompletion.Req.RecvUD.SrcLID;
pDgrm->RemoteQP = workCompletion.Req.RecvUD.SrcQPNumber;
pDgrm->RemoteQKey = QP1_WELL_KNOWN_Q_KEY;
pDgrm->PkeyIndex = workCompletion.Req.RecvUD.PkeyIndex;
pDgrm->ServiceLevel = workCompletion.Req.RecvUD.ServiceLevel;
// IB does not have Static rate in UD LRH
// so we can't be sure what rate of remote port is
// we use a constant value for GSI QPs
pDgrm->StaticRate = IB_STATIC_RATE_GSI;
pDgrm->GlobalRoute = workCompletion.Req.RecvUD.Flags.s.GlobalRoute;
if (workCompletion.Req.RecvUD.Flags.s.GlobalRoute)
{
IB_GRH *pGrh = GsiDgrmGetRecvGrh(pDgrm);
BSWAP_IB_GRH ( pGrh);
pDgrm->GRHInfo.DestGID = pGrh->SGID;
pDgrm->GRHInfo.FlowLabel = pGrh->u1.s.FlowLabel;
pDgrm->GRHInfo.SrcGIDIndex = 0; // BUGBUG - lookup DGID
pDgrm->GRHInfo.HopLimit = pGrh->HopLimit;
pDgrm->GRHInfo.TrafficClass = pGrh->u1.s.TClass;
_DBG_PRINT (_DBG_LVL_CALLBACK,(
"GRH Info:\n"
"\tDGID.........:x%"PRIx64":%"PRIx64"\n"
"\tSGID.........:x%"PRIx64":%"PRIx64"\n"
"\tPayload......:%d bytes\n",
pGrh->DGID.Type.Global.SubnetPrefix,
pGrh->DGID.Type.Global.InterfaceID,
pGrh->SGID.Type.Global.SubnetPrefix,
pGrh->SGID.Type.Global.InterfaceID,
pGrh->PayLen ));
}
//
//TBD:
// Chain replies to process later
//
//pDgrm->pNextCompDgrm
// For RMPP R bit be set in all packets which compose a response
// message Even ABORT, STOP, etc.
// transaction ID will be consistent in all packets for
// a given request/respone message pair.
// R bit be 0 in all packets which compose a request message
// event ACK, ABORT, STOP, etc
// See if the received MAD was in response to a previously
// sent MAD request.
if (MAD_IS_RESPONSE((MAD*)pMadHdr))
{
// Response to clients. client id was a field
// in the transaction ID sent on wire
clientId = (uint8)(pMadHdr->TransactionID & 0xff);
} else {
// This is a new request or not a request/response pair
// TID unmodified TID is visible to client (for a request
// TID was remotely generated and we can't modify)
clientId = MAX_CLIENT_ID;
status = FSUCCESS;
}
// Validate Attributes to be processed
if( status == FSUCCESS )
{
// Does the Client want us to do RMPP protocol for it?
if( IsSarClient( clientId, pMadHdr->MgmtClass ) )
{
// Process the segment. If all segments have been
// received, pDgrm will be updated to point to the
// first segment.
status = ProcessSARRecvCompletion((MAD*)pMadHdr,&pDgrm);
if( status == FCOMPLETED )
{
// remove clientid from all TIDs before give to user
IBT_ELEMENT *pElement;
for (pElement = &pDgrm->Element; pElement != NULL; pElement = pElement->pNextElement)
{
// first buffer is GRH, 2nd is start of MAD
RemoveClientIdFromTID(
(MAD_COMMON *)
pElement->pBufferList->pNextBuffer->pData);
}
// adjust pMadHdr to be first in list of Dgrms,
// pDgrm could have been changed by SAR processing
pMadHdr = (MAD_COMMON *)GsiDgrmGetRecvMad(pDgrm);
}
} else {
// No SAR is needed.
// remove clientid from TID before give to user
RemoveClientIdFromTID(pMadHdr);
status = FCOMPLETED;
}
// Notify the user of fully received MADs.
if( status == FCOMPLETED )
{
status = GsaNotifyClients( clientId, pMadHdr, pDgrm );
}
if (status == FNOT_FOUND)
{
// no client found
if (MAD_EXPECT_GET_RESP((MAD*)pMadHdr))
{
// reply with a bad class version response
GsiSendInvalid(pDgrm);
}
status = FREJECT; // free dgrm below
}
// Free the datagram(s) if not required for SAR and not
// being used by the client.
if( status == FCOMPLETED || status == FREJECT )
{
DgrmPoolPut( pDgrm );
}
}
// Post back a recv to Q
status = PostRecvMsgToPort( post_count, portGuid );
} // recv
} // while loop
if ( FNOT_DONE != status )
{
_DBG_ERROR (("Got %s Status!!!\n", _DBG_PTR(FSTATUS_MSG(status)) ));
}
// Rearm CQ for race conditions
if ( FALSE == bRearmed )
{
status = iba_rearm_cq( CqHandle, CQEventSelNextWC );
bRearmed = TRUE;
} else {
break;
}
} // while 1
_DBG_LEAVE_LVL(_DBG_LVL_CALLBACK);
}