inline iWARPEM_Status_t FlushSendBuffer()
{
int wlen = 0;
mNeedsBufferFlush = false;
pthread_spin_lock( &mAccessLock );
size_t DataLen = mSendBuffer->GetDataLen();
if( DataLen == 0)
{
BegLogLine( FXLOG_IT_API_O_SOCKETS_MULTIPLEX_LOG )
<< "Buffer is empty. Nothing to send."
<< EndLogLine;
pthread_spin_unlock( &mAccessLock );
return IWARPEM_SUCCESS;
}
iWARPEM_Status_t status = mSendBuffer->FlushToSocket( mRouterConnFd, &wlen );
BegLogLine( FXLOG_IT_API_O_SOCKETS_MULTIPLEX_LOG )
<< "Sent " << (int)(wlen)
<< " payload: " << mSendBuffer->GetDataLen()
<< " socket: " << mRouterConnFd
<< EndLogLine;
#ifdef MULTIPLEX_STATISTICS
mFlushCount++;
mMsgAvg = (mMsgAvg * 0.9997) + (mMsgCount * 0.0003);
BegLogLine( ( (mFlushCount & 0xfff) == 0 ) )
<< "average message count per send buffer:" << mMsgAvg
<< EndLogLine;
#endif
ResetSendBuffer();
pthread_spin_unlock( &mAccessLock );
return status;
}
iWARPEM_Status_t RemoveClient( iWARPEM_StreamId_t aClientId )
{
if( aClientId >= mMaxClientCount )
{
BegLogLine( FXLOG_IT_API_O_SOCKETS_MULTIPLEX_LOG )
<< "Requested client ID out of range [ 0 < " << aClientId << " < " << mMaxClientCount << " ]."
<< EndLogLine;
return IWARPEM_ERRNO_CONNECTION_RESET;
}
if( mClientCount < 1 )
{
BegLogLine( FXLOG_IT_API_O_SOCKETS_MULTIPLEX_LOG )
<< "No clients in ClientEP list."
<< EndLogLine;
return IWARPEM_ERRNO_CONNECTION_RESET;
}
if( mClientEPs[ aClientId ] == NULL )
{
BegLogLine( FXLOG_IT_API_O_SOCKETS_MULTIPLEX_LOG )
<< "Client ID: " << aClientId
<< " is not a virtual connected ID for this RouterEP" << mRouterConnFd
<< EndLogLine;
return IWARPEM_ERRNO_CONNECTION_RESET;
}
mClientEPs[ aClientId ] = NULL;
mClientCount--;
BegLogLine( FXLOG_IT_API_O_SOCKETS_MULTIPLEX_LOG )
<< "Removed client: " << aClientId
<< EndLogLine;
return IWARPEM_SUCCESS;
}
iWARPEM_Status_t ExtractNextMessage( iWARPEM_Message_Hdr_t **aHdr, char **aData, iWARPEM_StreamId_t *aClientId )
{
iWARPEM_Status_t status = IWARPEM_SUCCESS;
pthread_spin_lock( &mAccessLock );
if( ! RecvDataAvailable() )
{
BegLogLine( FXLOG_IT_API_O_SOCKETS_MULTIPLEX_LOG )
<< "No data available in ReadBuffer. Reading new set of data."
<< EndLogLine;
switch( mReceiveBuffer->FillFromSocket( mRouterConnFd ) )
{
case IWARPEM_SUCCESS:
break;
default:
status = IWARPEM_ERRNO_CONNECTION_RESET;
BegLogLine( FXLOG_IT_API_O_SOCKETS_MULTIPLEX_LOG )
<< "Error while reading data from router endpoint."
<< EndLogLine;
pthread_spin_unlock( &mAccessLock );
return status;
}
}
pthread_spin_unlock( &mAccessLock );
size_t rlen = sizeof( iWARPEM_Multiplexed_Msg_Hdr_t );
iWARPEM_Multiplexed_Msg_Hdr_t *MultHdr = (iWARPEM_Multiplexed_Msg_Hdr_t*)mReceiveBuffer->GetHdrPtr( &rlen );
AssertLogLine( rlen == sizeof( iWARPEM_Multiplexed_Msg_Hdr_t ) )
<< "Retrieval of HdrPtr failed: len=" << rlen
<< " expected=" << sizeof(iWARPEM_Multiplexed_Msg_Hdr_t)
<< EndLogLine;
MultHdr->ClientID = ntohs( MultHdr->ClientID );
*aClientId = MultHdr->ClientID;
rlen = sizeof( iWARPEM_Message_Hdr_t );
*aHdr = (iWARPEM_Message_Hdr_t*)mReceiveBuffer->GetHdrPtr( &rlen );
AssertLogLine( rlen == sizeof( iWARPEM_Message_Hdr_t ) )
<< "Retrieval of HdrPtr failed: len=" << rlen
<< " expected=" << sizeof(iWARPEM_Multiplexed_Msg_Hdr_t)
<< EndLogLine;
(*aHdr)->EndianConvert();
rlen = (*aHdr)->mTotalDataLen;
if( rlen )
rlen = mReceiveBuffer->GetData( aData, &rlen );
AssertLogLine( rlen == (*aHdr)->mTotalDataLen )
<< "Retrieval of HdrPtr failed: len=" << rlen
<< " expected=" << sizeof(iWARPEM_Multiplexed_Msg_Hdr_t)
<< EndLogLine;
BegLogLine( FXLOG_IT_API_O_SOCKETS_MULTIPLEX_LOG )
<< "Extracted new client message: "
<< " socket: " << mRouterConnFd
<< " client: " << MultHdr->ClientID
<< " MsgHdrSize: " << sizeof( iWARPEM_Multiplexed_Msg_Hdr_t )
<< " MsgPldSize: " << (*aHdr)->mTotalDataLen
<< " Processed: " << mReceiveBuffer->GetDataLen()
<< EndLogLine;
return status;
}
~iWARPEM_Multiplexed_Endpoint_t()
{
bzero( &mRouterInfo, sizeof( iWARPEM_Router_Info_t ));
// go over all active EPs and flush/destroy them
for( int i=0; i < mMaxClientCount; i++ )
if( IsValidClient( i ) )
RemoveClient( i );
delete mClientEPs;
delete mSendBuffer;
delete mReceiveBuffer;
pthread_spin_destroy( &mAccessLock );
BegLogLine( FXLOG_IT_API_O_SOCKETS_MULTIPLEX_LOG )
<< "Destroyed multiplexed router endpoint."
<< " socket: " << mRouterConnFd
<< EndLogLine;
#ifdef MULTIPLEX_STATISTICS
BegLogLine( 1 )
<< "average message count per send buffer:" << mMsgAvg
<< EndLogLine;
#endif
}
static
int
skv_bulk_insert_pack( char *aRow, char *Key, int KeyLength, char *Value, int ValueLength )
{
int TotalSize = 0;
int* KeyPtr = (int *) &aRow[TotalSize];
BegLogLine(SKV_CLIENT_ENDIAN_LOG)
<< "Endian-converting KeyLength from " << (void *) (intptr_t)KeyLength
<< EndLogLine ;
*KeyPtr = htonl(KeyLength);
TotalSize += sizeof(int);
memcpy( &aRow[TotalSize],
Key,
KeyLength );
TotalSize += KeyLength;
int* ValuePtr = (int *) &aRow[TotalSize];
BegLogLine(SKV_CLIENT_ENDIAN_LOG)
<< "Endian-converting ValueLength from " << (void *) (intptr_t)ValueLength
<< EndLogLine ;
*ValuePtr = htonl(ValueLength);
TotalSize += sizeof(int);
memcpy( &aRow[TotalSize],
Value,
ValueLength );
TotalSize += ValueLength;
return TotalSize;
}
// ??? We should allow the user to set a compare function on skv_store_t
// ??? Default behaviour should be lexicographical order
bool
operator<( const skv_store_t& aStore ) const
{
BegLogLine( SKV_STORE_T_LOG )
<< "skv_store_t::operator<():: Entering "
<< EndLogLine;
int MinDataSize = min( mSize(), aStore.mSize() );
AssertLogLine( mData != NULL )
<< "skv_store_t::operator<():: ERROR: "
<< " mData != NULL "
<< EndLogLine;
AssertLogLine( aStore.mData != NULL )
<< "skv_store_t::operator<():: ERROR: "
<< " aStore.mData != NULL "
<< EndLogLine;
int rc = memcmp( mData, aStore.mData, MinDataSize );
BegLogLine( SKV_STORE_T_LOG )
<< "skv_store_t::operator<():: "
<< " MinDataSize: " << MinDataSize
<< " mSize: " << mSize()
<< " aStore.mSize: " << aStore.mSize()
<< " rc: " << rc
<< EndLogLine;
return ( rc < 0 );
}
/***
* skv_server_t::InitNewStateForEP::
* Desc: Initiates the state for a new EP
* input:
* returns: SKV_SUCCESS or SKV_ERR_NO_EVENT
***/
skv_status_t
skv_server_network_event_manager_if_t::
FinalizeEPState( skv_server_epstate_map_t *aEPStateMap,
it_ep_handle_t aEP,
skv_server_ep_state_t* aStateForEP )
{
AssertLogLine( aStateForEP != NULL )
<< "skv_server_t::FinalizeEPState(): ERROR: "
<< " aEP: " << (void *) aEP
<< EndLogLine;
aStateForEP->Closing();
skv_server_finalizable_associated_ep_state_list_t::iterator iter = aStateForEP->mAssociatedStateList->begin();
skv_server_finalizable_associated_ep_state_list_t::iterator end = aStateForEP->mAssociatedStateList->end();
for( ; iter != end; iter++ )
{
switch( iter->mStateType )
{
case SKV_SERVER_FINALIZABLE_ASSOCIATED_EP_STATE_CREATE_CURSOR_TYPE:
{
skv_server_cursor_hdl_t ServCursorHdl = (skv_server_cursor_hdl_t) iter->mState;
ServCursorHdl->Finalize();
free( ServCursorHdl );
break;
}
default:
StrongAssertLogLine( 0 )
<< "FinalizeEPState(): ERROR:: "
<< " iter->mStateType: " << iter->mStateType
<< EndLogLine;
}
}
aEPStateMap->erase( aEP );
aStateForEP->Finalize();
BegLogLine(SKV_SERVER_CLEANUP_LOG)
<< "free(aStateForEP= " << (void *) aStateForEP
<< " )"
<< EndLogLine ;
bzero( aStateForEP, sizeof( skv_server_ep_state_t ) );
delete aStateForEP;
it_ep_free( aEP );
BegLogLine( SKV_SERVER_CLEANUP_LOG )
<< "skv_server::FinalizeEPState(): completed "
<< EndLogLine;
return SKV_SUCCESS;
}
static inline
skv_status_t insert_lookup_sequence( skv_local_kv_t *aLocalKV,
skv_server_ccb_t *aCommand,
skv_cmd_RIU_req_t **aReq,
skv_local_kv_cookie_t *aCookie,
skv_lmr_triplet_t *aValueRep )
{
// we have copied all Req data into response buffer already at cmd init
skv_cmd_RIU_req_t *Req = (skv_cmd_RIU_req_t *) aCommand->GetSendBuff();
// convert only if it's the first time we arrive here...
if( aCommand->GetCommandClass() == SKV_COMMAND_CLASS_IMMEDIATE )
Req->EndianConvert() ;
*aReq = Req;
if( FlagBasedLock( aLocalKV, Req, aCommand ) != SKV_SUCCESS )
{
BegLogLine( ( SKV_SERVER_LOCK_LOG | SKV_SERVER_INSERT_LOG ) )
<< "skv_server_insert_command_sm::Execute()::"
<< " record is locked."
<< EndLogLine;
return SKV_ERRNO_RECORD_IS_LOCKED;
}
// Check if the key is in the buffer
int KeySize = Req->mKeyValue.mKeySize;
BegLogLine( SKV_SERVER_INSERT_LOG )
<< "skv_server_insert_command_sm:: Lookup with flags: " << (void*)( (uintptr_t)Req->mFlags & (SKV_COMMAND_RIU_INSERT_EXPANDS_VALUE
| SKV_COMMAND_RIU_INSERT_OVERWRITE_VALUE_ON_DUP
| SKV_COMMAND_RIU_UPDATE
| SKV_COMMAND_RIU_APPEND) )
<< EndLogLine;
AssertLogLine( Req->mKeyValue.mKeySize >= 0 &&
Req->mKeyValue.mKeySize < SKV_KEY_LIMIT )
<< "skv_server_insert_command_sm:: Execute():: ERROR: "
<< "Req->mKeyValue.mKeySize: " << Req->mKeyValue.mKeySize
<< EndLogLine;
AssertLogLine( (Req->mFlags & SKV_COMMAND_RIU_INSERT_KEY_FITS_IN_CTL_MSG) ||
(Req->mFlags & SKV_COMMAND_RIU_INSERT_KEY_VALUE_FIT_IN_CTL_MSG) )
<< "skv_server_insert_command_sm:: Execute():: ERROR: "
<< " Assume that key fits into the control message"
<< EndLogLine;
// Check if the key exists
return aLocalKV->Lookup( Req->mPDSId,
Req->mKeyValue.mData,
KeySize,
Req->mFlags,
aValueRep,
aCookie );
}
static inline
skv_status_t insert_sequence( skv_local_kv_t *aLocalKV,
skv_server_ep_state_t *aEPState,
skv_server_ccb_t *aCommand,
int aCommandOrdinal,
skv_cmd_RIU_req_t *aReq,
skv_status_t lookup_status,
int *aSeqNo,
int aMyRank,
skv_lmr_triplet_t *aValueRepInStore )
{
skv_status_t status;
skv_lmr_triplet_t ValueRepForRdmaRead;
BegLogLine( SKV_SERVER_INSERT_LOG )
<< "skv_server_insert_command_sm::Execute():: Lookup returned: " << skv_status_to_string( lookup_status )
<< " will go for case: " << (void*)( (uintptr_t)aReq->mFlags & (SKV_COMMAND_RIU_INSERT_EXPANDS_VALUE
| SKV_COMMAND_RIU_INSERT_OVERWRITE_VALUE_ON_DUP
| SKV_COMMAND_RIU_UPDATE
| SKV_COMMAND_RIU_APPEND) )
<< EndLogLine;
#ifdef SKV_INSERT_DATA_LOG
if( aReq->mFlags & SKV_COMMAND_RIU_INSERT_KEY_VALUE_FIT_IN_CTL_MSG )
{
HexDump FxString( &aReq->mKeyValue.mData[ aReq->mKeyValue.mKeySize ], aReq->mKeyValue.mValueSize );
BegLogLine( 1 )
<< "skv_server_insert_command_sm::insert_sequence(): "
<< " Size: " << aReq->mKeyValue.mValueSize
<< " FxString: " << FxString
<< EndLogLine;
}
else
BegLogLine( 1 )
<< "skv_server_insert_command_sm::insert_sequence(): Data not in ctrl-msg. can't display."
<< EndLogLine;
#endif
skv_local_kv_cookie_t *cookie = &aCommand->mLocalKVCookie;
cookie->Set( aCommandOrdinal, aEPState );
status = aLocalKV->Insert( aReq,
lookup_status,
aValueRepInStore,
&ValueRepForRdmaRead,
cookie );
BegLogLine( SKV_SERVER_INSERT_LOG )
<< "skv_server_insert_command_sm::insert_sequence():: Insert returned: " << skv_status_to_string( status )
<< EndLogLine;
aValueRepInStore->Init( ValueRepForRdmaRead );
return status;
}
// needs to make sure to not change the status of the receive buffer
iWARPEM_Status_t GetNextMessageType( iWARPEM_Msg_Type_t *aMsgType, iWARPEM_StreamId_t *aClient )
{
iWARPEM_Status_t status = IWARPEM_SUCCESS;
pthread_spin_lock( &mAccessLock );
if( ! RecvDataAvailable() )
{
BegLogLine( FXLOG_IT_API_O_SOCKETS_MULTIPLEX_LOG )
<< "No data available in ReadBuffer. Reading new set of data."
<< EndLogLine;
status = mReceiveBuffer->FillFromSocket( mRouterConnFd );
switch( status )
{
case IWARPEM_SUCCESS:
break;
default:
BegLogLine( 1 )
<< "Error while reading data from router endpoint."
<< EndLogLine;
*aClient = IWARPEM_INVALID_CLIENT_ID;
*aMsgType = iWARPEM_UNKNOWN_REQ_TYPE;
pthread_spin_unlock( &mAccessLock );
return status;
}
}
pthread_spin_unlock( &mAccessLock );
size_t hdrlen = sizeof( iWARPEM_Multiplexed_Msg_Hdr_t );
iWARPEM_Multiplexed_Msg_Hdr_t *MultHdr = (iWARPEM_Multiplexed_Msg_Hdr_t*)mReceiveBuffer->GetHdrPtr( &hdrlen, 0 );
AssertLogLine( hdrlen == sizeof( iWARPEM_Multiplexed_Msg_Hdr_t ) )
<< "Retrieval of HdrPtr failed: len=" << hdrlen
<< " expected=" << sizeof(iWARPEM_Multiplexed_Msg_Hdr_t)
<< EndLogLine;
MultHdr->ClientID = be64toh( MultHdr->ClientID );
*aClient = MultHdr->ClientID;
// advance the hdr ptr ourselves to peek into MsgType without advancing the receive buffer ptr
char *HdrData = (char*)MultHdr;
HdrData += sizeof( iWARPEM_Multiplexed_Msg_Hdr_t );
HdrData = (char*)mReceiveBuffer->AlignedHeaderPosition( HdrData );
iWARPEM_Message_Hdr_t *MsgPtr = (iWARPEM_Message_Hdr_t*)HdrData;
BegLogLine( FXLOG_IT_API_O_SOCKETS_MULTIPLEX_LOG )
<< "Retrieving msg_type: " << MsgPtr->mMsg_Type
<< " from client: " << *aClient
<< " readptr@: " << (void*)(MsgPtr)
<< " len: " << MsgPtr->mTotalDataLen
<< EndLogLine;
*aMsgType = MsgPtr->mMsg_Type;
return status;
}
skv_status_t
skv_local_kv_asyncmem::PerformRetrieve( skv_local_kv_request_t *aReq )
{
size_t TotalSize;
skv_lmr_triplet_t StoredValueRep;
skv_local_kv_retrieve_request_t *Retrieve = &aReq->mRequest.mRetrieve;
skv_status_t status = mPDSManager.Retrieve( Retrieve->mPDSId,
Retrieve->mKeyData,
Retrieve->mKeySize,
Retrieve->mValueOffset,
Retrieve->mValueSize,
Retrieve->mFlags,
&StoredValueRep );
int ValueOversize = 0;
if (status == SKV_SUCCESS)
{
TotalSize = StoredValueRep.GetLen();
ValueOversize = StoredValueRep.GetLen() - Retrieve->mValueSize;
if( ValueOversize > 0 )
{
BegLogLine( SKV_LOCAL_KV_BACKEND_LOG )
<< "skv_local_kv_asyncmem:: ADJUSTING rdma_write length to match smaller client buffer"
<< " client: " << Retrieve->mValueSize
<< " store: " << StoredValueRep.GetLen()
<< EndLogLine;
StoredValueRep.SetLenIfSmaller( Retrieve->mValueSize );
}
int RoomForData = ((skv_header_as_cmd_buffer_t*)aReq)->GetRoomForData( sizeof( skv_cmd_retrieve_value_rdma_write_ack_t) );
if( RoomForData < StoredValueRep.GetLen() )
status = SKV_ERRNO_NEED_DATA_TRANSFER;
}
else
TotalSize = 0;
BegLogLine( SKV_LOCAL_KV_BACKEND_LOG )
<< "skv_local_kv_asyncmem:: storing valueRep:" << StoredValueRep
<< " status:" << skv_status_to_string( status )
<< EndLogLine;
status = InitKVRDMAEvent( aReq->mCookie,
&StoredValueRep,
TotalSize,
status );
return status;
}
skv_status_t
skv_local_kv_inmem::Retrieve( skv_pds_id_t aPDSId,
char* aKeyData,
int aKeySize,
int aValueOffset,
int aValueSize,
skv_cmd_RIU_flags_t aFlags,
skv_lmr_triplet_t* aStoredValueRep,
int *aTotalSize,
skv_local_kv_cookie_t *aCookie )
{
skv_status_t status = mPDSManager.Retrieve( aPDSId,
aKeyData,
aKeySize,
aValueOffset,
aValueSize,
aFlags,
aStoredValueRep );
int ValueOversize = 0;
if ((status == SKV_SUCCESS) || (status == SKV_ERRNO_NEED_DATA_TRANSFER))
{
*aTotalSize = aStoredValueRep->GetLen();
ValueOversize = aStoredValueRep->GetLen() - aValueSize;
if( ValueOversize > 0 )
{
BegLogLine( SKV_LOCAL_KV_BACKEND_LOG )
<< "skv_local_kv_inmem:: ADJUSTING rdma_write length to match smaller client buffer"
<< " client: " << aValueSize
<< " store: " << aStoredValueRep->GetLen()
<< EndLogLine;
aStoredValueRep->SetLenIfSmaller( aValueSize );
}
int RoomForData = ((skv_header_as_cmd_buffer_t*)aReq)->GetRoomForData( sizeof( skv_cmd_retrieve_value_rdma_write_ack_t) );
if( RoomForData < StoredValueRep.GetLen() )
status = SKV_ERRNO_NEED_DATA_TRANSFER;
}
else
*aTotalSize = 0;
BegLogLine( SKV_LOCAL_KV_BACKEND_LOG )
<< "skv_local_kv_inmem:: storing valueRep:" << *aStoredValueRep
<< " status:" << skv_status_to_string( status )
<< EndLogLine;
return status;
}
/***************************************************
* Layout of the bulk row
* { key_size, key, value_size, value }
**************************************************/
static
int
skv_bulk_insert_get_key_value_refs( char *aRow, char **Key, int &KeyLength, char **Value, int & ValueLength )
{
int Sint = sizeof(int);
char * Cursor = aRow;
int *klen = (int *) Cursor;
BegLogLine(SKV_CLIENT_ENDIAN_LOG)
<< " &KeyLength= " << (void *) &KeyLength
<< " " << (long) &KeyLength
<< " klen= " << (void *) klen
<< " " << (long) klen
<< " " << EndLogLine ;
int Starklen = *klen ;
int LocalKeyLength = ntohl(Starklen) ;
BegLogLine(SKV_CLIENT_ENDIAN_LOG)
<< "Endian-converted from " << (void *) (intptr_t)Starklen
<< " to " << LocalKeyLength
<< EndLogLine ;
KeyLength = LocalKeyLength;
BegLogLine(SKV_CLIENT_ENDIAN_LOG)
<< "Endian-converting KeyLength from " << (void *) (intptr_t)(*klen)
<< EndLogLine ;
Cursor += Sint;
*Key = Cursor;
Cursor += KeyLength;
int *vlen = (int *) Cursor;
ValueLength = ntohl(*vlen);
BegLogLine(SKV_CLIENT_ENDIAN_LOG)
<< "Endian-converting ValueLength from " << (void *) (intptr_t)(*vlen)
<< EndLogLine ;
Cursor += Sint;
*Value = Cursor;
BegLogLine(SKV_CLIENT_ENDIAN_LOG)
<< "KeyLength=" << KeyLength
<< " ValueLength=" << ValueLength
<< EndLogLine ;
return KeyLength + ValueLength + 2 * Sint;
}
/****************************************************
* Insert the record into local store.
****************************************************/
static inline
skv_status_t
InsertLocal( skv_cmd_RIU_req_t *aReq,
char *aAddr,
int aKeySize,
int aTotalValueSize,
skv_pds_manager_if_t *aPDSManager,
int aMyRank )
{
gSKVServerInsertInTreeStart.HitOE( SKV_SERVER_INSERT_TRACE,
"SKVServerInsertInTree",
aMyRank,
gSKVServerInsertInTreeStart );
skv_status_t status = aPDSManager->Insert( aReq->mPDSId,
aAddr,
aKeySize,
aTotalValueSize );
gSKVServerInsertInTreeFinis.HitOE( SKV_SERVER_INSERT_TRACE,
"SKVServerInsertInTree",
aMyRank,
gSKVServerInsertInTreeFinis );
BegLogLine( (SKV_LOCAL_KV_BACKEND_LOG && (status != SKV_SUCCESS )) )
<< "skv_local_kv_inmem::InsertLocal():: ERROR: "
<< " status: " << skv_status_to_string( status )
<< EndLogLine;
return status;
}
skv_status_t Init( it_pz_handle_t aPZ_Hdl,
size_t aDataAreaSize,
size_t aDataChunkSize,
size_t aAlignment=sizeof(uintptr_t) )
{
StrongAssertLogLine( aDataAreaSize % aAlignment == 0 )
<< "skv_local_kv_rdma_data_buffer_t::Init(): Area size has to be multiple of Alignment"
<< " size=" << aDataAreaSize
<< " align=" << aAlignment
<< EndLogLine;
mAlignment = aAlignment;
size_t pad = sizeof( skv_lmr_wait_queue_t) - (sizeof( skv_lmr_wait_queue_t ) % aAlignment);
if( pad==aAlignment ) pad = 0;
mHeadSpace = sizeof( skv_lmr_wait_queue_t) + pad;
mDataArea = new char[ aDataAreaSize ];
StrongAssertLogLine( mDataArea != NULL )
<< "skv_local_kv_rdma_data_buffer_t::Init(): Failed to allocate data area of size: " << aDataAreaSize
<< EndLogLine;
mChunkSize = aDataChunkSize;
it_mem_priv_t privs = (it_mem_priv_t) ( IT_PRIV_LOCAL | IT_PRIV_REMOTE );
it_lmr_flag_t lmr_flags = IT_LMR_FLAG_NON_SHAREABLE;
it_lmr_handle_t lmr;
it_rmr_context_t rmr;
it_status_t itstatus = it_lmr_create( aPZ_Hdl,
mDataArea,
NULL,
aDataAreaSize,
IT_ADDR_MODE_ABSOLUTE,
privs,
lmr_flags,
0,
& lmr,
& rmr );
StrongAssertLogLine( itstatus == IT_SUCCESS )
<< "skv_tree_based_container_t::Init:: ERROR:: itstatus == IT_SUCCESS "
<< " itstatus: " << itstatus
<< EndLogLine;
mLMR.InitAbs( lmr, mDataArea, aDataAreaSize );
mRMR.Init( rmr, mDataArea, aDataAreaSize );
mFirstFree.Init( mDataArea, aDataAreaSize, mAlignment );
mLastBusy.Init( mDataArea, aDataAreaSize, mAlignment );
BegLogLine( SKV_LOCAL_KV_RDMA_DATA_BUFFER_LOG )
<< "skv_local_kv_rdma_data_buffer_t: FF[" << (uintptr_t)mFirstFree.GetPtr() << "]"
<< " LB[" << (uintptr_t)mLastBusy.GetPtr() << "]"
<< " mHeadSpace=" << mHeadSpace
<< " mAlignment=" << mAlignment
<< EndLogLine;
return SKV_SUCCESS;
}
skv_status_t
skv_local_kv_asyncmem::PDS_Open( char *aPDSName,
skv_pds_priv_t aPrivs,
skv_cmd_open_flags_t aFlags,
skv_pds_id_t *aPDSId,
skv_local_kv_cookie_t *aCookie )
{
skv_local_kv_request_t *kvReq = mRequestQueue.AcquireRequestEntry();
if( !kvReq )
return SKV_ERRNO_COMMAND_LIMIT_REACHED;
kvReq->InitCommon( SKV_LOCAL_KV_REQUEST_TYPE_OPEN, aCookie );
memcpy( &kvReq->mData[ 0 ], aPDSName, SKV_MAX_PDS_NAME_SIZE );
kvReq->mRequest.mOpen.mPDSName = &kvReq->mData[ 0 ];
kvReq->mRequest.mOpen.mPrivs = aPrivs;
kvReq->mRequest.mOpen.mFlags = aFlags;
mRequestQueue.QueueRequest( kvReq );
BegLogLine( SKV_LOCAL_KV_BACKEND_LOG )
<< "skv_local_kv_asyncmem: Open Request stored:"
<< " PDSName:" << kvReq->mRequest.mOpen.mPDSName
<< " Priv:" << kvReq->mRequest.mOpen.mPrivs
<< " Flags: " << kvReq->mRequest.mOpen.mFlags
<< EndLogLine;
return SKV_ERRNO_LOCAL_KV_EVENT;
}
skv_status_t
skv_local_kv_asyncmem::PerformLookup( skv_local_kv_request_t *aReq )
{
BegLogLine( SKV_LOCAL_KV_BACKEND_LOG )
<< "skv_local_kv_asyncmem::Lookup():: Entering"
<< EndLogLine;
skv_status_t status;
gSKVServerInsertRetrieveFromTreeStart.HitOE( SKV_SERVER_INSERT_TRACE,
"SKVServerInsertRetrieveFromTree",
mMyRank,
gSKVServerInsertRetrieveFromTreeStart );
skv_lmr_triplet_t StoredValueRep;
// Check if the key exists
status = mPDSManager.Retrieve( aReq->mRequest.mLookup.mPDSId,
aReq->mRequest.mLookup.mKeyData,
aReq->mRequest.mLookup.mKeySize,
0,
0,
aReq->mRequest.mLookup.mFlags,
&StoredValueRep );
gSKVServerInsertRetrieveFromTreeFinis.HitOE( SKV_SERVER_INSERT_TRACE,
"SKVServerInsertRetrieveFromTree",
mMyRank,
gSKVServerInsertRetrieveFromTreeFinis );
status = InitKVEvent( aReq->mCookie, &StoredValueRep, status );
return status;
}
static inline
skv_status_t insert_create_multi_stage( skv_server_ep_state_t *aEPState,
skv_local_kv_t *aLocalKV,
skv_server_ccb_t *aCommand,
int aCommandOrdinal,
skv_cmd_RIU_req_t *aReq )
{
skv_status_t status = SKV_SUCCESS;
BegLogLine( SKV_SERVER_INSERT_LOG )
<< "skv_server_insert_command_sm::Execute()::insert_create_multi_stage"
<< " Command requires async operation (async storage or data transfer)..."
<< EndLogLine;
// check if we're already multi-stage class command
if ( aCommand->GetCommandClass() == SKV_COMMAND_CLASS_MULTI_STAGE )
return status;
/*******************************************************************
* Save local command state
******************************************************************/
aCommand->mCommandState.mCommandInsert.mHdr = aReq->mHdr;
aCommand->mCommandState.mCommandInsert.mFlags = aReq->mFlags;
/******************************************************************/
aEPState->ReplaceAndInitCommandBuffer( aCommand, aCommandOrdinal );
return status;
}
skv_status_t
skv_local_kv_inmem::Init( int aRank,
int aNodeCount,
skv_server_internal_event_manager_if_t *aInternalEventMgr,
it_pz_handle_t aPZ,
char* aCheckpointPath )
{
skv_status_t status;
mMyRank = aRank;
/************************************************************
* Initialize the local partition dataset manager
***********************************************************/
BegLogLine( SKV_LOCAL_KV_BACKEND_LOG )
<< "skv_local_kv_inmem::Init(): Entering..."
<< EndLogLine;
status = mPDSManager.Init( aRank,
aNodeCount,
aInternalEventMgr,
aPZ,
aCheckpointPath );
StrongAssertLogLine( status == SKV_SUCCESS )
<< "skv_local_kv_inmem::Init():: ERROR:: mPDSManager.Init() failed. "
<< " status: " << skv_status_to_string( status )
<< " Rank: " << aRank
<< " PartitionSize: " << aNodeCount
<< EndLogLine;
/***********************************************************/
return status;
}
iWARPEM_Multiplexed_Endpoint_t( int aRouterFd = 0,
int aMaxClientCount = IT_API_MULTIPLEX_MAX_PER_SOCKET,
iWARPEM_Object_Accept_t *aListenerCtx = NULL )
: mClientCount(0), mRouterConnFd( aRouterFd ), mMaxClientCount( aMaxClientCount ),
mReceiveDataLen( 0 ), mPendingRequests( 0 ), mNeedsBufferFlush( false )
{
bzero( &mRouterInfo, sizeof( iWARPEM_Router_Info_t ));
mClientEPs = new MultiplexedConnectionType*[ aMaxClientCount ];
bzero( mClientEPs, sizeof( MultiplexedConnectionType*) * aMaxClientCount );
pthread_spin_init( &mAccessLock, PTHREAD_PROCESS_PRIVATE );
mListenerContext = aListenerCtx;
mSendBuffer = new SocketBufferType();
mReceiveBuffer = new ReceiveBufferType();
mReceiveBuffer->ConfigureAsReceiveBuffer();
ResetSendBuffer();
ResetRecvBuffer();
BegLogLine( FXLOG_IT_API_O_SOCKETS_MULTIPLEX_LOG )
<< "Creating multiplexed router endpoint..."
<< " socket: " << mRouterConnFd
<< " maxClients: " << aMaxClientCount
<< " BufferSizes: " << IT_API_MULTIPLEX_SOCKET_BUFFER_SIZE
<< " Recv: @" << (void*)mReceiveBuffer
<< " Send: @" << (void*)mSendBuffer
<< EndLogLine;
#ifdef MULTIPLEX_STATISTICS
mMsgCount = 0;
mFlushCount = 0;
mMsgAvg = 1.0;
#endif
}
/***
* skv_client_internal_t::CloseLocalCursor::
* Desc: Get the first element in the cursor
* returns: SKV_SUCCESS on success or error code
***/
skv_status_t
skv_client_internal_t::
CloseCursor( skv_client_cursor_handle_t aCursorHdl )
{
BegLogLine( SKV_CLIENT_CURSOR_LOG )
<< "skv_client_internal_t::CloseCursor(): Entering "
<< " aCursorHdl: " << (void *) aCursorHdl
<< EndLogLine;
mCursorManagerIF.FinalizeCursorHdl( aCursorHdl );
BegLogLine( SKV_CLIENT_CURSOR_LOG )
<< "skv_client_internal_t::CloseCursor(): Leaving "
<< EndLogLine;
return SKV_SUCCESS;
}
skv_status_t
skv_local_kv_inmem::Deallocate ( skv_lmr_triplet_t *aRDMARep )
{
BegLogLine(SKV_LOCAL_KV_BACKEND_LOG)
<< "Deallocating buffer"
<< EndLogLine ;
return mPDSManager.Deallocate( aRDMARep );
}
static inline
void insert_post_rdma( skv_server_ep_state_t *aEPState,
skv_local_kv_t *aLocalKV,
int aCommandOrdinal,
skv_cmd_RIU_req_t *aReq,
skv_lmr_triplet_t *aValueRepForRdmaRead,
int* aSeqNo,
int aMyRank )
{
skv_server_cookie_t Cookie;
Cookie.Init( aEPState,
*aSeqNo,
aCommandOrdinal );
it_dto_cookie_t* DtoCookie = (it_dto_cookie_t*) &Cookie;
it_dto_flags_t dto_flags = (it_dto_flags_t) (IT_COMPLETION_FLAG | IT_NOTIFY_FLAG);
gSKVServerInsertRDMAReadStart.HitOE( SKV_SERVER_INSERT_TRACE,
"SKVServerInsertRdmaRead",
aMyRank,
gSKVServerInsertRDMAReadStart );
aLocalKV->RDMABoundsCheck( "Insert1",
(char *) aValueRepForRdmaRead->GetAddr(),
aValueRepForRdmaRead->GetLen() );
BegLogLine( SKV_SERVER_INSERT_LOG )
<< "skv_server_insert_command_sm:: Execute():: Before rdma_read "
<< " LMR triplet { "
<< " len: " << aValueRepForRdmaRead->GetLen()
<< " addr: " << (void *) aValueRepForRdmaRead->GetAddr()
<< " lmr: " << (void *) aValueRepForRdmaRead->GetLMRHandle()
<< " } "
<< " RMR info "
<< " rmr addr: " << (void *) aReq->mRMRTriplet.GetAddr()
<< " rmr context: " << (void *) aReq->mRMRTriplet.GetRMRContext()
<< EndLogLine;
// rdma_read the value
it_status_t itstatus = it_post_rdma_read( aEPState->mEPHdl,
aValueRepForRdmaRead->GetTripletPtr(),
1,
*DtoCookie,
dto_flags,
(it_rdma_addr_t) aReq->mRMRTriplet.GetAddr(),
aReq->mRMRTriplet.GetRMRContext() );
gSKVServerInsertRDMAReadFinis.HitOE( SKV_SERVER_INSERT_TRACE,
"SKVServerInsertRdmaRead",
aMyRank,
gSKVServerInsertRDMAReadFinis );
AssertLogLine( itstatus == IT_SUCCESS )
<< "skv_server_insert_command_sm::Execute():: ERROR: "
<< " istatus: " << itstatus
<< EndLogLine;
}
// if *aClient is passed as NULL, data is extracted without the multiplexed message header - pure raw retrieval
// user has to make sure that this doesn't break the protocol!!
iWARPEM_Status_t ExtractRawData( char *aBuffer, int aSize, int *aRecvd, iWARPEM_StreamId_t *aClient = NULL )
{
iWARPEM_Status_t status = IWARPEM_SUCCESS;
pthread_spin_lock( &mAccessLock );
if( ! RecvDataAvailable() )
{
BegLogLine( FXLOG_IT_API_O_SOCKETS_MULTIPLEX_LOG )
<< "No data available in ReadBuffer. Reading new set of data."
<< EndLogLine;
status = mReceiveBuffer->FillFromSocket( mRouterConnFd );
switch( status )
{
case IWARPEM_SUCCESS:
break;
default:
BegLogLine( FXLOG_IT_API_O_SOCKETS_MULTIPLEX_LOG )
<< "Error while reading data from router endpoint."
<< EndLogLine;
pthread_spin_unlock( &mAccessLock );
return status;
}
}
pthread_spin_unlock( &mAccessLock );
// data can only be an iWARPEM_Message_Hdr_t if the client is requested too
if( aClient != NULL )
{
size_t hdrlen = sizeof( iWARPEM_Multiplexed_Msg_Hdr_t );
iWARPEM_Multiplexed_Msg_Hdr_t *MultHdr = (iWARPEM_Multiplexed_Msg_Hdr_t*)mReceiveBuffer->GetHdrPtr( &hdrlen );
AssertLogLine( hdrlen == sizeof( iWARPEM_Multiplexed_Msg_Hdr_t ) )
<< "Retrieval of HdrPtr failed: len=" << hdrlen
<< " expected=" << sizeof(iWARPEM_Multiplexed_Msg_Hdr_t)
<< EndLogLine;
MultHdr->ClientID = be64toh( MultHdr->ClientID );
*aClient = MultHdr->ClientID;
hdrlen = sizeof( iWARPEM_Message_Hdr_t );
char* databuf = mReceiveBuffer->GetHdrPtr( &hdrlen );
memcpy( aBuffer, databuf, hdrlen );
}
else if( aSize )
mReceiveBuffer->GetData( &aBuffer, (size_t*)&aSize );
return IWARPEM_SUCCESS;
}
void *test_case(void* vdev_mem)
{
BegLogLine(1) << __FUNCTION__ << "()" << " STARTED " << EndLogLine;
SRSW_QueueSet_WriterConnect((union SRSW_QueueSet_t *) &RoQ_DevMem->mTxDoorbellQueueSet[0], // Queue Set pointer
0, DD_DoorbellQueue_Size, NULL, sizeof(u32));
int part_size = ND_TORUS_SIZE(personality->Network_Config);
void* bla = roq_verbs_init(vdev_mem);
if (!bla)
return 0;
roq_ud_qp_main(RoQ_LocalRank, part_size, bla);
BegLogLine(1) << __FUNCTION__ << "()" << " BYEBYE " << EndLogLine;
while(1);
}
int mSize(void) const
{
int rc=ntohl(mSizeBE);
BegLogLine(SKV_SERVER_ENDIAN_LOG)
<< "Endian-converting " << (void *) (intptr_t)mSizeBE
<< " to " << rc
<< EndLogLine ;
return rc ;
}
skv_status_t
skv_local_kv_inmem::Allocate( int aBuffSize,
skv_lmr_triplet_t *aRDMARep )
{
BegLogLine(SKV_LOCAL_KV_BACKEND_LOG)
<< "Allocating buffer, size=" << aBuffSize
<< EndLogLine ;
return mPDSManager.Allocate( aBuffSize, aRDMARep );
}
void Report()
{
if( HistOnFlag )
{
BegLogLine( 1 )
<< "histogram_t(): "
<< " Name: " << mName
<< " mLow: " << mLow
<< " mHigh: " << mHigh
<< " mBucketCount: " << mBucketCount
<< " mDataPointCount: " << mDataPointCount
<< " mDataPointValueCount: " << mDataPointValueCount
<< EndLogLine;
double CountsPerBucket = 1.0 / mBucketReciprocal;
BegLogLine( 1 )
<< "histogram_t(): "
<< " Name: " << mName
<< " mOutOfRangeLowCBin: " << mOutOfRangeLowCBin
<< " mOutOfRangeHighCBin: " << mOutOfRangeHighCBin
<< EndLogLine;
BegLogLine( 1 )
<< "histogram_t(): "
<< " Name: " << mName
<< " mOutOfRangeLowVBin: " << mOutOfRangeLowVBin
<< " mOutOfRangeHighVBin: " << mOutOfRangeHighVBin
<< EndLogLine;
for( int i = 0; i < mBucketCount; i++)
{
BegLogLine( 1 )
<< "histogram_t(): "
<< " Name: " << mName
<< " Interval: { " << (long long) (CountsPerBucket*i + mLow) << " , "
<< (long long) (CountsPerBucket * (i+1) + mLow ) << " } "
<< " CountBin[ " << i << " ]: " << mCountBins[ i ]
<< " ValueBin[ " << i << " ]: " << mValueBins[ i ]
<< EndLogLine;
}
}
}
skv_status_t
skv_local_kv_asyncmem::Init( int aRank,
int aNodeCount,
skv_server_internal_event_manager_if_t *aInternalEventMgr,
it_pz_handle_t aPZ,
char* aCheckpointPath )
{
skv_status_t status;
mMyRank = aRank;
BegLogLine( SKV_LOCAL_KV_BACKEND_LOG )
<< "skv_local_kv_asyncmem::Init(): Entering..."
<< EndLogLine;
mEventQueue.Init();
mRequestQueue.Init();
/************************************************************
* Initialize the local partition dataset manager
***********************************************************/
status = mPDSManager.Init( aRank,
aNodeCount,
aInternalEventMgr,
aPZ,
aCheckpointPath );
if( status != SKV_SUCCESS )
{
BegLogLine( status != SKV_SUCCESS )
<< "skv_local_kv_asyncmem::Init():: ERROR:: mPDSManager.Init() failed. "
<< " status: " << skv_status_to_string( status )
<< " Rank: " << aRank
<< " PartitionSize: " << aNodeCount
<< EndLogLine;
return status;
}
/***********************************************************/
mKeepProcessing = true;
mReqProcessor = new std::thread(AsyncProcessing, this );
return status;
}
void
Finalize()
{
BegLogLine( SKV_CLIENT_CURSOR_LOG )
<< "skv_client_cursor_control_block_t::Finalize():: Entering..."
<< " mKeysDataLMRHdl: " << (void *) mKeysDataLMRHdl
<< EndLogLine;
it_status_t status = it_lmr_free( mKeysDataLMRHdl );
StrongAssertLogLine( status == IT_SUCCESS )
<< "skv_client_cursor_control_block_t::Finalize():: ERROR:: "
<< " status: " << status
<< EndLogLine;
BegLogLine( SKV_CLIENT_CURSOR_LOG )
<< "skv_client_cursor_control_block_t::Finalize():: Leaving..."
<< EndLogLine;
}
