//
// Worker thread that handles all I/O requests on any socket handle added to the IOCP.
//
DWORD WINAPI WorkerThread (LPVOID WorkThreadContext) {
HANDLE hIOCP = (HANDLE)WorkThreadContext;
BOOL bSuccess = FALSE;
int nRet = 0;
LPWSAOVERLAPPED lpOverlapped = NULL;
PPER_SOCKET_CONTEXT lpPerSocketContext = NULL;
PPER_SOCKET_CONTEXT lpAcceptSocketContext = NULL;
PPER_IO_CONTEXT lpIOContext = NULL;
WSABUF buffRecv;
WSABUF buffSend;
DWORD dwRecvNumBytes = 0;
DWORD dwSendNumBytes = 0;
DWORD dwFlags = 0;
DWORD dwIoSize = 0;
HRESULT hRet;
while( TRUE ) {
//
// continually loop to service io completion packets
//
bSuccess = GetQueuedCompletionStatus(
hIOCP,
&dwIoSize,
(PDWORD_PTR)&lpPerSocketContext,
(LPOVERLAPPED *)&lpOverlapped,
INFINITE
);
if( !bSuccess )
myprintf("GetQueuedCompletionStatus() failed: %d\n", GetLastError());
if( lpPerSocketContext == NULL ) {
//
// CTRL-C handler used PostQueuedCompletionStatus to post an I/O packet with
// a NULL CompletionKey (or if we get one for any reason). It is time to exit.
//
return(0);
}
if( g_bEndServer ) {
//
// main thread will do all cleanup needed - see finally block
//
return(0);
}
lpIOContext = (PPER_IO_CONTEXT)lpOverlapped;
//
//We should never skip the loop and not post another AcceptEx if the current
//completion packet is for previous AcceptEx
//
if( lpIOContext->IOOperation != ClientIoAccept ) {
if( !bSuccess || (bSuccess && (0 == dwIoSize)) ) {
//
// client connection dropped, continue to service remaining (and possibly
// new) client connections
//
CloseClient(lpPerSocketContext, FALSE);
continue;
}
}
//
// determine what type of IO packet has completed by checking the PER_IO_CONTEXT
// associated with this socket. This will determine what action to take.
//
switch( lpIOContext->IOOperation ) {
case ClientIoAccept:
//
// When the AcceptEx function returns, the socket sAcceptSocket is
// in the default state for a connected socket. The socket sAcceptSocket
// does not inherit the properties of the socket associated with
// sListenSocket parameter until SO_UPDATE_ACCEPT_CONTEXT is set on
// the socket. Use the setsockopt function to set the SO_UPDATE_ACCEPT_CONTEXT
// option, specifying sAcceptSocket as the socket handle and sListenSocket
// as the option value.
//
nRet = setsockopt(
lpPerSocketContext->pIOContext->SocketAccept,
SOL_SOCKET,
SO_UPDATE_ACCEPT_CONTEXT,
(char *)&g_sdListen,
sizeof(g_sdListen)
);
if( nRet == SOCKET_ERROR ) {
//
//just warn user here.
//
myprintf("setsockopt(SO_UPDATE_ACCEPT_CONTEXT) failed to update accept socket\n");
WSASetEvent(g_hCleanupEvent[0]);
return(0);
}
lpAcceptSocketContext = UpdateCompletionPort(
lpPerSocketContext->pIOContext->SocketAccept,
ClientIoAccept, TRUE);
if( lpAcceptSocketContext == NULL ) {
//
//just warn user here.
//
myprintf("failed to update accept socket to IOCP\n");
WSASetEvent(g_hCleanupEvent[0]);
return(0);
}
if( dwIoSize ) {
lpAcceptSocketContext->pIOContext->IOOperation = ClientIoWrite;
lpAcceptSocketContext->pIOContext->nTotalBytes = dwIoSize;
lpAcceptSocketContext->pIOContext->nSentBytes = 0;
lpAcceptSocketContext->pIOContext->wsabuf.len = dwIoSize;
hRet = StringCbCopyN(lpAcceptSocketContext->pIOContext->Buffer,
MAX_BUFF_SIZE,
lpPerSocketContext->pIOContext->Buffer,
sizeof(lpPerSocketContext->pIOContext->Buffer)
);
lpAcceptSocketContext->pIOContext->wsabuf.buf = lpAcceptSocketContext->pIOContext->Buffer;
nRet = WSASend(
lpPerSocketContext->pIOContext->SocketAccept,
&lpAcceptSocketContext->pIOContext->wsabuf, 1,
&dwSendNumBytes,
0,
&(lpAcceptSocketContext->pIOContext->Overlapped), NULL);
if( nRet == SOCKET_ERROR && (ERROR_IO_PENDING != WSAGetLastError()) ) {
myprintf ("WSASend() failed: %d\n", WSAGetLastError());
CloseClient(lpAcceptSocketContext, FALSE);
} else if( g_bVerbose ) {
myprintf("WorkerThread %d: Socket(%d) AcceptEx completed (%d bytes), Send posted\n",
GetCurrentThreadId(), lpPerSocketContext->Socket, dwIoSize);
}
} else {
//
// AcceptEx completes but doesn't read any data so we need to post
// an outstanding overlapped read.
//
lpAcceptSocketContext->pIOContext->IOOperation = ClientIoRead;
dwRecvNumBytes = 0;
dwFlags = 0;
buffRecv.buf = lpAcceptSocketContext->pIOContext->Buffer,
buffRecv.len = MAX_BUFF_SIZE;
nRet = WSARecv(
lpAcceptSocketContext->Socket,
&buffRecv, 1,
&dwRecvNumBytes,
&dwFlags,
&lpAcceptSocketContext->pIOContext->Overlapped, NULL);
if( nRet == SOCKET_ERROR && (ERROR_IO_PENDING != WSAGetLastError()) ) {
myprintf ("WSARecv() failed: %d\n", WSAGetLastError());
CloseClient(lpAcceptSocketContext, FALSE);
}
}
//
//Time to post another outstanding AcceptEx
//
if( !CreateAcceptSocket(FALSE) ) {
myprintf("Please shut down and reboot the server.\n");
WSASetEvent(g_hCleanupEvent[0]);
return(0);
}
break;
case ClientIoRead:
//
// a read operation has completed, post a write operation to echo the
// data back to the client using the same data buffer.
//
lpIOContext->IOOperation = ClientIoWrite;
lpIOContext->nTotalBytes = dwIoSize;
lpIOContext->nSentBytes = 0;
lpIOContext->wsabuf.len = dwIoSize;
dwFlags = 0;
nRet = WSASend(
lpPerSocketContext->Socket,
&lpIOContext->wsabuf, 1, &dwSendNumBytes,
dwFlags,
&(lpIOContext->Overlapped), NULL);
if( nRet == SOCKET_ERROR && (ERROR_IO_PENDING != WSAGetLastError()) ) {
myprintf("WSASend() failed: %d\n", WSAGetLastError());
CloseClient(lpPerSocketContext, FALSE);
} else if( g_bVerbose ) {
myprintf("WorkerThread %d: Socket(%d) Recv completed (%d bytes), Send posted\n",
GetCurrentThreadId(), lpPerSocketContext->Socket, dwIoSize);
}
break;
case ClientIoWrite:
//
// a write operation has completed, determine if all the data intended to be
// sent actually was sent.
//
lpIOContext->IOOperation = ClientIoWrite;
lpIOContext->nSentBytes += dwIoSize;
dwFlags = 0;
if( lpIOContext->nSentBytes < lpIOContext->nTotalBytes ) {
//
// the previous write operation didn't send all the data,
// post another send to complete the operation
//
buffSend.buf = lpIOContext->Buffer + lpIOContext->nSentBytes;
buffSend.len = lpIOContext->nTotalBytes - lpIOContext->nSentBytes;
nRet = WSASend (
lpPerSocketContext->Socket,
&buffSend, 1, &dwSendNumBytes,
dwFlags,
&(lpIOContext->Overlapped), NULL);
if( nRet == SOCKET_ERROR && (ERROR_IO_PENDING != WSAGetLastError()) ) {
myprintf ("WSASend() failed: %d\n", WSAGetLastError());
CloseClient(lpPerSocketContext, FALSE);
} else if( g_bVerbose ) {
myprintf("WorkerThread %d: Socket(%d) Send partially completed (%d bytes), Recv posted\n",
GetCurrentThreadId(), lpPerSocketContext->Socket, dwIoSize);
}
} else {
//
// previous write operation completed for this socket, post another recv
//
lpIOContext->IOOperation = ClientIoRead;
dwRecvNumBytes = 0;
dwFlags = 0;
buffRecv.buf = lpIOContext->Buffer,
buffRecv.len = MAX_BUFF_SIZE;
nRet = WSARecv(
lpPerSocketContext->Socket,
&buffRecv, 1, &dwRecvNumBytes,
&dwFlags,
&lpIOContext->Overlapped, NULL);
if( nRet == SOCKET_ERROR && (ERROR_IO_PENDING != WSAGetLastError()) ) {
myprintf ("WSARecv() failed: %d\n", WSAGetLastError());
CloseClient(lpPerSocketContext, FALSE);
} else if( g_bVerbose ) {
myprintf("WorkerThread %d: Socket(%d) Send completed (%d bytes), Recv posted\n",
GetCurrentThreadId(), lpPerSocketContext->Socket, dwIoSize);
}
}
break;
} //switch
} //while
return(0);
}
DWORD WINAPI IocpServerWorker(LPVOID p)
{
SYSTEM_INFO SystemInfo;
DWORD ThreadID;
SOCKET Accept;
int nRet;
int ClientIndex;
SOCKADDR_IN cAddr;
IN_ADDR cInAddr;
int cAddrlen = sizeof( cAddr );
LPPER_SOCKET_CONTEXT lpPerSocketContext = NULL;
DWORD RecvBytes;
DWORD Flags=0;
InitializeCriticalSection(&criti);
GetSystemInfo(&SystemInfo);
g_dwThreadCount = SystemInfo.dwNumberOfProcessors * 2;
__try
{
g_CompletionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
if ( g_CompletionPort == NULL)
{
LogAdd("CreateIoCompletionPort failed with error: %d", GetLastError());
__leave;
}
for(DWORD dwCPU = 0; dwCPU < g_dwThreadCount; dwCPU++)
{
HANDLE ThreadHandle;
// Create a server worker thread and pass the completion port to the thread.
ThreadHandle = CreateThread(NULL, 0, ServerWorkerThread, g_CompletionPort, 0, &ThreadID);
if ( ThreadHandle == NULL)
{
LogAdd("CreateThread() failed with error %d", GetLastError());
__leave;
}
g_ThreadHandles[dwCPU] = ThreadHandle;
CloseHandle(ThreadHandle);
}
if (!CreateListenSocket() )
__leave;
while(TRUE)
{
Accept = WSAAccept(g_Listen, (LPSOCKADDR)&cAddr, &cAddrlen, NULL, 0);
if (Accept==SOCKET_ERROR)
{
EnterCriticalSection(&criti);
LogAdd("WSAAccept() failed with error %d", WSAGetLastError());
LeaveCriticalSection(&criti);
continue;
}
EnterCriticalSection(&criti);
memcpy( &cInAddr, &cAddr.sin_addr.s_addr, 4 );
ClientIndex = gObjAddSearch(Accept, inet_ntoa(cInAddr) );
if ( ClientIndex == -1 )
{
LogAddL("error-L2 : ClientIndex = -1");
closesocket(Accept);
LeaveCriticalSection(&criti);
continue;
}
if (UpdateCompletionPort(Accept, ClientIndex, 1) == 0 )
{
LogAddL("error-L1 : %d %d CreateIoCompletionPort failed with error %d", Accept, ClientIndex, GetLastError() );
closesocket(Accept);
LeaveCriticalSection(&criti);
continue;
}
if (gObjAdd(Accept, inet_ntoa(cInAddr), ClientIndex) == -1 )
{
LogAddL("error-L1 : %d %d gObjAdd() failed with error %d", Accept, ClientIndex, GetLastError() );
LeaveCriticalSection(&criti);
closesocket(Accept);
continue;
}
memset(&gObj[ClientIndex].PerSocketContext->IOContext[0].Overlapped, 0, sizeof(OVERLAPPED));
memset(&gObj[ClientIndex].PerSocketContext->IOContext[1].Overlapped, 0, sizeof(OVERLAPPED));
gObj[ClientIndex].PerSocketContext->IOContext[0].wsabuf.buf = gObj[ClientIndex].PerSocketContext->IOContext[0].Buffer;
gObj[ClientIndex].PerSocketContext->IOContext[0].wsabuf.len = MAX_IO_BUFFER_SIZE;
gObj[ClientIndex].PerSocketContext->IOContext[0].nTotalBytes = 0;
gObj[ClientIndex].PerSocketContext->IOContext[0].nSentBytes = 0;
gObj[ClientIndex].PerSocketContext->IOContext[0].nWaitIO = 0;
gObj[ClientIndex].PerSocketContext->IOContext[0].nSecondOfs = 0;
gObj[ClientIndex].PerSocketContext->IOContext[0].IOOperation = RECV_IO;
gObj[ClientIndex].PerSocketContext->IOContext[1].wsabuf.buf = gObj[ClientIndex].PerSocketContext->IOContext[0].Buffer;
gObj[ClientIndex].PerSocketContext->IOContext[1].wsabuf.len = MAX_IO_BUFFER_SIZE;
gObj[ClientIndex].PerSocketContext->IOContext[1].nTotalBytes= 0;
gObj[ClientIndex].PerSocketContext->IOContext[1].nSentBytes = 0;
gObj[ClientIndex].PerSocketContext->IOContext[1].nWaitIO = 0;
gObj[ClientIndex].PerSocketContext->IOContext[1].nSecondOfs = 0;
gObj[ClientIndex].PerSocketContext->IOContext[1].IOOperation = SEND_IO;
gObj[ClientIndex].PerSocketContext->m_socket = Accept;
gObj[ClientIndex].PerSocketContext->nIndex = ClientIndex;
nRet = WSARecv(Accept, &(gObj[ClientIndex].PerSocketContext->IOContext[0].wsabuf), 1, &RecvBytes, &Flags,
&(gObj[ClientIndex].PerSocketContext->IOContext[0].Overlapped), NULL);
if( nRet == SOCKET_ERROR && WSAGetLastError() != ERROR_IO_PENDING )
{
LogAdd("error-L1 : WSARecv() failed with error %d", WSAGetLastError());
gObj[ClientIndex].PerSocketContext->IOContext[0].nWaitIO = 4;
CloseClient(gObj[ClientIndex].PerSocketContext, FALSE);
LeaveCriticalSection(&criti);
continue;
}
gObj[ClientIndex].PerSocketContext->IOContext[0].nWaitIO = 1;
gObj[ClientIndex].PerSocketContext->dwIOCount++;
LeaveCriticalSection(&criti);
SCPJoinResultSend(ClientIndex, 1);
}
}
__finally
{
if( g_CompletionPort )
{
for (DWORD i = 0; i < g_dwThreadCount; i++)
PostQueuedCompletionStatus(g_CompletionPort, 0, 0, NULL);
}
if( g_CompletionPort )
{
CloseHandle(g_CompletionPort);
g_CompletionPort = NULL;
}
if (g_Listen != INVALID_SOCKET)
{
closesocket(g_Listen);
g_Listen = INVALID_SOCKET;
}
}
return TRUE;
}
//
// Create a socket and invoke AcceptEx. Only the original call to to this
// function needs to be added to the IOCP.
//
// If the expected behaviour of connecting client applications is to NOT
// send data right away, then only posting one AcceptEx can cause connection
// attempts to be refused if a client connects without sending some initial
// data (notice that the associated iocpclient does not operate this way
// but instead makes a connection and starts sending data write away).
// This is because the IOCP packet does not get delivered without the initial
// data (as implemented in this sample) thus preventing the worker thread
// from posting another AcceptEx and eventually the backlog value set in
// listen() will be exceeded if clients continue to try to connect.
//
// One technique to address this situation is to simply cause AcceptEx
// to return right away upon accepting a connection without returning any
// data. This can be done by setting dwReceiveDataLength=0 when calling AcceptEx.
//
// Another technique to address this situation is to post multiple calls
// to AcceptEx. Posting multiple calls to AcceptEx is similar in concept to
// increasing the backlog value in listen(), though posting AcceptEx is
// dynamic (i.e. during the course of running your application you can adjust
// the number of AcceptEx calls you post). It is important however to keep
// your backlog value in listen() high in your server to ensure that the
// stack can accept connections even if your application does not get enough
// CPU cycles to repost another AcceptEx under stress conditions.
//
// This sample implements neither of these techniques and is therefore
// susceptible to the behaviour described above.
//
BOOL CreateAcceptSocket(BOOL fUpdateIOCP) {
int nRet = 0;
DWORD dwRecvNumBytes = 0;
DWORD bytes = 0;
//
// GUID to Microsoft specific extensions
//
GUID acceptex_guid = WSAID_ACCEPTEX;
//
//The context for listening socket uses the SockAccept member to store the
//socket for client connection.
//
if( fUpdateIOCP ) {
g_pCtxtListenSocket = UpdateCompletionPort(g_sdListen, ClientIoAccept, FALSE);
if( g_pCtxtListenSocket == NULL ) {
myprintf("failed to update listen socket to IOCP\n");
return(FALSE);
}
// Load the AcceptEx extension function from the provider for this socket
nRet = WSAIoctl(
g_sdListen,
SIO_GET_EXTENSION_FUNCTION_POINTER,
&acceptex_guid,
sizeof(acceptex_guid),
&g_pCtxtListenSocket->fnAcceptEx,
sizeof(g_pCtxtListenSocket->fnAcceptEx),
&bytes,
NULL,
NULL
);
if (nRet == SOCKET_ERROR)
{
myprintf("failed to load AcceptEx: %d\n", WSAGetLastError());
return (FALSE);
}
}
g_pCtxtListenSocket->pIOContext->SocketAccept = CreateSocket();
if( g_pCtxtListenSocket->pIOContext->SocketAccept == INVALID_SOCKET) {
myprintf("failed to create new accept socket\n");
return(FALSE);
}
//
// pay close attention to these parameters and buffer lengths
//
nRet = g_pCtxtListenSocket->fnAcceptEx(g_sdListen, g_pCtxtListenSocket->pIOContext->SocketAccept,
(LPVOID)(g_pCtxtListenSocket->pIOContext->Buffer),
MAX_BUFF_SIZE - (2 * (sizeof(SOCKADDR_STORAGE) + 16)),
sizeof(SOCKADDR_STORAGE) + 16, sizeof(SOCKADDR_STORAGE) + 16,
&dwRecvNumBytes,
(LPOVERLAPPED) &(g_pCtxtListenSocket->pIOContext->Overlapped));
if( nRet == SOCKET_ERROR && (ERROR_IO_PENDING != WSAGetLastError()) ) {
myprintf("AcceptEx() failed: %d\n", WSAGetLastError());
return(FALSE);
}
return(TRUE);
}
