bool ClientSession::PostSend()
{
if (!IsConnected())
return false;
FastSpinlockGuard criticalSection(mBufferLock);
if ( 0 == mBuffer.GetContiguiousBytes() )
return true;
OverlappedSendContext* sendContext = new OverlappedSendContext(this);
DWORD sendbytes = 0;
DWORD flags = 0;
sendContext->mWsaBuf.len = (ULONG) mBuffer.GetContiguiousBytes();
sendContext->mWsaBuf.buf = mBuffer.GetBufferStart();
/// start async send
if (SOCKET_ERROR == WSASend(mSocket, &sendContext->mWsaBuf, 1, &sendbytes, flags, (LPWSAOVERLAPPED)sendContext, NULL))
{
if (WSAGetLastError() != WSA_IO_PENDING)
{
DeleteIoContext(sendContext);
printf_s("ClientSession::PostSend Error : %d\n", GetLastError());
return false;
}
}
return true;
}
bool ClientSession::PreRecv()
{
if (!IsConnected())
return false;
OverlappedPreRecvContext* recvContext = new OverlappedPreRecvContext(this);
DWORD recvbytes = 0;
DWORD flags = 0;
recvContext->mWsaBuf.len = 0;
recvContext->mWsaBuf.buf = nullptr;
/// start async recv
if (SOCKET_ERROR == WSARecv(mSocket, &recvContext->mWsaBuf, 1, &recvbytes, &flags, (LPWSAOVERLAPPED)recvContext, NULL))
{
if (WSAGetLastError() != WSA_IO_PENDING)
{
DeleteIoContext(recvContext);
printf_s("ClientSession::PreRecv Error : %d\n", GetLastError());
return false;
}
}
return true;
}
bool ClientSession::PostRecv()
{
if (!IsConnected())
return false;
FastSpinlockGuard criticalSection(mBufferLock);
if (0 == mBuffer.GetFreeSpaceSize())
return false;
OverlappedRecvContext* recvContext = new OverlappedRecvContext(this);
DWORD recvbytes = 0;
DWORD flags = 0;
recvContext->mWsaBuf.len = (ULONG)mBuffer.GetFreeSpaceSize();
recvContext->mWsaBuf.buf = mBuffer.GetBuffer();
/// start real recv
if (SOCKET_ERROR == WSARecv(mSocket, &recvContext->mWsaBuf, 1, &recvbytes, &flags, (LPWSAOVERLAPPED)recvContext, NULL))
{
if (WSAGetLastError() != WSA_IO_PENDING)
{
DeleteIoContext(recvContext);
printf_s("ClientSession::PostRecv Error : %d\n", GetLastError());
return false;
}
}
return true;
}
bool DummyClientSession::ConnectRequest()
{
if (mConnected)
{
/// 이미 접속 된 상태인데 이쪽으로 들어오면 잘못된 것
CRASH_ASSERT(false);
return false;
}
// Set up our socket address structure
ZeroMemory(&mConnectAddr, sizeof(mConnectAddr));
mConnectAddr.sin_port = htons(CONNECT_PORT);
mConnectAddr.sin_family = AF_INET;
mConnectAddr.sin_addr.s_addr = inet_addr(CONNECT_ADDR);
OverlappedConnectContext* context = new OverlappedConnectContext(this);
if (FALSE == ConnectEx(mSocket, (sockaddr*)&mConnectAddr, sizeof(SOCKADDR_IN), NULL, 0, NULL, (LPWSAOVERLAPPED)context))
{
if (WSAGetLastError() != WSA_IO_PENDING)
{
DeleteIoContext(context);
printf_s("DummyClientSession::ConnectRequest Error : %d\n", GetLastError());
}
}
return true;
}
bool ClientSession::PostConnect()
{
CRASH_ASSERT(LThreadType == THREAD_MAIN);
// 바인드 한번만 하게 처리해야함
{
struct sockaddr_in addr;
ZeroMemory( &addr, sizeof( addr ) );
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = INADDR_ANY;
addr.sin_port = 0;
int rc = bind( mSocket, (SOCKADDR*)&addr, sizeof( addr ) );
if ( rc != 0 )
{
printf( "bind failed: %d\n", WSAGetLastError() );
return 1;
}
}
HANDLE handle = CreateIoCompletionPort( (HANDLE)mSocket, GIocpManager->GetComletionPort(), ( ULONG_PTR )this, 0 );
if ( handle != GIocpManager->GetComletionPort() )
{
printf_s( "[DEBUG] CreateIoCompletionPort error: %d\n", GetLastError() );
return false;
}
OverlappedConnectContext* connectContext = new OverlappedConnectContext( this );
DWORD bytes = 0;
DWORD flags = 0;
connectContext->mWsaBuf.len = 0;
connectContext->mWsaBuf.buf = nullptr;
sockaddr_in addr;
ZeroMemory( &addr, sizeof( addr ) );
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = inet_addr( "127.0.0.1" );
addr.sin_port = htons( CONNECT_PORT );
if ( FALSE == ConnectEx( mSocket, (SOCKADDR*)&addr, sizeof( addr ), NULL, 0, NULL, ( LPOVERLAPPED )connectContext ) )
{
if (WSAGetLastError() != WSA_IO_PENDING)
{
DeleteIoContext( connectContext );
printf_s("AcceptEx Error : %d\n", GetLastError());
return false;
}
}
return true;
}
bool Session::FlushSend()
{
if (!IsConnected())
{
DisconnectRequest(DR_SENDFLUSH_ERROR);
return true;
}
FastSpinlockGuard criticalSection(mSendBufferLock);
/// 보낼 데이터가 없는 경우
if (0 == mSendBuffer.GetContiguiousBytes())
{
/// 보낼 데이터도 없는 경우
if (0 == mSendPendingCount)
return true;
return false;
}
/// 이전의 send가 완료 안된 경우
if (mSendPendingCount > 0)
return false;
OverlappedSendContext* sendContext = new OverlappedSendContext(this);
DWORD sendbytes = 0;
DWORD flags = 0;
sendContext->mWsaBuf.len = (ULONG)mSendBuffer.GetContiguiousBytes();
sendContext->mWsaBuf.buf = mSendBuffer.GetBufferStart();
/// start async send
if (SOCKET_ERROR == WSASend(mSocket, &sendContext->mWsaBuf, 1, &sendbytes, flags, (LPWSAOVERLAPPED)sendContext, NULL))
{
if (WSAGetLastError() != WSA_IO_PENDING)
{
DeleteIoContext(sendContext);
printf_s("Session::FlushSend Error : %d\n", GetLastError());
DisconnectRequest(DR_SENDFLUSH_ERROR);
return true;
}
}
mSendPendingCount++;
return mSendPendingCount == 1;
}
bool ClientSession::PostConnect()
{
CRASH_ASSERT( LThreadType == THREAD_MAIN );
HANDLE handle = CreateIoCompletionPort( (HANDLE)mSocket, GIocpManager->GetComletionPort(), ( ULONG_PTR )this, 0 );
if ( handle != GIocpManager->GetComletionPort() )
{
printf_s( "[DEBUG] CreateIoCompletionPort error: %d\n", GetLastError() );
return false;
}
OverlappedConnectContext* connectContext = new OverlappedConnectContext( this );
DWORD bytes = 0;
connectContext->mWsaBuf.len = 0;
connectContext->mWsaBuf.buf = nullptr;
if ( SERVER_PORT <= 4000 || SERVER_PORT > 10000 )
{
SERVER_PORT = 9001;
}
if ( HOST_NAME.length() == 0 )
{
HOST_NAME = "127.0.0.1";
}
sockaddr_in addr;
ZeroMemory( &addr, sizeof( addr ) );
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = inet_addr( HOST_NAME.c_str() );
addr.sin_port = htons( SERVER_PORT );
if ( FALSE == IocpManager::ConnectEx( mSocket, (SOCKADDR*)&addr, sizeof( addr ),
GIocpManager->mConnectBuf, 0, &bytes, (LPOVERLAPPED)connectContext ) )
{
if ( WSAGetLastError() != WSA_IO_PENDING )
{
DeleteIoContext( connectContext );
printf_s( "ConnectEx Error : %d\n", GetLastError() );
return false;
}
}
return true;
}
void ClientSession::DisconnectRequest(DisconnectReason dr)
{
/// 이미 끊겼거나 끊기는 중이거나
if (0 == InterlockedExchange(&mConnected, 0))
return ;
OverlappedDisconnectContext* context = new OverlappedDisconnectContext(this, dr);
if (FALSE == IocpManager::DisconnectEx(mSocket, (LPWSAOVERLAPPED)context, TF_REUSE_SOCKET, 0))
{
if (WSAGetLastError() != WSA_IO_PENDING)
{
DeleteIoContext(context);
printf_s("ClientSession::DisconnectRequest Error : %d\n", GetLastError());
}
}
}
bool ServerSession::ConnectRequest()
{
if (mConnected)
{
/// 이미 접속 된 상태인데 이쪽으로 들어오면 잘못된 것
CRASH_ASSERT(false);
return false;
}
// Set up our socket address structure
SOCKADDR_IN serverSockAddr;
ZeroMemory(&serverSockAddr, sizeof(serverSockAddr));
serverSockAddr.sin_port = htons(mPort);
serverSockAddr.sin_family = AF_INET;
serverSockAddr.sin_addr.s_addr = inet_addr(mServerAddr);
if (SOCKET_ERROR == bind(mSocket, (SOCKADDR*)&serverSockAddr, sizeof(serverSockAddr)))
{
printf_s("ServerSession::ConnectRequest() bind error: %d\n", GetLastError());
return false;
}
HANDLE handle = CreateIoCompletionPort((HANDLE)mSocket, GIocpManager->GetComletionPort(), (ULONG_PTR)this, 0);
if (handle != GIocpManager->GetComletionPort())
{
printf_s("ServerSession::ConnectRequest() CreateIoCompletionPort error: %d\n", GetLastError());
return false;
}
OverlappedConnectContext* context = new OverlappedConnectContext(this);
if (FALSE == ConnectEx(mSocket, (sockaddr*)&serverSockAddr, sizeof(SOCKADDR_IN), NULL, 0, NULL, (LPWSAOVERLAPPED)context))
{
if (WSAGetLastError() != WSA_IO_PENDING)
{
DeleteIoContext(context);
printf_s("ServerSession::ConnectRequest Error : %d\n", GetLastError());
}
}
return true;
}
bool ClientSession::PostAccept()
{
CRASH_ASSERT(LThreadType == THREAD_MAIN);
OverlappedAcceptContext* acceptContext = new OverlappedAcceptContext(this);
DWORD bytes = 0;
DWORD flags = 0;
acceptContext->mWsaBuf.len = 0;
acceptContext->mWsaBuf.buf = nullptr;
if (FALSE == AcceptEx(*GIocpManager->GetListenSocket(), mSocket, GIocpManager->mAcceptBuf, 0,
sizeof(SOCKADDR_IN)+16, sizeof(SOCKADDR_IN)+16, &bytes, (LPOVERLAPPED)acceptContext))
{
if (WSAGetLastError() != WSA_IO_PENDING)
{
DeleteIoContext(acceptContext);
printf_s("AcceptEx Error : %d\n", GetLastError());
return false;
}
}
return true;
}
void ClientSession::ConnectCompletion()
{
CRASH_ASSERT( LThreadType == THREAD_IO_WORKER );
if ( 1 == InterlockedExchange( &mConnected, 1 ) )
{
/// already exists?
CRASH_ASSERT( false );
return;
}
bool resultOk = true;
do
{
if ( SOCKET_ERROR == setsockopt( mSocket, SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, NULL, 0 ) )
{
printf_s( "[DEBUG] SO_UPDATE_CONNECT_CONTEXT error: %d\n", GetLastError() );
resultOk = false;
break;
}
int opt = 1;
if ( NO_DELAY )
{
int opt = 1;
if ( SOCKET_ERROR == setsockopt( mSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&opt, sizeof( int ) ) )
{
printf_s( "[DEBUG] TCP_NODELAY error: %d\n", GetLastError() );
resultOk = false;
break;
}
}
opt = 0;
if ( SOCKET_ERROR == setsockopt( mSocket, SOL_SOCKET, SO_RCVBUF, (const char*)&opt, sizeof( int ) ) )
{
printf_s( "[DEBUG] SO_RCVBUF change error: %d\n", GetLastError() );
resultOk = false;
break;
}
int addrlen = sizeof( SOCKADDR_IN );
if ( SOCKET_ERROR == getpeername( mSocket, (SOCKADDR*)&mClientAddr, &addrlen ) )
{
printf_s( "[DEBUG] getpeername error: %d\n", GetLastError() );
resultOk = false;
break;
}
} while ( false );
if ( !resultOk )
{
DisconnectRequest( DR_ONCONNECT_ERROR );
return;
}
FastSpinlockGuard criticalSection( mBufferLock );
if ( BUFFER_SIZE <= 0 || BUFFER_SIZE > BUFSIZE )
{
BUFFER_SIZE = 4096;
}
char* temp = new char[BUFFER_SIZE];
ZeroMemory( temp, sizeof( char ) * BUFFER_SIZE );
for ( int i = 0; i < BUFFER_SIZE - 1; ++i )
{
temp[i] = 'a' + ( mSocket % 26 );
}
temp[BUFFER_SIZE - 1] = '\0';
char* bufferStart = mBuffer.GetBuffer();
memcpy( bufferStart, temp, BUFFER_SIZE );
mBuffer.Commit( BUFFER_SIZE );
CRASH_ASSERT( 0 != mBuffer.GetContiguiousBytes() );
OverlappedSendContext* sendContext = new OverlappedSendContext( this );
DWORD sendbytes = 0;
DWORD flags = 0;
sendContext->mWsaBuf.len = (ULONG)mBuffer.GetContiguiousBytes();
sendContext->mWsaBuf.buf = mBuffer.GetBufferStart();
delete[] temp;
/// start async send
if ( SOCKET_ERROR == WSASend( mSocket, &sendContext->mWsaBuf, 1, &sendbytes, flags, (LPWSAOVERLAPPED)sendContext, NULL ) )
{
if ( WSAGetLastError() != WSA_IO_PENDING )
{
DeleteIoContext( sendContext );
printf_s( "ClientSession::PostSend Error : %d\n", GetLastError() );
}
}
++mUseCount;
}
unsigned int WINAPI IocpManager::IoWorkerThread(LPVOID lpParam)
{
LThreadType = THREAD_IO_WORKER;
LIoThreadId = reinterpret_cast<int>(lpParam);
LTimer = new Timer;
LLockOrderChecker = new LockOrderChecker(LIoThreadId);
HANDLE hComletionPort = GIocpManager->GetComletionPort();
while (true)
{
/// 타이머 작업은 항상 돌리고
LTimer->DoTimerJob();
/// IOCP 작업 돌리기
DWORD dwTransferred = 0;
OverlappedIOContext* context = nullptr;
ULONG_PTR completionKey = 0;
int ret = GetQueuedCompletionStatus(hComletionPort, &dwTransferred, (PULONG_PTR)&completionKey, (LPOVERLAPPED*)&context, GQCS_TIMEOUT);
ClientSession* theClient = context ? context->mSessionObject : nullptr ;
if (ret == 0 || dwTransferred == 0)
{
int gle = GetLastError();
/// check time out first
if (gle == WAIT_TIMEOUT)
continue;
if (context->mIoType == IO_RECV || context->mIoType == IO_SEND )
{
CRASH_ASSERT(nullptr != theClient);
/// In most cases in here: ERROR_NETNAME_DELETED(64)
theClient->DisconnectRequest(DR_COMPLETION_ERROR);
DeleteIoContext(context);
continue;
}
}
CRASH_ASSERT(nullptr != theClient);
bool completionOk = false;
switch (context->mIoType)
{
case IO_DISCONNECT:
theClient->DisconnectCompletion(static_cast<OverlappedDisconnectContext*>(context)->mDisconnectReason);
completionOk = true;
break;
case IO_ACCEPT:
theClient->AcceptCompletion();
completionOk = true;
break;
case IO_RECV_ZERO:
completionOk = PreReceiveCompletion(theClient, static_cast<OverlappedPreRecvContext*>(context), dwTransferred);
break;
case IO_SEND:
completionOk = SendCompletion(theClient, static_cast<OverlappedSendContext*>(context), dwTransferred);
break;
case IO_RECV:
completionOk = ReceiveCompletion(theClient, static_cast<OverlappedRecvContext*>(context), dwTransferred);
break;
default:
printf_s("Unknown I/O Type: %d\n", context->mIoType);
CRASH_ASSERT(false);
break;
}
if ( !completionOk )
{
/// connection closing
theClient->DisconnectRequest(DR_IO_REQUEST_ERROR);
}
DeleteIoContext(context);
}
return 0;
}
void IOThread::DoIocpJob()
{
DWORD dwTransferred = 0;
LPOVERLAPPED overlapped = nullptr;
ULONG_PTR completionKey = 0;
int ret = GetQueuedCompletionStatus(mCompletionPort, &dwTransferred, (PULONG_PTR)&completionKey, &overlapped, GQCS_TIMEOUT);
/// general I/O processing
OverlappedIOContext* context = reinterpret_cast<OverlappedIOContext*>(overlapped);
SessionPtr remote = context ? context->mSessionObject : nullptr;
if (ret == 0 || dwTransferred == 0)
{
/// check time out first
if ( context == nullptr && GetLastError() == WAIT_TIMEOUT)
return;
if (context->mIoType == IO_RECV || context->mIoType == IO_SEND)
{
CRASH_ASSERT(nullptr != remote);
/// In most cases in here: ERROR_NETNAME_DELETED(64)
remote->Disconnect(DR_COMPLETION_ERROR);
DeleteIoContext(context);
return;
}
}
CRASH_ASSERT(nullptr != remote);
bool completionOk = false;
switch (context->mIoType)
{
case IO_RECV_ZERO:
completionOk = remote->PostRecv();
break;
case IO_SEND:
remote->SendCompletion(dwTransferred);
if (context->mWsaBuf.len != dwTransferred)
GConsoleLog->PrintOut(true, "Partial SendCompletion requested [%d], sent [%d]\n", context->mWsaBuf.len, dwTransferred);
else
completionOk = true;
break;
case IO_RECV:
remote->RecvCompletion(dwTransferred);
completionOk = remote->PreRecv();
break;
default:
GConsoleLog->PrintOut(true, "Unknown I/O Type: %d\n", context->mIoType);
CRASH_ASSERT(false);
break;
}
if (!completionOk)
{
/// connection closing
remote->Disconnect(DR_IO_REQUEST_ERROR);
}
DeleteIoContext(context);
}
void IOThread::DoIocpJob()
{
DWORD dwTransferred = 0;
LPOVERLAPPED overlapped = nullptr;
ULONG_PTR completionKey = 0;
int ret = GetQueuedCompletionStatus(mCompletionPort, &dwTransferred, (PULONG_PTR)&completionKey, &overlapped, GQCS_TIMEOUT);
if (CK_DB_RESULT == completionKey)
{
//-todo: DB 처리 결과가 담겨오는 경우 처리
DatabaseJobContext* dbContext = reinterpret_cast<DatabaseJobContext*>(overlapped);
dbContext->OnResult();
// make switch deleter? like OverlappedIOContext in deleter...
delete dbContext;
return;
}
/// 아래로는 일반적인 I/O 처리
OverlappedIOContext* context = reinterpret_cast<OverlappedIOContext*>(overlapped);
Session* remote = context ? context->mSessionObject : nullptr;
if (ret == 0 || dwTransferred == 0)
{
/// check time out first
if ( context == nullptr && GetLastError() == WAIT_TIMEOUT)
return;
if (context->mIoType == IO_RECV || context->mIoType == IO_SEND)
{
CRASH_ASSERT(nullptr != remote);
/// In most cases in here: ERROR_NETNAME_DELETED(64)
remote->DisconnectRequest(DR_COMPLETION_ERROR);
DeleteIoContext(context);
return;
}
}
CRASH_ASSERT(nullptr != remote);
bool completionOk = false;
switch (context->mIoType)
{
case IO_CONNECT:
dynamic_cast<ServerSession*>(remote)->ConnectCompletion();
completionOk = true;
break;
case IO_DISCONNECT:
remote->DisconnectCompletion(static_cast<OverlappedDisconnectContext*>(context)->mDisconnectReason);
completionOk = true;
break;
case IO_ACCEPT:
dynamic_cast<ClientSession*>(remote)->AcceptCompletion();
completionOk = true;
break;
case IO_RECV_ZERO:
completionOk = remote->PostRecv();
break;
case IO_SEND:
remote->SendCompletion(dwTransferred);
if (context->mWsaBuf.len != dwTransferred)
printf_s("Partial SendCompletion requested [%d], sent [%d]\n", context->mWsaBuf.len, dwTransferred);
else
completionOk = true;
break;
case IO_RECV:
remote->RecvCompletion(dwTransferred);
/// for test
//remote->EchoBack();
remote->Handle_CS_MOVE(); // test
completionOk = remote->PreRecv();
break;
default:
printf_s("Unknown I/O Type: %d\n", context->mIoType);
CRASH_ASSERT(false);
break;
}
if (!completionOk)
{
/// connection closing
remote->DisconnectRequest(DR_IO_REQUEST_ERROR);
}
DeleteIoContext(context);
}
void IOThread::DoIocpJob()
{
DWORD dwTransferred = 0;
OverlappedIOContext* context = nullptr;
ULONG_PTR completionKey = 0;
int ret = GetQueuedCompletionStatus( mCompletionPort, &dwTransferred, (PULONG_PTR)&completionKey, (LPOVERLAPPED*)&context, GQCS_TIMEOUT );
if ( completionKey == THREAD_QUIT_KEY )
{
mIsContinue = false;
return;
}
ClientSession* session = context ? context->mSessionObject : nullptr;
if ( ret == 0 || dwTransferred == 0 )
{
int gle = GetLastError();
/// check time out first
if ( gle == WAIT_TIMEOUT && context == nullptr )
{
return;
}
if ( context->mIoType == IO_RECV || context->mIoType == IO_SEND )
{
CRASH_ASSERT( nullptr != session );
/// In most cases in here: ERROR_NETNAME_DELETED(64)
session->DisconnectRequest( DR_COMPLETION_ERROR );
DeleteIoContext( context );
return;
}
}
CRASH_ASSERT( nullptr != session );
bool completionOk = false;
switch ( context->mIoType )
{
case IO_CONNECT:
session->ConnectCompletion();
completionOk = true;
break;
case IO_DISCONNECT:
session->DisconnectCompletion( static_cast<OverlappedDisconnectContext*>( context )->mDisconnectReason );
completionOk = true;
break;
case IO_RECV_ZERO:
completionOk = session->PostRecv();
break;
case IO_SEND:
session->SendCompletion( dwTransferred );
if ( context->mWsaBuf.len != dwTransferred )
printf_s( "Partial SendCompletion requested [%d], sent [%d]\n", context->mWsaBuf.len, dwTransferred );
else
completionOk = true;
break;
case IO_RECV:
session->RecvCompletion( dwTransferred );
// 일단 테스트용 에코!
// session->EchoBack();
completionOk = session->PreRecv();
break;
default:
printf_s( "Unknown I/O Type: %d\n", context->mIoType );
CRASH_ASSERT( false );
break;
}
if ( !completionOk )
{
/// connection closing
session->DisconnectRequest( DR_IO_REQUEST_ERROR );
}
DeleteIoContext( context );
}