//
// Function: FreeSocketObj
//
// Description:
//    Frees a socket object along with any queued buffer objects.
//
void FreeSocketObj(SOCKET_OBJ *obj)
{
    BUFFER_OBJ  *ptr=NULL,
                *tmp=NULL;

    ptr = obj->pending;
    while (ptr)
    {
        tmp = ptr;
        ptr = ptr->next;

        FreeBufferObj(tmp);
    }

    DeleteCriticalSection(&obj->SockCritSec);

    HeapFree(GetProcessHeap(), 0, obj);
}
//
// Function: FreeSocketObj
//
// Description:
//    Frees a socket object along with any queued buffer objects.
//
void FreeSocketObj(SOCKET_OBJ *obj)
{
    BUFFER_OBJ  *ptr=NULL,
                *tmp=NULL;

    ptr = obj->pending;
    while (ptr)
    {
        tmp = ptr;
        ptr = ptr->next;

        FreeBufferObj(tmp);
    }

    WSACloseEvent(obj->event);

    if (obj->s != INVALID_SOCKET)
    {
        closesocket(obj->s);
    }

    HeapFree(GetProcessHeap(), 0, obj);
}
Example #3
0
//---------------------------------------------------------------------------------------------------------------------------
//CLConnect()
//NULL - connect failed .
//---------------------------------------------------------------------------------------------------------------------------
SOCKET_OBJ * CIocpServer::CLConnect(const char * ip,const char *port,int flag)
{
	SOCKET_OBJ *sockobj = NULL;
	BUFFER_OBJ *recvobj = NULL;
	struct addrinfo *res=NULL;
	bool   bFlag = false;

	res = ResolveAddress(ip, port, AF_INET,SOCK_STREAM, IPPROTO_TCP);
	if (res == NULL)
	{
		fprintf(stderr, "CLConnect ResolveAddress failed to return any addresses!\n");
		return NULL;
	}

	sockobj = GetSocketObj(INVALID_SOCKET);

	// create the socket
	sockobj->s = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
	if (sockobj->s == INVALID_SOCKET)
	{
		fprintf(stderr,"CLConnect socket failed: %d\n", WSAGetLastError());
		FreeSocketObj(sockobj);
		return NULL;
	}

#ifdef ENABLE_UNLIMIT_PORT
	SOCKADDR_IN local_addr={0};
	local_addr.sin_family = AF_INET;
	local_addr.sin_addr.s_addr  = ADDR_ANY;
	BOOL optvar = true;
	for(int i=0;i<40;i++)
	{
		local_addr.sin_port = htons(m_UnlimitPort);
		setsockopt(sockobj->s,SOL_SOCKET,SO_REUSEADDR,(char*)&optvar,sizeof(optvar));

		if(bind(sockobj->s,(SOCKADDR *)&local_addr,sizeof(SOCKADDR)) == -1)
		{
			if(m_UnlimitPort < UNLIMIT_PORT_END) 
				m_UnlimitPort++;
			else
				m_UnlimitPort = UNLIMIT_PORT_START;

			if(WSAGetLastError() != WSAEADDRINUSE )
			{ //bind出错了
				fprintf(stderr,"bind() failed: %d\n", WSAGetLastError());
				closesocket(sockobj->s);
				sockobj->s = INVALID_SOCKET;
				FreeSocketObj(sockobj);
				return NULL;
			}

		}
		else
			break;
	}

	if(m_UnlimitPort < UNLIMIT_PORT_END) 
		m_UnlimitPort++;
	else
		m_UnlimitPort = UNLIMIT_PORT_START;

#endif

	if( connect(sockobj->s,res->ai_addr,(int)res->ai_addrlen)!=SOCKET_ERROR )
	{
		if( CreateIoCompletionPort((HANDLE)sockobj->s, CLCompletionPort, (ULONG_PTR)sockobj, 0) == NULL)
			fprintf(stderr, "CLConnect : CLCreateIoCompletionPort failed: %d\n", GetLastError());
		else 
			bFlag = true;
	}

#ifdef LOG_LEVEL2
	if(!bFlag)
		Log_Server.Write("CLConnect失败: %d\n",WSAGetLastError());
#endif

	if(bFlag)
	{
		recvobj = GetBufferObj();

#ifdef DEBUG_IOCP
		recvobj->sclient = sockobj->s;
#endif

		LOCK(&sockobj->cs);

		if (PostRecv(sockobj, recvobj) != NO_ERROR)
		{
			UNLOCK(&sockobj->cs);
			FreeBufferObj(recvobj);
		}
		else
		{
			sockobj->flag = flag;
			sockobj->flag_accept = 1;
			sockobj->flag_close = 1;

			sockobj->recvobj = recvobj;

#ifdef ENABLE_KEEPALIVE
			AddAlive(sockobj,GetTickCount());
#endif

#ifdef DEBUG_IOCP
			sockobj->freeed = 0;
			sockobj->onclosed = 0;
#endif
			UNLOCK(&sockobj->cs);

			CLOnConnect(sockobj,sockobj->flag);

			return sockobj;
		}
	}

	closesocket(sockobj->s);
	sockobj->s = INVALID_SOCKET;
	FreeSocketObj(sockobj);

	return NULL;
}
Example #4
0
//---------------------------------------------------------------------------------------------------------------------------
//StartServer()
//start  listen
//---------------------------------------------------------------------------------------------------------------------------
int  CIocpServer::StartServer(const char *local_ip,const char *local_port)
{
	if(!m_Inited)
	{
		fprintf(stderr, "Run Init() first !\n");
		return -1;
	}

	SOCKET_OBJ      *sockobj=NULL;
	HANDLE           hrc;
	int              endpointcount=0,
					 rc,
					 i;
	HANDLE			 event_accept; 

	struct addrinfo *res=NULL,
		*ptr=NULL;


#ifdef LOG_LEVEL2
	printf("LOG_LEVEL2");
#else 
//
#ifdef LOG_LEVEL1
	printf("LOG_LEVEL1");
#else
	printf("LOG_NONE");
#endif
//
#endif

	printf("\n\n连接线程:%d: 监听线程:%d; 平均连接数:%d ; ",
		m_CLThreads,m_Threads,AVERAGE_CONNECTIONS);
	printf("IOCP版本: %s;\n",IOCP_VERSION);

#ifdef ENABLE_KEEPALIVE
	printf("心跳检测:开启; ");
#else
	printf("心跳检测:关闭; ");
#endif
	printf("心跳超时:%d; 关闭延时:%d\n",KEEPALIVE_TIME,CLOSE_DELAY);


	res = ResolveAddress(local_ip, local_port, AF_INET, SOCK_STREAM, IPPROTO_TCP);
	if (res == NULL)
	{
		fprintf(stderr, "ResolveAddress failed to return any addresses!\n");
		return -1;
	}

	ptr = res;
	if (ptr)
	{
		sockobj = GetSocketObj(INVALID_SOCKET);
		// create the socket
		sockobj->s = socket(ptr->ai_family, ptr->ai_socktype, ptr->ai_protocol);
		if (sockobj->s == INVALID_SOCKET)
		{
			fprintf(stderr,"socket failed: %d\n", WSAGetLastError());
			return -1;
		}

		// Associate the socket and its SOCKET_OBJ to the completion port
		hrc = CreateIoCompletionPort((HANDLE)sockobj->s, CompletionPort, (ULONG_PTR)sockobj, 0);
		if (hrc == NULL)
		{
			fprintf(stderr, "CreateIoCompletionPort failed: %d\n", GetLastError());
			return -1;
		}

		// bind the socket to a local address and port
		rc = bind(sockobj->s, ptr->ai_addr, (int)ptr->ai_addrlen);
		if (rc == SOCKET_ERROR)
		{
			fprintf(stderr, "bind failed: %d\n", WSAGetLastError());
			return -1;
		}

		BUFFER_OBJ *acceptobj=NULL;
		GUID        guidAcceptEx = WSAID_ACCEPTEX;
		GUID	    guidGetAcceptExSockaddrs = WSAID_GETACCEPTEXSOCKADDRS;
		DWORD       bytes;

		// Need to load the Winsock extension functions from each provider
		//    -- e.g. AF_INET and AF_INET6. 
		rc = WSAIoctl(
			sockobj->s,
			SIO_GET_EXTENSION_FUNCTION_POINTER,
			&guidAcceptEx,
			sizeof(guidAcceptEx),
			&lpfnAcceptEx,
			sizeof(lpfnAcceptEx),
			&bytes,
			NULL,
			NULL
			);
		if (rc == SOCKET_ERROR)
		{
			fprintf(stderr, "WSAIoctl: SIO_GET_EXTENSION_FUNCTION_POINTER failed: %d\n",
				WSAGetLastError());
			return -1;
		}

		rc = WSAIoctl(
			sockobj->s,
			SIO_GET_EXTENSION_FUNCTION_POINTER,
			&guidGetAcceptExSockaddrs,
			sizeof(guidGetAcceptExSockaddrs),
			&lpfnGetAcceptExSockaddrs,
			sizeof(lpfnGetAcceptExSockaddrs),
			&bytes,
			NULL,
			NULL
			);
		if (rc == SOCKET_ERROR)
		{
			fprintf(stderr, "WSAIoctl: SIO_GET_EXTENSION_FUNCTION_POINTER faled: %d\n",
				WSAGetLastError());
			return -1;
		}

		// For TCP sockets, we need to "listen" on them
		rc = _(sockobj->s, 32);
		if (rc == SOCKET_ERROR)
		{
			fprintf(stderr, "listen failed: %d\n", WSAGetLastError());
			return -1;
		}
		
		event_accept =  CreateEvent(NULL, TRUE, FALSE, NULL);
		if(event_accept == NULL)
		{
			fprintf(stderr,"event_accept CreateEvent failed: %d\n",GetLastError());
			return -1;
		}

		rc = WSAEventSelect(sockobj->s,event_accept,FD_ACCEPT);
		if (rc == SOCKET_ERROR)
		{
			fprintf(stderr, "WSAEventSelect failed: %d\n", WSAGetLastError());
			return -1;
		}

        
		// Post the AcceptEx(s)
		for(i=0; i < DEFAULT_ACCEPTEX_COUNT ;i++)
		{
			acceptobj = GetBufferObj();
			sockobj->recvobj = acceptobj;
			if(PostAccept(sockobj, acceptobj) != 0)
			{
				FreeBufferObj(acceptobj);
				fprintf(stderr, "PostAccept failed: %d\n", WSAGetLastError());
				return -1;
			}
		}


		endpointcount++;
		ptr = ptr->ai_next;
	}
	// free the addrinfo structure for the 'bind' address
	freeaddrinfo(res);

	DWORD sleeptime = CLOSE_DELAY / 2;				 //每次sleep的时间

#ifdef LOG_STATUS
	long		sock=0;
	long		buf=0;
#endif

	if(sleeptime < 50) //限制在50ms以外
		sleeptime = 50;

	DWORD time_ontimer=GetTickCount();
	DWORD time_close = time_ontimer;
	DWORD time_alive = time_ontimer;
	DWORD newtime;
	BUFFER_OBJ *acceptobj;

	printf("\n>>> 服务器地址: %s: 端口号: %s <<<\n\n",
		local_ip, local_port);

	while (1)
	{
		rc = WaitForSingleObject(event_accept,sleeptime);
		if(rc == WAIT_FAILED)
		{
			fprintf(stderr,"WaitForSingleObject Failed:%d\n",GetLastError());
			break;
		} 
		else if(rc != WAIT_TIMEOUT) //GOT FD_ACCEPT
		{
		//	acceptobj = GetBufferObj();
		//	PostAccept(sockobj,acceptobj);
			if(m_accept_count < DEFAULT_ACCEPTEX_COUNT /2)
			{
				for(int i=0;i<DEFAULT_ACCEPTEX_COUNT/2;i++)
				{
					acceptobj = GetBufferObj();
					PostAccept(sockobj,acceptobj);
				}
			}
			ResetEvent(event_accept);
		}

		newtime = GetTickCount();

		if(newtime - time_ontimer > 3000) // 3秒
		{ 
			OnTimer();  //call virtual timer();
			time_ontimer = newtime;			
#ifdef LOG_STATUS		//显示服务器状态
			if(sock != gTotalSock || buf != gTotalBuf)
			{
				Log_Server.Write("服务器状态 : sock : %d ; buf : %d ; IoCount : %d ;",gTotalSock,gTotalBuf,gIoCount);

				sock = gTotalSock;
				buf = gTotalBuf;
			}
#endif

		}

		//Check Close
		if(newtime - time_close > sleeptime)
		{
			time_close = newtime;
			CheckClose();
		}

		if(newtime - time_alive > KEEPALIVE_TIME / 3)
		{
			CheckAlive();
			time_alive = newtime;
		}

	}

	WSACleanup();

	return 0;
}
Example #5
0
//---------------------------------------------------------------------------------------------------------------------------
//HanelIO()
//---------------------------------------------------------------------------------------------------------------------------
void CIocpServer::HandleIo(SOCKET_OBJ *sock, BUFFER_OBJ *buf, 
						   HANDLE CompPort, DWORD BytesTransfered, DWORD error,
						   ADOConn *pAdo,bool isClient)
{

#ifdef LOG_STATUS
	InterlockedDecrement(&gIoCount);
#endif

	switch(buf->operation) 
	{	
	case OP_ACCEPT:		//处理AcceptEx的返回	
		LOCK(&sock->cs);//加锁
		{
			if(error == 0)
			{			
				SOCKET_OBJ *clientobj = GetSocketObj(buf->sclient);//创建一个客户端sockobj
				HANDLE hrc = CreateIoCompletionPort(	           //将sockobj添加到完成端口处理列表	
					(HANDLE)buf->sclient,
					CompPort,
					(ULONG_PTR)clientobj,
					0
					);

				if (hrc != NULL) 
				{
					//读取ip地址
					SOCKADDR *local=NULL;
					SOCKADDR *remote=NULL;
					int local_len;
					int remote_len;

					lpfnGetAcceptExSockaddrs(buf->buf,0,sizeof(SOCKADDR_STORAGE) + 16
						,sizeof(SOCKADDR_STORAGE) + 16,
						&local,&local_len,&remote,&remote_len);
					clientobj->addr = ((SOCKADDR_IN *)remote)->sin_addr;

#ifdef LOG_LEVEL1
					Log_Server.Write("--新连接 ip = %s , sock = %d ",GetAddress(clientobj),
						clientobj->s);
#endif

					BUFFER_OBJ *recvobj = GetBufferObj();
//禁用发送缓冲
#ifdef SEND_BUF_DISABLE
					BOOL sndbuf=0;
					setsockopt(buf->sclient,SOL_SOCKET,SO_SNDBUF,(char *)&sndbuf,sizeof(sndbuf));
#endif
					LOCK(&clientobj->cs);

					if(PostRecv(clientobj,recvobj) == 0)
					{ 
#ifdef DEBUG_IOCP
						clientobj->sockfd = clientobj->s;
						recvobj->sclient = clientobj->s;
#endif
						clientobj->recvobj = recvobj;
						AddAlive(clientobj,GetTickCount()); //把他添加到心跳列表,如果一个客户端连上来以后不发数据,会被踢掉
					}
					else
					{
						FreeBufferObj(recvobj);
						closesocket(clientobj->s);     //注意关闭句柄,clientobj->s的句柄在PostAccept时候创建
						clientobj->s = INVALID_SOCKET;
						FreeSocketObj(clientobj);
					}

					UNLOCK(&clientobj->cs);
				} 
				else
				{	//创建完成端口失败,像new失败一样,基本上不会出现
#ifdef LOG_LEVEL2
					Log_Server.Write("CreateIoCompletionPort 错误:%d",GetLastError());
#endif
					closesocket(clientobj->s);
					clientobj->s = INVALID_SOCKET;
					FreeSocketObj(clientobj);
				}
			}
			else	//如果AcceptEx返回出错,说明有一个客户端连了一半就退了,需要把他的句柄关了.
				closesocket(buf->sclient);

			InterlockedDecrement(&m_accept_count);
			// 一般情况下重新PostAccept
			if(m_accept_count < DEFAULT_ACCEPTEX_COUNT *2)
			{
				if(PostAccept(sock,buf)!=0)
					FreeBufferObj(buf);
			}
		}

		UNLOCK(&sock->cs);
		break;
	case OP_READ: //收到数据了
		{
			LOCK(&sock->cs); //锁一下

			bool bflag = false;
			_ASSERTE(buf == sock->recvobj);

			if(error == 0 && BytesTransfered > 0 )
			{ 
#ifdef LOG_LEVEL1  
				char head[256];
				sprintf(head,"(%d) :接收",sock->s);
				if(isClient)
					strcat(head,"CL");
				Log_Server.WriteHex(buf->buf+buf->buflen,BytesTransfered,head,(int)strlen(head));
#endif

				buf->buflen += BytesTransfered; 
				int nret = 0; 

				if(isClient)
				{	//调用可以重载的虚函数
					//开锁,否则在On系列函数中调用Send(),CloseSock()时候会死锁!
					UNLOCK(&sock->cs);
					nret = CLOnRead(sock,buf->buf,buf->buflen,sock->flag,pAdo);
					LOCK(&sock->cs);
				}
				else if(sock->flag_accept == 1)	
				{
					UNLOCK(&sock->cs);
					nret = OnRead(sock,buf->buf,buf->buflen,pAdo);
					LOCK(&sock->cs);
				}
				else
				{
					sock->flag_close = 1;  //设置允许标志,否则调用Send将会失败
					sock->flag_accept = 1; //设置accept标志
#ifndef ENABLE_KEEPALIVE		//没有启用KeepAlive	
					DeleteAlive(sock);
#endif
#ifdef DEBUG_IOCP
					sock->freeed = 0;
					sock->onclosed = 0;
#endif
					UNLOCK(&sock->cs);
					nret = OnAccept(sock,buf->buf,buf->buflen,pAdo);
					LOCK(&sock->cs);
				}

				_ASSERTE(nret >= 0 && nret <= buf->buflen);
				if(nret < 0 || nret > buf->buflen)
				{
#ifdef LOG_LEVEL2
					Log_Server.Write("指令处理出错啦\n");
					nret = buf->buflen; //强制设置为完全处理
#endif 
				}

#ifdef ENABLE_KEEPALIVE
				sock->timeAlive = GetTickCount(); //设置新的心跳时间
#endif
				buf->buflen -= nret;
				if(nret > 0 && buf->buflen > 0)
					memmove(buf->buf,buf->buf+nret,buf->buflen);

				if(PostRecv(sock,buf) == 0) //重新递交一个接收操作
					bflag = true;
			}

			if(!bflag)	
			{
				DeleteClose(sock); //看看是否存在于延时关闭列表中
				DeleteAlive(sock); //看看是否存在于心跳列表中

				if(sock->s != INVALID_SOCKET) {
					closesocket(sock->s);
					sock->s = INVALID_SOCKET;
				}

				sock->flag_close = 0; //设置关闭标志

				FreeBufferObj(buf);

				if( sock->flag_accept == 1) 
				{
					UNLOCK(&sock->cs);

					if(!isClient)
						OnClose(sock,pAdo); //调用一次OnClose(),告诉上层的程序,不要再去用这个sock了
					else
						CLOnClose(sock,sock->flag,pAdo);

					LOCK(&sock->cs);
#ifdef DEBUG_IOCP
					sock->onclosed = 1; //调试时设置关闭标志,用于检测是否存在逻辑问题
#endif
				}

				sock->recvobj = NULL;
				if(sock->recvobj == NULL && sock->sendobj == NULL) 
				{
					UNLOCK(&sock->cs);
					FreeSocketObj(sock); //释放该客户端对应的sockobj对象
					return;
				}
			}

			UNLOCK(&sock->cs);
		}
		break;
	case OP_WRITE:
		LOCK(&sock->cs);
		{
			_ASSERTE(buf == sock->sendobj);
			bool bflag = false;
			BUFFER_OBJ *tmpobj = sock->sendobj;
			sock->sendobj = sock->sendobj->next;

			if(error == 0 && BytesTransfered > 0) //前一个发送已经完成
			{ 
#ifdef LOG_LEVEL1
				char head[256];
				sprintf(head,"(%d) :发送",sock->s);
				if(isClient)
					strcat(head,"CL");
				Log_Server.WriteHex(buf->buf,BytesTransfered,head,(int)strlen(head));
#endif

				//检查发送队列
				if(sock->sendobj == NULL)
					bflag = true;
				else if(PostSend(sock,sock->sendobj) == 0) 
					bflag = true;				
			}

			FreeBufferObj(tmpobj);

			if(!bflag)
			{
				sock->flag_close = 0; //设置关闭标志
				while(sock->sendobj)
				{
					tmpobj = sock->sendobj;
					sock->sendobj = sock->sendobj->next;
					FreeBufferObj(tmpobj); 
				}

				if(sock->recvobj == NULL && sock->sendobj == NULL)
				{
					UNLOCK(&sock->cs);
					FreeSocketObj(sock); //如果OP_READ时,sock->sendobj!=NULL,那么需要在这里释放客户端的SocketOBJ
					return;
				}
			}
		}

		UNLOCK(&sock->cs);
		break;
	}

}
Example #6
0
//如果发送队列存在,Send()首先尝试将数据添加到队列的最后一个BufferOBJ.如果BufferOBJ的可用空间不足,
//Send()将开辟一个新的BufferOBJ用于发送.当发送队列为空时,Send()将调用PostSend()立即提交一个发送操作
//
int CIocpServer::Send(SOCKET_OBJ *sock,const void * buffer,int len)//发送向客户端数据
{
	_ASSERTE(len <= DEFAULT_BUFFER_SIZE*MAX_UNSENDS_COUNT && len > 0);

	if(sock == NULL || len > DEFAULT_BUFFER_SIZE*MAX_UNSENDS_COUNT || len <= 0)
		return -1;

	BUFFER_OBJ * tmpbuf;
	int rc=NO_ERROR;

#ifdef DEBUG_IOCP
	_ASSERTE(sock->onclosed == 0); //已经调用过OnClose()还发?
	_ASSERTE(sock->freeed == 0);   //已经free掉了!
#endif

	InterlockedIncrement(&sock->sending_count); //为了安全的freeobj

	if( ( sock->flag_close == 0 ) || (sock->s == INVALID_SOCKET) )
	{
		InterlockedDecrement(&sock->sending_count);
		return -1;
	}

	int i=0;
	int len2=len;
	char *buf2;

	if(len2 > DEFAULT_BUFFER_SIZE)
		len2 = DEFAULT_BUFFER_SIZE;

	LOCK(&sock->cs);

	tmpbuf = sock->sendobj;
	if(tmpbuf == NULL)
	{ //发送队列为空
		tmpbuf = GetBufferObj();

#ifdef DEBUG_IOCP
		tmpbuf->sclient = sock->s;
#endif

		memcpy(tmpbuf->buf,buffer,len2);
		tmpbuf->buflen = len2;

		rc = PostSend(sock,tmpbuf);
		if(rc == 0)
			sock->sendobj = tmpbuf;
		else
			FreeBufferObj(tmpbuf);

	}
	else
	{  
		while(tmpbuf->next)
		{
			tmpbuf = tmpbuf->next;
			i++;
		}

		if(i > MAX_UNSENDS_COUNT) 
		{
			rc = -1;
			CloseSock(sock);
		} 
		else
		{
			if(tmpbuf->buflen + len2 > DEFAULT_BUFFER_SIZE)
			{

				tmpbuf->next = GetBufferObj();
				tmpbuf = tmpbuf->next;
			}
			memcpy(tmpbuf->buf+tmpbuf->buflen,buffer,len2);
			tmpbuf->buflen += len2;
		}
	}

	len -= len2;
	buf2 = (char *)buffer+len2;

	while(rc == 0 && len >0)
	{
		len2 = len;

		if(len2 > DEFAULT_BUFFER_SIZE)
			len2 = DEFAULT_BUFFER_SIZE;

		tmpbuf->next = GetBufferObj();
		tmpbuf = tmpbuf->next;

		memcpy(tmpbuf->buf,buf2,len2);
		tmpbuf->buflen = len2;
		len -= len2;
		buf2 += len2;
		i++;

		if(i > MAX_UNSENDS_COUNT)
		{
#ifdef LOG_LEVEL1
			Log_Server.Write("Send(): 发送失败(当前用户的发送队列超出 %d!",MAX_UNSENDS_COUNT);
#endif
			rc = -1;			
			break;
		}
	}

	UNLOCK(&sock->cs);

	InterlockedDecrement(&sock->sending_count);

	return rc;
	//注意:在PostSend()里面,buflen被设置为DEFAULT_BUFFER_SIZE,Send()调用时候的数据不会添加到正在发送的BufferObj后面
}
//
// Function: SendPendingData
//
// Description:
//    Send any data pending on the socket. This routine goes through the 
//    queued buffer objects within the socket object and attempts to
//    send all of them. If the send fails with WSAEWOULDBLOCK, put the
//    remaining buffer back in the queue (at the front) for sending
//    later when select indicates sends can be made. This routine returns
//    -1 to indicate that an error has occured on the socket and the
//    calling routine should remove the socket structure; otherwise, zero
//    is returned.
//
int SendPendingData(SOCKET_OBJ *sock)
{
    BUFFER_OBJ *bufobj=NULL;
    BOOL        breakouter;
    int         nleft,
                idx,
                ret,
                rc;

    // Attempt to dequeue all the buffer objects on the socket
    ret = 0;
    while (bufobj = DequeueBufferObj(sock))
    {
        if (gProtocol == IPPROTO_TCP)
        {
            breakouter = FALSE;

            nleft = bufobj->buflen;
            idx = 0;

            // In the event not all the data was sent we need to increment
            // through the buffer. This only needs to be done for stream
            // sockets since UDP is datagram and its all or nothing for that.
            while (nleft)
            {
                rc = send(
                        sock->s,
                       &bufobj->buf[idx],
                        nleft,
                        0
                        );
                if (rc == SOCKET_ERROR)
                {
                    if (WSAGetLastError() == WSAEWOULDBLOCK)
                    {
                        BUFFER_OBJ *newbuf=NULL;

                        // Copy the unsent portion of the buffer and put it back
                        // at the head of the send queue
                        newbuf = GetBufferObj(nleft);
                        memcpy(newbuf->buf, &bufobj->buf[idx], nleft);

                        EnqueueBufferObj(sock, newbuf, TRUE);

                        ret = WSAEWOULDBLOCK;
                    }
                    else
                    {
                        // The connection was broken, indicate failure
                        ret = -1;
                    }
                    breakouter = TRUE;

                    break;
                }
                else
                {
                    // Update the stastics and increment the send counters
                    InterlockedExchangeAdd(&gBytesSent, rc);
                    InterlockedExchangeAdd(&gBytesSentLast, rc);

                    nleft -= rc;
                    idx += 0;
                }
            }
            FreeBufferObj(bufobj);

            if (breakouter)
                break;
        }
        else
        {
            rc = sendto(
                    sock->s,
                    bufobj->buf,
                    bufobj->buflen,
                    0,
                    (SOCKADDR *)&bufobj->addr,
                    bufobj->addrlen
                    );
            if (rc == SOCKET_ERROR)
            {
                if (WSAGetLastError() == WSAEWOULDBLOCK)
                {
                    // If the send couldn't be made, put the buffer
                    // back at the head of the queue
                    EnqueueBufferObj(sock, bufobj, TRUE);

                    ret = WSAEWOULDBLOCK;
                }
                else
                {
                    // Socket error occured so indicate the error to the caller
                    ret = -1;
                }
                break;
            }
            else
            {
                FreeBufferObj(bufobj);
            }
        }
    }
    // If no more sends are pending and the socket was marked as closing (the
    // receiver got zero bytes) then close the socket and indicate to the caller
    // to remove the socket structure.
    if ((sock->pending == NULL) && (sock->closing))
    {
        closesocket(sock->s);
        sock->s = INVALID_SOCKET;
        ret = -1;
    }
    return ret;
}
//
// Function: ReceivePendingData
//
// Description:
//    Receive data pending on the socket into a SOCKET_OBJ buffer. Enqueue
//    the buffer into the socket object for sending later. This routine returns
//    -1 indicating that the socket is no longer valid and the calling function
//    should clean up (remove) the socket object. Zero is returned for success.
//
int ReceivePendingData(SOCKET_OBJ *sockobj)
{
    BUFFER_OBJ *buffobj=NULL;
    int         rc,
                ret;

    // Get a buffer to receive the data
    buffobj = GetBufferObj(gBufferSize);

    ret = 0;

    if (gProtocol == IPPROTO_TCP)
    {
        rc = recv(
                sockobj->s,
                buffobj->buf,
                buffobj->buflen,
                0
                );
    }
    else 
    {
        rc = recvfrom(
                sockobj->s,
                buffobj->buf,
                buffobj->buflen,
                0,
                (SOCKADDR *)&buffobj->addr,
               &buffobj->addrlen
                );
    }
    if (rc == SOCKET_ERROR)
    {
        if (WSAGetLastError() != WSAEWOULDBLOCK)
        {
            // Socket connection has failed, close the socket
            fprintf(stderr, "recv(from) failed: %d\n", WSAGetLastError());

            closesocket(sockobj->s);
            sockobj->s = INVALID_SOCKET;

            ret = -1;
        }
        else
        {
            ret = WSAEWOULDBLOCK;
        }
        FreeBufferObj(buffobj);
    }
    else if (rc == 0)
    {
        // Graceful close
        if (gProtocol == IPPROTO_TCP)
        {
            FreeBufferObj(buffobj);
        }
        else
        {
            // Always enqueue the zero byte datagrams for UDP
            buffobj->buflen = 0;
            EnqueueBufferObj(sockobj, buffobj, FALSE);
        }

        // Set the socket object to closing
        sockobj->closing = TRUE;

        if (sockobj->pending == NULL)
        {
            // If no sends are pending, close the socket for good
            closesocket(sockobj->s);
            sockobj->s = INVALID_SOCKET;

            ret = -1;
        }
        else
        {
            // Sends are pending, just return
            ret = 0;
        }
    }
    else
    {
        // Read data, updated the counters and enqueue the buffer for sending
        InterlockedExchangeAdd(&gBytesRead, rc);
        InterlockedExchangeAdd(&gBytesReadLast, rc);

        buffobj->buflen = rc;
        EnqueueBufferObj(sockobj, buffobj, FALSE);

        ret = 1;
    }
    return ret;
}
BOOL HandleIO(PBUFFER_OBJ pBuffer)
{
    PSOCKET_OBJ pSocket = pBuffer->pSocket; // 从BUFFER_OBJ对象中提取SOCKET_OBJ对象指针,为的是方便引用
    pSocket->nOutstandingOps --;

    DWORD dwTrans;   //用于取得实际传输字节的数量
    DWORD dwFlags;   //用于取得完成状态
    // 获取重叠操作结果 true 重叠操作成功  false 套接字上有错误发生
    BOOL bRet = ::WSAGetOverlappedResult(pSocket->s, &pBuffer->ol, &dwTrans, FALSE, &dwFlags);
    if(!bRet)
    {
        // 在此套节字上有错误发生,因此,关闭套节字,移除此缓冲区对象。
        // 如果没有其它抛出的I/O请求了,释放此缓冲区对象,否则,等待此套节字上的其它I/O也完成
        if(pSocket->s != INVALID_SOCKET)
        {
            ::closesocket(pSocket->s);
            pSocket->s = INVALID_SOCKET;
        }

        if(pSocket->nOutstandingOps == 0)
            FreeSocketObj(pSocket);

        FreeBufferObj(pBuffer);
        return FALSE;
    }

    // 没有错误发生,处理已完成的I/O
    switch(pBuffer->nOperation)
    {
    case OP_ACCEPT:	// 接收到一个新的连接,并接收到了对方发来的第一个封包
    {
        // 为新客户创建一个SOCKET_OBJ对象
        PSOCKET_OBJ pClient = GetSocketObj(pBuffer->sAccept);

        // 为收发数据创建一个BUFFER_OBJ对象,这个对象会在套节字出错或者关闭时释放
        PBUFFER_OBJ pSend = GetBufferObj(pClient, BUFFER_SIZE);
        if(pSend == NULL)
        {
            printf(" Too much connections! \n");
            FreeSocketObj(pClient);
            return FALSE;
        }

        SOCKADDR_IN LocalSockAddr;
        SOCKADDR_IN RemoteSockAddr;
        int nLocalLen, nRmoteLen;
        LPSOCKADDR pLocalAddr, pRemoteAddr;
        g_pfnGetAcceptExSockaddrs(
            pBuffer->buff,
            pBuffer->nLen - ((sizeof(sockaddr_in) + 16) * 2),
            sizeof(sockaddr_in) + 16,
            sizeof(sockaddr_in) + 16,
            (SOCKADDR **)&pLocalAddr,
            &nLocalLen,
            (SOCKADDR **)&pRemoteAddr,
            &nRmoteLen);

        memcpy(&LocalSockAddr, pLocalAddr, nLocalLen);
        memcpy(&RemoteSockAddr, pRemoteAddr, nRmoteLen);
        printf("Local Accepted client:%s:%d\n", inet_ntoa(LocalSockAddr.sin_addr), ntohs(LocalSockAddr.sin_port));
        printf("Remote Accepted client:%s:%d\n", inet_ntoa(RemoteSockAddr.sin_addr), ntohs(RemoteSockAddr.sin_port));

        RebuildArray();

        // 将数据复制到发送缓冲区
        pSend->nLen = dwTrans;
        memcpy(pSend->buff, pBuffer->buff, dwTrans);

        // 投递此发送I/O(将数据回显给客户)
        if(!PostSend(pSend))
        {
            // 万一出错的话,释放上面刚申请的两个对象
            FreeSocketObj(pSocket);
            FreeBufferObj(pSend);
            return FALSE;
        }
        // 继续投递接受I/O
        PostAccept(pBuffer);
    }
    break;
    case OP_RECV:	// 接收数据完成
    {
        if(dwTrans > 0)
        {
            // 创建一个缓冲区,以发送数据。这里就使用原来的缓冲区
            PBUFFER_OBJ pSend = pBuffer;
            printf("收到:%s\r\n", pBuffer->buff);
            pSend->nLen = dwTrans;

            // 投递发送I/O(将数据回显给客户)
            PostSend(pSend);
        }
        else	// 套节字关闭
        {

            // 必须先关闭套节字,以便在此套节字上投递的其它I/O也返回
            if(pSocket->s != INVALID_SOCKET)
            {
                ::closesocket(pSocket->s);
                pSocket->s = INVALID_SOCKET;
            }

            if(pSocket->nOutstandingOps == 0)
                FreeSocketObj(pSocket);

            FreeBufferObj(pBuffer);
            return FALSE;
        }
    }
    break;
    case OP_SEND:		// 发送数据完成
    {
        if(dwTrans > 0)
        {
            // 继续使用这个缓冲区投递接收数据的请求
            pBuffer->nLen = BUFFER_SIZE;
            PostRecv(pBuffer);
        }
        else	// 套节字关闭
        {
            // 同样,要先关闭套节字
            if(pSocket->s != INVALID_SOCKET)
            {
                ::closesocket(pSocket->s);
                pSocket->s = INVALID_SOCKET;
            }

            if(pSocket->nOutstandingOps == 0)
                FreeSocketObj(pSocket);

            FreeBufferObj(pBuffer);
            return FALSE;
        }
    }
    break;
    }
    return TRUE;
}