/*
Each socket has two write buffer list, high priority and low priority.
1. send high list as far as possible.
2. If high list is empty, try to send low list.
3. If low list head is uncomplete (send a part before), move the head of low list to empty high list (call raise_uncomplete) .
4. If two lists are both empty, turn off the event. (call check_close)
*/
static int
send_buffer(struct socket_server *ss, struct socket *s, struct socket_message *result) {
assert(!list_uncomplete(&s->low));
// step 1
if (send_list(ss,s,&s->high,result) == SOCKET_CLOSE) {
return SOCKET_CLOSE;
}
if (s->high.head == NULL) {
// step 2
if (s->low.head != NULL) {
if (send_list(ss,s,&s->low,result) == SOCKET_CLOSE) {
return SOCKET_CLOSE;
}
// step 3
if (list_uncomplete(&s->low)) {
raise_uncomplete(s);
}
} else {
// step 4
sp_write(ss->event_fd, s->fd, s, false);
if (s->type == SOCKET_TYPE_HALFCLOSE) {
force_close(ss, s, result);
return SOCKET_CLOSE;
}
}
}
return -1;
}
static int
close_socket(struct socket_server *ss, struct request_close *request, struct socket_message *result) {
int id = request->id;
struct socket * s = &ss->slot[id % MAX_SOCKET];
if (s->type == SOCKET_TYPE_INVALID || s->id != id) {
result->id = id;
result->opaque = request->opaque;
result->ud = 0;
result->data = NULL;
return SOCKET_CLOSE;
}
if (s->head) {
int type = send_buffer(ss,s,result); // 关闭socket的时候 如果有未发送的数据 先发送出去
if (type != -1)
return type;
}
if (s->head == NULL) {
force_close(ss,s,result);
result->id = id;
result->opaque = request->opaque;
return SOCKET_CLOSE;
}
s->type = SOCKET_TYPE_HALFCLOSE;
return -1;
}
// 尝试连接中的套接字可写事件 可能失败
static int
report_connect(struct socket_server *ss, struct socket *s, struct socket_message *result) {
int error;
socklen_t len = sizeof(error);
int code = getsockopt(s->fd, SOL_SOCKET, SO_ERROR, &error, &len);
if (code < 0 || error) {
force_close(ss,s, result);
return SOCKET_ERROR;
} else {
s->type = SOCKET_TYPE_CONNECTED;
result->opaque = s->opaque;
result->id = s->id;
result->ud = 0;
sp_write(ss->event_fd, s->fd, s, false); // 取消关注可写事件
union sockaddr_all u;
socklen_t slen = sizeof(u);
if (getpeername(s->fd, &u.s, &slen) == 0) {
void * sin_addr = (u.s.sa_family == AF_INET) ? (void*)&u.v4.sin_addr : (void *)&u.v6.sin6_addr;
if (inet_ntop(u.s.sa_family, sin_addr, ss->buffer, sizeof(ss->buffer))) {
result->data = ss->buffer;
return SOCKET_OPEN;
}
}
result->data = NULL;
return SOCKET_OPEN;
}
}
static int
close_socket(struct socket_server *ss, struct request_close *request, struct socket_message *result) {
int id = request->id;
struct socket * s = &ss->slot[HASH_ID(id)];
if (s->type == SOCKET_TYPE_INVALID || s->id != id) {
result->id = id;
result->opaque = request->opaque;
result->ud = 0;
result->data = NULL;
return SOCKET_CLOSE;
}
if (!send_buffer_empty(s)) {
int type = send_buffer(ss,s,result);
if (type != -1)
return type;
}
if (send_buffer_empty(s)) {
force_close(ss,s,result);
result->id = id;
result->opaque = request->opaque;
return SOCKET_CLOSE;
}
s->type = SOCKET_TYPE_HALFCLOSE;
return -1;
}
static int
send_list(struct socket_server *ss, struct socket *s, struct wb_list *list, struct socket_message *result) {
while (list->head) {
struct write_buffer * tmp = list->head;
for (;;) {
int sz = write(s->fd, tmp->ptr, tmp->sz);
if (sz < 0) {
switch(errno) {
case EINTR:
continue;
case EAGAIN:
return -1;
}
force_close(ss,s, result);
return SOCKET_CLOSE;
}
s->wb_size -= sz;
if (sz != tmp->sz) {
tmp->ptr += sz;
tmp->sz -= sz;
return -1;
}
break;
}
list->head = tmp->next;
FREE(tmp->buffer);
FREE(tmp);
}
list->tail = NULL;
return -1;
}
// return -1 (ignore) when error
static int
forward_message(struct socket_server *ss, struct socket *s, union socket_message * result) {
int sz = s->size;
char * buffer = MALLOC(sz);
int n = (int)read(s->fd, buffer, sz);
if (n<0) {
FREE(buffer);
switch(errno) {
case EINTR:
break;
case EAGAIN:
fprintf(stderr, "socket-server: EAGAIN capture.\n");
break;
default:
// close when error
result->error.id = s->id;
result->error.session = 0;
force_close(ss, s);
return SOCKET_ERROR;
}
return -1;
}
if (n==0) {
FREE(buffer);
result->close.id = s->id;
force_close(ss, s);
return SOCKET_CLOSE;
}
if (s->type == SOCKET_TYPE_HALFCLOSE) {
// discard recv data
return -1;
}
if (n == sz) {
s->size *= 2;
} else if (sz > MIN_READ_BUFFER && n*2 < sz) {
s->size /= 2;
}
result->data.id = s->id;
result->data.size = n;
result->data.data = buffer;
return SOCKET_DATA;
}
// return -1 (ignore) when error
static int
forward_message_tcp(struct socket_server *ss, struct socket *s, struct socket_message * result) {
int sz = s->p.size;
char * buffer = MALLOC(sz);
int n = (int)read(s->fd, buffer, sz);
if (n<0) {
FREE(buffer);
switch(errno) {
case EINTR:
break;
case AGAIN_WOULDBLOCK:
fprintf(stderr, "socket-server: EAGAIN capture.\n");
break;
default:
// close when error
force_close(ss, s, result);
result->data = strerror(errno);
return SOCKET_ERROR;
}
return -1;
}
if (n==0) {
FREE(buffer);
force_close(ss, s, result);
return SOCKET_CLOSE;
}
if (s->type == SOCKET_TYPE_HALFCLOSE) {
// discard recv data
FREE(buffer);
return -1;
}
if (n == sz) {
s->p.size *= 2;
} else if (sz > MIN_READ_BUFFER && n*2 < sz) {
s->p.size /= 2;
}
result->opaque = s->opaque;
result->id = s->id;
result->ud = n;
result->data = buffer;
return SOCKET_DATA;
}
// 可写事件到来，从应用层发送缓冲区取数据发送
int socket_proxy::send_buffer()
{
while (this->head)
{
struct write_buffer * tmp = this->head;
for (;;)
{
int sz = write(this->fd, tmp->ptr, tmp->sz);
if (sz < 0)
{
switch(errno)
{
case EINTR:
continue;
case EAGAIN: // 内核发送缓冲区已满
return -1;
}
force_close();
return SOCKET_CLOSE;
}
this->wb_size -= sz; //发送缓冲区未发的字节数需要更新
if (sz != tmp->sz)
{ // 未将该块缓冲区完全发送出去
tmp->ptr += sz; // 该块缓冲区未发送数据首地址更新
tmp->sz -= sz; // 当前块未发送字节数更新
return -1;
}
break;
}
this->head = tmp->next;
FREE(tmp->buffer);
FREE(tmp);
}
this->tail = NULL;
sp_write(this->ss->event_fd,this->fd, this, false);
if (this->type == SOCKET_TYPE_HALFCLOSE)
{
force_close();
return SOCKET_CLOSE;
}
return -1;
}
void
socket_server_release(struct socket_server *ss) {
int i;
for (i=0;i<MAX_SOCKET;i++) {
force_close(ss, &ss->slot[i]);
}
close(ss->client_fd);
close(ss->server_fd);
sp_release(ss->event_fd);
FREE(ss);
}
static int
send_buffer(struct socket_server *ss, struct socket *s, struct socket_message *result) {
while (s->head) {
struct write_buffer * tmp = s->head;
for (;;) {
int sz = write(s->fd, tmp->ptr, tmp->sz);
if (sz < 0) {
switch(errno) {
case EINTR:
continue;
case EAGAIN:
return -1;
}
force_close(ss,s, result);
return SOCKET_CLOSE;
}
s->wb_size -= sz;
if (sz != tmp->sz) {
tmp->ptr += sz;
tmp->sz -= sz;
return -1;
}
break;
}
s->head = tmp->next;
FREE(tmp->buffer);
FREE(tmp);
}
s->tail = NULL;
sp_write(ss->event_fd, s->fd, s, false); // 取消关注
// 这里 在关闭的时候 处理不够完善 因为第二次发送的时候直接关闭了socket 没有继续发送完毕 所以skynet不适合大流量的网络应用
if (s->type == SOCKET_TYPE_HALFCLOSE) {
force_close(ss, s, result);
return SOCKET_CLOSE;
}
return -1;
}
void socket_server::release()
{
struct socket_message dummy;
for (int i = 0; i < MAX_SOCKET; i++) {
struct socket * s = &slot[i];
if (s->type != SOCKET_TYPE_RESERVE) {
force_close(s, &dummy);
}
}
close(sendctrl_fd);
close(recvctrl_fd);
event_fd.release();
}
static int
send_socket(struct socket_server *ss, struct request_send * request, struct socket_message *result) {
int id = request->id;
struct socket * s = &ss->slot[id % MAX_SOCKET];
if (s->type == SOCKET_TYPE_INVALID || s->id != id
|| s->type == SOCKET_TYPE_HALFCLOSE
|| s->type == SOCKET_TYPE_PACCEPT) {
FREE(request->buffer);
return -1;
}
assert(s->type != SOCKET_TYPE_PLISTEN && s->type != SOCKET_TYPE_LISTEN);
if (s->head == NULL) {
int n = write(s->fd, request->buffer, request->sz);
if (n<0) {
switch(errno) {
case EINTR:
case EAGAIN:
n = 0;
break;
default:
fprintf(stderr, "socket-server: write to %d (fd=%d) error.",id,s->fd);
force_close(ss,s,result);
return SOCKET_CLOSE;
}
}
if (n == request->sz) {
FREE(request->buffer);
return -1;
}
struct write_buffer * buf = MALLOC(sizeof(*buf));
buf->next = NULL;
buf->ptr = request->buffer+n;
buf->sz = request->sz - n;
buf->buffer = request->buffer;
s->head = s->tail = buf;
sp_write(ss->event_fd, s->fd, s, true);
} else {
struct write_buffer * buf = MALLOC(sizeof(*buf));
buf->ptr = request->buffer;
buf->buffer = request->buffer;
buf->sz = request->sz;
assert(s->tail != NULL);
assert(s->tail->next == NULL);
buf->next = s->tail->next;
s->tail->next = buf;
s->tail = buf;
}
return -1;
}
/*
When send a package , we can assign the priority : PRIORITY_HIGH or PRIORITY_LOW
If socket buffer is empty, write to fd directly.
If write a part, append the rest part to high list. (Even priority is PRIORITY_LOW)
Else append package to high (PRIORITY_HIGH) or low (PRIORITY_LOW) list.
*/
static int
send_socket(struct socket_server *ss, struct request_send * request, struct socket_message *result, int priority)
{
int id = request->id;
struct socket * s = &ss->slot[id % MAX_SOCKET];
if (s->type == SOCKET_TYPE_INVALID || s->id != id
|| s->type == SOCKET_TYPE_HALFCLOSE
|| s->type == SOCKET_TYPE_PACCEPT)
{
FREE(request->buffer);
return -1;
}
assert(s->type != SOCKET_TYPE_PLISTEN && s->type != SOCKET_TYPE_LISTEN);
if (send_buffer_empty(s))
{
int n = write(s->fd, request->buffer, request->sz);
if (n < 0)
{
switch (errno)
{
case EINTR:
case EAGAIN:
n = 0;
break;
default:
fprintf(stderr, "socket-server: write to %d (fd=%d) error.", id, s->fd);
force_close(ss, s, result);
return SOCKET_CLOSE;
}
}
if (n == request->sz)
{
FREE(request->buffer);
return -1;
}
append_sendbuffer(s, request, n); // add to high priority list, even priority == PRIORITY_LOW
sp_write(ss->event_fd, s->fd, s, true);
}
else
{
if (priority == PRIORITY_LOW)
{
append_sendbuffer_low(s, request);
}
else
{
append_sendbuffer(s, request, 0);
}
}
return -1;
}
void
socket_server_release(struct socket_server *ss) {
int i;
struct socket_message dummy;
for (i=0;i<MAX_SOCKET;i++) {
struct socket *s = &ss->slot[i];
if (s->type != SOCKET_TYPE_RESERVE) {
force_close(ss, s , &dummy);
}
}
close(ss->sendctrl_fd);
close(ss->recvctrl_fd);
sp_release(ss->event_fd);
FREE(ss);
}
//执行send事件
int socket_proxy::send_socket(char *buffer, int sz)
{
if (this->type == SOCKET_TYPE_INVALID
|| this->type == SOCKET_TYPE_HALFCLOSE
|| this->type == SOCKET_TYPE_PACCEPT )
{
//FREE(buffer);
return -1;
}
assert(this->type != SOCKET_TYPE_PLISTEN && this->type != SOCKET_TYPE_LISTEN);
char * buf = (char *)malloc(sz);
memcpy(buf, buffer, sz);
if (this->head == NULL)
{
int n = write(this->fd, buf, sz);
//printf("n=%d, errno = %d", n, errno);
if (n<0)
{
switch(errno)
{
case EINTR:
case EAGAIN:
n = 0;
break;
default:
//fprintf(stderr, "socket-server: write to %d (fd=%d) error.",id,s->fd);
force_close();
return SOCKET_CLOSE;
}
}
if (n == sz)
{
FREE(buf);
return -1;
}
this->append_sendbuffer(buf, sz, n);
sp_write(this->ss->event_fd, this->fd, this, true);
}
else
{
printf("append");
this->append_sendbuffer(buf, sz, 0);
}
return -1;
}
// 发送tcp数据
// 正常返回 -1
// 失败返回 SOCKET_CLOSE
static int
send_list_tcp(struct socket_server *ss, struct socket *s, struct wb_list *list, struct socket_message *result) {
while (list->head) {
struct write_buffer * tmp = list->head;
for (;;) {
// 往套接字写入数据
int sz = write(s->fd, tmp->ptr, tmp->sz);
if (sz < 0) {
switch(errno) {
case EINTR:
// 信用中断，循环等待
continue;
case AGAIN_WOULDBLOCK:
// 写缓冲区满，直接返回
return -1;
}
// 关闭socket
force_close(ss,s, result);
return SOCKET_CLOSE;
}
s->wb_size -= sz;
if (sz != tmp->sz) {
// 如果缓冲区数据没有发送完，直接返回-1
// tmp缓冲区仍然在缓冲区链表中，等待下一次发送
tmp->ptr += sz;
tmp->sz -= sz;
return -1;
}
break;
}
// head指针偏移到tmp下一个缓冲区
list->head = tmp->next;
// 释放tmp缓冲区
write_buffer_free(ss,tmp);
}
list->tail = NULL;
return -1;
}
// 从套接字接受数据，返回SOCKET_UDP
static int
forward_message_udp(struct socket_server *ss, struct socket *s, struct socket_message * result) {
union sockaddr_all sa;
socklen_t slen = sizeof(sa);
int n = recvfrom(s->fd, ss->udpbuffer,MAX_UDP_PACKAGE,0,&sa.s,&slen);
if (n<0) {
switch(errno) {
case EINTR:
case AGAIN_WOULDBLOCK:
break;
default:
// close when error
force_close(ss, s, result);
result->data = strerror(errno);
return SOCKET_ERROR;
}
return -1;
}
uint8_t * data;
if (slen == sizeof(sa.v4)) {
if (s->protocol != PROTOCOL_UDP)
return -1;
data = MALLOC(n + 1 + 2 + 4);
gen_udp_address(PROTOCOL_UDP, &sa, data + n);
} else {
if (s->protocol != PROTOCOL_UDPv6)
return -1;
data = MALLOC(n + 1 + 2 + 16);
gen_udp_address(PROTOCOL_UDPv6, &sa, data + n);
}
memcpy(data, ss->udpbuffer, n);
result->opaque = s->opaque;
result->id = s->id;
result->ud = n;
result->data = (char *)data;
return SOCKET_UDP;
}
static int
send_socket(struct socket_server *ss, struct request_send * request, struct socket_message *result) {
int id = request->id;
struct socket * s = &ss->slot[id % MAX_SOCKET];
if (s->type == SOCKET_TYPE_INVALID || s->id != id
|| s->type == SOCKET_TYPE_HALFCLOSE
|| s->type == SOCKET_TYPE_PACCEPT) {
FREE(request->buffer);
return -1;
}
assert(s->type != SOCKET_TYPE_PLISTEN && s->type != SOCKET_TYPE_LISTEN);
// 应用层缓冲区 没有数据直接发送
if (s->head == NULL) {
int n = write(s->fd, request->buffer, request->sz);
if (n<0) {
switch(errno) {
case EINTR:
case EAGAIN: // 内核缓冲区满了
n = 0;
break;
default:
fprintf(stderr, "socket-server: write to %d (fd=%d) error.",id,s->fd);
force_close(ss,s,result);
return SOCKET_CLOSE;
}
}
// 可以把数据拷贝到内核缓冲区中
if (n == request->sz) {
FREE(request->buffer);
return -1;
}
// 将未发送的部分添加到应用层缓冲区中 关注可写事件
append_sendbuffer(s, request, n);
sp_write(ss->event_fd, s->fd, s, true);
} else { // 将未发送的数据添加到应用层缓冲区 应用层缓冲区已经有数据了
append_sendbuffer(s, request, 0);
}
return -1;
}
int socket_server::forward_message_tcp(struct socket * s, struct socket_message * result)
{
int sz = s->size;
char * buffer = (char *)malloc(sz);
int n = (int)read(s->fd, buffer, sz);
if (n < 0) {
free(buffer);
switch(errno) {
case EINTR:
break;
case EAGAIN:
fprintf(stderr, "socket-server: EAGAIN capture.\n");
break;
default:
force_close(s, result);
}
return -1;
}
if (n == 0) {
free(buffer);
return -1;
}
if (s->type == SOCKET_TYPE_HALFCLOSE) {
free(buffer);
return -1;
}
if (n == sz) {
s->size *= 2;
} else if (sz > MIN_READ_BUFFER && n*2 < sz) {
s->size /= 2;
}
result->opaque = s->opaque;
result->id = s->id;
result->ud = n;
result->data = buffer;
return SOCKET_DATA;
}
static int
close_socket(struct socket_server *ss, struct request_close *request, union socket_message *result) {
int id = request->id;
struct socket * s = &ss->slot[id % MAX_SOCKET];
if (s->type == SOCKET_TYPE_INVALID || s->id != id) {
result->close.id = id;
return SOCKET_CLOSE;
}
if (s->head) {
int type = send_buffer(ss,s,result);
if (type != -1)
return type;
}
if (s->head == NULL) {
force_close(ss,s);
result->close.id = id;
return SOCKET_CLOSE;
}
s->type = SOCKET_TYPE_HALFCLOSE;
return -1;
}
// 连接逻辑
// 正常返回SOCKET_OPEN
static int
report_connect(struct socket_server *ss, struct socket *s, struct socket_message *result) {
int error;
socklen_t len = sizeof(error);
int code = getsockopt(s->fd, SOL_SOCKET, SO_ERROR, &error, &len);
if (code < 0 || error) {
force_close(ss,s, result);
if (code >= 0)
result->data = strerror(error);
else
result->data = strerror(errno);
return SOCKET_ERROR;
} else {
// 如果无错误，修改socket为CONNECTED类型
s->type = SOCKET_TYPE_CONNECTED;
result->opaque = s->opaque;
result->id = s->id;
result->ud = 0;
if (send_buffer_empty(s)) {
// 如果链表缓冲区为空，关闭事件循环中fd的可写权限
sp_write(ss->event_fd, s->fd, s, false);
}
union sockaddr_all u;
socklen_t slen = sizeof(u);
if (getpeername(s->fd, &u.s, &slen) == 0) {
// 连接对等方ip地址写入result->data
void * sin_addr = (u.s.sa_family == AF_INET) ? (void*)&u.v4.sin_addr : (void *)&u.v6.sin6_addr;
if (inet_ntop(u.s.sa_family, sin_addr, ss->buffer, sizeof(ss->buffer))) {
result->data = ss->buffer;
return SOCKET_OPEN;
}
}
result->data = NULL;
return SOCKET_OPEN;
}
}
int socket_server::send_socket(struct request_send * request, struct socket_message * result, int priority)
{
int id = request->id;
struct socket *s = &slot[HASH_ID(id)];
if (s->type == SOCKET_TYPE_INVALID
|| s->id != id
|| s->type == SOCKET_TYPE_HALFCLOSE
|| s->type == SOCKET_TYPE_PACCEPT) {
return -1;
}
assert(s->type != SOCKET_TYPE_PLISTEN && s->type != SOCKET_TYPE_LISTEN);
if (send_buffer_empty(s) && s->type == SOCKET_TYPE_CONNECTED) {
if (s->protocol == PROTOCOL_TCP) {
int n = write(s->fd, request->buffer, request->sz);
if (n < 0) {
switch (errno) {
case EINTR:
case EAGAIN:
n = 0;
break;
default:
fprintf(stderr, "socket-server: write to %d (fd=%d) error: %s.\n", id, s->fd, strerror(errno));
force_close(s, result);
return SOCKET_CLOSE;
}
}
if (n == request->sz)
{
return -1;
}
event_fd.write(s->fd, s, true);
}
}
return -1;
}
/*
When send a package , we can assign the priority : PRIORITY_HIGH or PRIORITY_LOW
If socket buffer is empty, write to fd directly.
If write a part, append the rest part to high list. (Even priority is PRIORITY_LOW)
Else append package to high (PRIORITY_HIGH) or low (PRIORITY_LOW) list.
*/
static int
send_socket(struct socket_server *ss, struct request_send * request, struct socket_message *result, int priority, const uint8_t *udp_address) {
int id = request->id;
struct socket * s = &ss->slot[HASH_ID(id)];
struct send_object so;
send_object_init(ss, &so, request->buffer, request->sz);
if (s->type == SOCKET_TYPE_INVALID || s->id != id
|| s->type == SOCKET_TYPE_HALFCLOSE
|| s->type == SOCKET_TYPE_PACCEPT) {
so.free_func(request->buffer);
return -1;
}
if (s->type == SOCKET_TYPE_PLISTEN || s->type == SOCKET_TYPE_LISTEN) {
fprintf(stderr, "socket-server: write to listen fd %d.\n", id);
so.free_func(request->buffer);
return -1;
}
if (send_buffer_empty(s) && s->type == SOCKET_TYPE_CONNECTED) {
// 如果已连接且发送缓冲区为空
if (s->protocol == PROTOCOL_TCP) {
// 直接写入套接字
int n = write(s->fd, so.buffer, so.sz);
if (n<0) {
switch(errno) {
case EINTR:
case AGAIN_WOULDBLOCK:
// 信号中断或者缓冲区写满
n = 0;
break;
default:
fprintf(stderr, "socket-server: write to %d (fd=%d) error :%s.\n",id,s->fd,strerror(errno));
force_close(ss,s,result);
so.free_func(request->buffer);
return SOCKET_CLOSE;
}
}
if (n == so.sz) {
// 如果全部发送完，释放数据缓冲区，返回-1
so.free_func(request->buffer);
return -1;
}
// 如果没有发送完，剩余数据放到高优先级写缓冲区链表
append_sendbuffer(ss, s, request, n); // add to high priority list, even priority == PRIORITY_LOW
} else {
// udp
if (udp_address == NULL) {
udp_address = s->p.udp_address;
}
union sockaddr_all sa;
socklen_t sasz = udp_socket_address(s, udp_address, &sa);
int n = sendto(s->fd, so.buffer, so.sz, 0, &sa.s, sasz);
if (n != so.sz) {
append_sendbuffer_udp(ss,s,priority,request,udp_address);
} else {
so.free_func(request->buffer);
return -1;
}
}
// 打开事件循环中fd的可写权限
sp_write(ss->event_fd, s->fd, s, true);
} else {
if (s->protocol == PROTOCOL_TCP) {
if (priority == PRIORITY_LOW) {
append_sendbuffer_low(ss, s, request);
} else {
append_sendbuffer(ss, s, request, 0);
}
} else {
if (udp_address == NULL) {
udp_address = s->p.udp_address;
}
append_sendbuffer_udp(ss,s,priority,request,udp_address);
}
}
return -1;
}
zk::~zk() {
force_close();
if(logfilep_) {
fclose(logfilep_);
}
};