static int tcpconn_recv(int s, void *buf, size_t len, int64_t deadline) {
struct tcpconn *conn = bsockdata(s, tcpconn_type);
if(dill_slow(!conn)) return -1;
/* If there's enough data in the buffer use it. */
if(dill_fast(len <= conn->ilen)) {
memcpy(buf, conn->ibuf, len);
memmove(conn->ibuf, conn->ibuf + len, conn->ilen - len);
conn->ilen -= len;
return 0;
}
/* If there's not enough data in the buffer, yet the message is small,
read whole chunk of data to avoid excessive system calls. */
if(len <= BATCHSIZE) {
/* First try a non-blocking batch read. */
size_t recvd = BATCHSIZE - conn->ilen;
/* TODO: Make sure that deadline of 0 works as intended! */
dsrecv(conn->fd, conn->ibuf + conn->ilen, &recvd, 0);
conn->ilen += recvd;
if(dill_fast(len <= conn->ilen)) {
memcpy(buf, conn->ibuf, len);
memmove(conn->ibuf, conn->ibuf + len, conn->ilen - len);
conn->ilen -= len;
return 0;
}
}
/* Either message is big or we weren't able to read it in non-blocking
fashion. In both cases let's try to read the missing data directly
to the user's buffer. If the operation fails we'll copy the data that
was already read to the input buffer. To make sure that the operation
is atomic even in face of memory shortage let's resize the buffer first
do receiving second. */
if(dill_slow(len > conn->ilen)) {
char *newbuf = realloc(conn->ibuf, len);
if(dill_slow(!newbuf)) {errno = ENOMEM; return -1;}
conn->ibuf = newbuf;
conn->icap = len;
}
/* Read the missing part directly into user's buffer. */
size_t recvd = len - conn->ilen;
int rc = dsrecv(conn->fd, ((char*)buf) + conn->ilen, &recvd, deadline);
/* If succesfull copy the first part of the message from input buffer. */
if(dill_fast(rc == 0)) {
memcpy(buf, conn->ibuf, conn->ilen);
conn->ilen = 0;
return 0;
}
/* If not successful, store the aready received data into input buffer. */
int err = errno;
memcpy(conn->ibuf + conn->ilen, ((char*)buf) + conn->ilen, recvd);
conn->ibuf += recvd;
errno = err;
return -1;
}
static int tcpconn_send(int s, const void *buf, size_t len, int64_t deadline) {
struct tcpconn *conn = bsockdata(s, tcpconn_type);
if(dill_slow(!conn)) return -1;
/* If the data fit into the buffer without exceeding the BATCHSIZE
we can store it and be done with it. That way sending a lot of little
messages won't result in lot of system calls. */
if(dill_fast(conn->olen + len <= BATCHSIZE)) {
memcpy(conn->obuf + conn->olen, buf, len);
conn->olen += len;
return 0;
}
/* Message won't fit into buffer, so let's try to flush the buffer first. */
int rc = tcpconn_flush(s, deadline);
if(dill_slow(rc < 0)) return -1;
/* Try to fit the message into buffer again. */
if(dill_fast(len <= BATCHSIZE)) {
dill_assert(conn->olen == 0);
memcpy(conn->obuf, buf, len);
conn->olen = len;
return 0;
}
/* Resize the buffer so that it can hold the message if needed. By doing
this in advance we won't be faced with ugly non-atomic case when only
part of the message is sent to the network but there's no buffer space
to buffer the remaining data. */
if(dill_slow(len > conn->olen)) {
char *newbuf = realloc(conn->obuf, len);
if(dill_slow(!newbuf)) {errno = ENOMEM; return -1;}
conn->obuf = newbuf;
conn->ocap = len;
}
/* Try to send the message directly from the user's buffer. */
size_t sent = len;
rc = dssend(conn->fd, buf, &sent, deadline);
if(dill_fast(rc >= 0)) return 0;
int err = errno;
/* Store the remainder of the message into the output buffer. */
dill_assert(conn->olen == 0);
memcpy(conn->obuf, ((char*)buf) + sent, len - sent);
conn->olen = len - sent;
/* ETIMEDOUT can be ignored. The send operation have already completed
successfully so there's no point in reporting the error. */
if(dill_slow(err == ETIMEDOUT)) return 0;
/* Other errors, such as ECONNRESET or ECANCELED are fatal so we can report
them straight away even though message data were buffered. */
errno = err;
return -1;
}
int64_t now(void) {
#if (defined __GNUC__ || defined __clang__) && \
(defined __i386__ || defined __x86_64__)
/* Get the timestamp counter. This is time since startup, expressed in CPU
cycles. Unlike gettimeofday() or similar function, it's extremely fast -
it takes only few CPU cycles to evaluate. */
uint32_t low;
uint32_t high;
__asm__ volatile("rdtsc" : "=a" (low), "=d" (high));
int64_t tsc = (int64_t)((uint64_t)high << 32 | low);
/* These global variables are used to hold the last seen timestamp counter
and last seen time measurement. We'll initilise them the first time
this function is called. */
static int64_t last_tsc = -1;
static int64_t last_now = -1;
if(dill_slow(last_tsc < 0)) {
last_tsc = tsc;
last_now = dill_now();
}
/* If TSC haven't jumped back or progressed more than 1/2 ms, we can use
the cached time value. */
if(dill_fast(tsc - last_tsc <= (DILL_CLOCK_PRECISION / 2) &&
tsc >= last_tsc))
return last_now;
/* It's more than 1/2 ms since we've last measured the time.
We'll do a new measurement now. */
last_tsc = tsc;
last_now = dill_now();
return last_now;
#else
return dill_now();
#endif
}
int udp_sendl_(struct msock_vfs *mvfs, const struct ipaddr *addr,
struct iolist *first, struct iolist *last) {
struct udp_sock *obj = dill_cont(mvfs, struct udp_sock, mvfs);
/* If no destination IP address is provided, fall back to the stored one. */
const struct ipaddr *dstaddr = addr;
if(!dstaddr) {
if(dill_slow(!obj->hasremote)) {errno = EINVAL; return -1;}
dstaddr = &obj->remote;
}
struct msghdr hdr;
memset(&hdr, 0, sizeof(hdr));
hdr.msg_name = (void*)ipaddr_sockaddr(dstaddr);
hdr.msg_namelen = ipaddr_len(dstaddr);
/* Make a local iovec array. */
/* TODO: This is dangerous, it may cause stack overflow.
There should probably be a on-heap per-socket buffer for that. */
size_t niov;
int rc = iol_check(first, last, &niov, NULL);
if(dill_slow(rc < 0)) return -1;
struct iovec iov[niov];
iol_toiov(first, iov);
hdr.msg_iov = (struct iovec*)iov;
hdr.msg_iovlen = niov;
ssize_t sz = sendmsg(obj->fd, &hdr, 0);
if(dill_fast(sz >= 0)) return 0;
if(errno == EAGAIN || errno == EWOULDBLOCK) return 0;
return -1;
}
struct dill_ctx *dill_getctx_(void) {
int rc = pthread_once(&dill_keyonce, dill_makekey);
dill_assert(rc == 0);
struct dill_ctx *ctx = pthread_getspecific(dill_key);
if(dill_fast(ctx)) return ctx;
ctx = malloc(sizeof(struct dill_ctx));
dill_assert(ctx);
rc = dill_ctx_now_init(&ctx->now);
dill_assert(rc == 0);
rc = dill_ctx_cr_init(&ctx->cr);
dill_assert(rc == 0);
rc = dill_ctx_handle_init(&ctx->handle);
dill_assert(rc == 0);
rc = dill_ctx_stack_init(&ctx->stack);
dill_assert(rc == 0);
rc = dill_ctx_pollset_init(&ctx->pollset);
dill_assert(rc == 0);
if(dill_ismain()) {
dill_main = ctx;
rc = atexit(dill_ctx_atexit);
dill_assert(rc == 0);
}
rc = pthread_setspecific(dill_key, ctx);
dill_assert(rc == 0);
return ctx;
}
static int sf_send(int s, const void *buf, size_t len, int64_t deadline) {
struct sf *conn = msockdata(s, sf_type);
if(dill_slow(!conn)) return -1;
if(dill_slow(conn->res != SF_ACTIVE)) {errno = ECONNRESET; return -1;}
/* Create a message object. */
struct msg msg;
msg.buf = malloc(len);
if(dill_slow(!msg.buf)) {errno = ENOMEM; return -1;}
memcpy(msg.buf, buf, len);
msg.len = len;
/* Send it to the worker. */
int rc = chsend(conn->ochan, &msg, sizeof(msg), deadline);
if(dill_fast(rc >= 0)) return 0;
/* Closed pipe means that the connection was terminated. */
if(errno == EPIPE) {
dill_assert(conn->ores == SF_RESET);
conn->res = SF_RESET;
errno = ECONNRESET;
}
/* Clean up. */
int err = errno;
free(msg.buf);
errno = err;
return -1;
}
int tcppeer(int s, ipaddr *addr) {
struct tcpconn *conn = bsockdata(s, tcpconn_type);
if(dill_slow(!conn)) return -1;
if(dill_fast(addr))
*addr = conn->addr;
return 0;
}
/* Get memory page size. The query is done once only. The value is cached. */
static size_t dill_page_size(void) {
static long pgsz = 0;
if(dill_fast(pgsz))
return (size_t)pgsz;
pgsz = sysconf(_SC_PAGE_SIZE);
dill_assert(pgsz > 0);
return (size_t)pgsz;
}
static void dill_prefix_hclose(struct dill_hvfs *hvfs) {
struct dill_prefix_sock *self = (struct dill_prefix_sock*)hvfs;
if(dill_fast(self->u >= 0)) {
int rc = dill_hclose(self->u);
dill_assert(rc == 0);
}
if(!self->mem) free(self);
}
static void dill_tls_hclose(struct dill_hvfs *hvfs) {
struct dill_tls_sock *self = (struct dill_tls_sock*)hvfs;
SSL_free(self->ssl);
SSL_CTX_free(self->ctx);
if(dill_fast(self->u >= 0)) {
int rc = dill_hclose(self->u);
dill_assert(rc == 0);
}
if(!self->mem) free(self);
}
static int dill_tcp_bsendl(struct dill_bsock_vfs *bvfs,
struct dill_iolist *first, struct dill_iolist *last, int64_t deadline) {
struct dill_tcp_conn *self = dill_cont(bvfs, struct dill_tcp_conn, bvfs);
if(dill_slow(self->sbusy)) {errno = EBUSY; return -1;}
if(dill_slow(self->outdone)) {errno = EPIPE; return -1;}
if(dill_slow(self->outerr)) {errno = ECONNRESET; return -1;}
self->sbusy = 1;
ssize_t sz = dill_fd_send(self->fd, first, last, deadline);
self->sbusy = 0;
if(dill_fast(sz >= 0)) return sz;
self->outerr = 1;
return -1;
}
static int dill_tcp_brecvl(struct dill_bsock_vfs *bvfs,
struct dill_iolist *first, struct dill_iolist *last, int64_t deadline) {
struct dill_tcp_conn *self = dill_cont(bvfs, struct dill_tcp_conn, bvfs);
if(dill_slow(self->rbusy)) {errno = EBUSY; return -1;}
if(dill_slow(self->indone)) {errno = EPIPE; return -1;}
if(dill_slow(self->inerr)) {errno = ECONNRESET; return -1;}
self->rbusy = 1;
int rc = dill_fd_recv(self->fd, &self->rxbuf, first, last, deadline);
self->rbusy = 0;
if(dill_fast(rc == 0)) return 0;
if(errno == EPIPE) self->indone = 1;
else self->inerr = 1;
return -1;
}
struct dill_ctx *dill_getctx_(void) {
int rc = pthread_once(&dill_keyonce, dill_makekey);
dill_assert(rc == 0);
struct dill_ctx *ctx = pthread_getspecific(dill_key);
if(dill_fast(ctx)) return ctx;
ctx = calloc(1, sizeof(struct dill_ctx));
dill_assert(ctx);
dill_ctx_init_(ctx);
if(dill_ismain()) {
dill_main = ctx;
rc = atexit(dill_ctx_atexit);
dill_assert(rc == 0);
}
rc = pthread_setspecific(dill_key, ctx);
dill_assert(rc == 0);
return ctx;
}
int dsaccept(int s, struct sockaddr *addr, socklen_t *addrlen,
int64_t deadline) {
int as;
while(1) {
/* Try to accept new connection synchronously. */
as = accept(s, addr, addrlen);
if(dill_fast(as >= 0))
break;
if(dill_slow(errno != EAGAIN && errno != EWOULDBLOCK)) return -1;
/* Operation is in progress. Wait till new connection is available. */
int rc = fdin(s, deadline);
if(dill_slow(rc < 0)) return -1;
}
int rc = dsunblock(as);
dill_assert(rc == 0);
return as;
}
static int tcpconn_flush(int s, int64_t deadline) {
struct tcpconn *conn = bsockdata(s, tcpconn_type);
if(dill_slow(!conn)) return -1;
/* Try to send the buffered data. */
if(conn->olen == 0) return 0;
size_t sent = conn->olen;
int rc = dssend(conn->fd, conn->obuf, &sent, deadline);
if(dill_fast(rc == 0)) {
conn->olen = 0;
return 0;
}
/* Drop sent data from the buffer. */
int err = errno;
memmove(conn->obuf, conn->obuf + sent, conn->olen - sent);
conn->olen -= sent;
errno = err;
return -1;
}
static void *dill_cr_query(struct dill_hvfs *vfs, const void *type) {
struct dill_cr *cr = dill_cont(vfs, struct dill_cr, vfs);
if(dill_fast(type == dill_cr_type)) return cr;
errno = ENOTSUP;
return NULL;
}
static void *dill_bundle_query(struct dill_hvfs *vfs, const void *type) {
if(dill_fast(type == dill_bundle_type)) return vfs;
errno = ENOTSUP;
return NULL;
}