static int do_vmsplice(MPID_Request *sreq, int pipe_fd, MPL_IOV iov[],
int *iov_offset, int *iov_count, int *complete)
{
int mpi_errno = MPI_SUCCESS;
ssize_t err;
#if 1
err = vmsplice(pipe_fd, &iov[*iov_offset], *iov_count, SPLICE_F_NONBLOCK);
#else
err = writev(pipe_fd, &iov[*iov_offset], *iov_count);
#endif
if (err < 0) {
if (errno == EAGAIN) goto fn_exit;
MPIR_ERR_CHKANDJUMP2(errno != EAGAIN, mpi_errno, MPI_ERR_OTHER, "**vmsplice",
"**vmsplice %d %s", errno, MPIU_Strerror(errno));
}
*complete = adjust_partially_xferred_iov(iov, iov_offset, iov_count, err);
if (*complete) {
/* look for additional data to send and reload IOV if there is more */
mpi_errno = check_req_complete(sreq->ch.vc, sreq, complete);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
if (*complete) {
err = close(pipe_fd);
MPIR_ERR_CHKANDJUMP(err < 0, mpi_errno, MPI_ERR_OTHER, "**close");
MPL_DBG_MSG(MPIDI_CH3_DBG_CHANNEL, VERBOSE, ".... complete");
}
}
fn_fail:
fn_exit:
return mpi_errno;
}
static int check_unaligned_vmsplice(void)
{
int p[2], ret;
char buf; /* :) */
struct iovec iov;
ret = pipe(p);
if (ret < 0) {
pr_perror("Can't create pipe");
return ret;
}
iov.iov_base = &buf;
iov.iov_len = sizeof(buf);
ret = vmsplice(p[1], &iov, 1, SPLICE_F_GIFT | SPLICE_F_NONBLOCK);
if (ret < 0) {
pr_perror("Unaligned vmsplice doesn't work");
goto err;
}
pr_info("Unaligned vmsplice works OK\n");
ret = 0;
err:
close(p[0]);
close(p[1]);
return ret;
}
static void *Thread (void *data)
{
stream_t *stream = data;
stream_sys_t *p_sys = stream->p_sys;
#ifdef HAVE_VMSPLICE
ssize_t page_mask = sysconf (_SC_PAGE_SIZE) - 1;
#endif
int fd = p_sys->write_fd;
bool error = false;
do
{
ssize_t len;
int canc = vlc_savecancel ();
#ifdef HAVE_VMSPLICE
unsigned char *buf = mmap (NULL, bufsize, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
vlc_cleanup_push (cleanup_mmap, buf);
#else
unsigned char buf[bufsize];
#endif
len = stream_Read (stream->p_source, buf, bufsize);
vlc_restorecancel (canc);
error = len <= 0;
for (ssize_t i = 0, j; i < len; i += j)
{
#ifdef HAVE_VMSPLICE
if ((len - i) <= page_mask) /* incomplete last page */
j = write (fd, buf + i, len - i);
else
{
struct iovec iov = { buf + i, (len - i) & ~page_mask, };
j = vmsplice (fd, &iov, 1, SPLICE_F_GIFT);
}
if (j == -1 && errno == ENOSYS) /* vmsplice() not supported */
#endif
j = write (fd, buf + i, len - i);
if (j <= 0)
{
if (j == 0)
errno = EPIPE;
msg_Err (stream, "cannot write data (%m)");
error = true;
break;
}
}
#ifdef HAVE_VMSPLICE
vlc_cleanup_run (); /* munmap (buf, bufsize) */
#endif
}
while (!error);
msg_Dbg (stream, "compressed stream at EOF");
return NULL;
}
static void vmsplice_test(void)
{
int pipes[2];
long written;
int ret;
int fd_out;
struct iovec v;
loff_t offset;
v.iov_base = buffer;
v.iov_len = TEST_BLOCK_SIZE;
SAFE_PIPE(cleanup, pipes);
fd_out = SAFE_OPEN(cleanup, TESTFILE, O_WRONLY | O_CREAT | O_TRUNC, 0644);
struct pollfd pfd = {.fd = pipes[1], .events = POLLOUT};
offset = 0;
while (v.iov_len) {
/*
* in a real app you'd be more clever with poll of course,
* here we are basically just blocking on output room and
* not using the free time for anything interesting.
*/
if (poll(&pfd, 1, -1) < 0)
tst_brkm(TBROK | TERRNO, cleanup, "poll() failed");
written = vmsplice(pipes[1], &v, 1, 0);
if (written < 0) {
tst_brkm(TBROK | TERRNO, cleanup, "vmsplice() failed");
} else {
if (written == 0) {
break;
} else {
v.iov_base += written;
v.iov_len -= written;
}
}
ret = splice(pipes[0], NULL, fd_out, &offset, written, 0);
if (ret < 0)
tst_brkm(TBROK | TERRNO, cleanup, "splice() failed");
//printf("offset = %lld\n", (long long)offset);
}
close(pipes[0]);
close(pipes[1]);
close(fd_out);
fd_out = 0;
check_file();
}
static int dump_pages(struct page_pipe *pp, struct page_xfer *xfer, void *addr)
{
struct page_pipe_buf *ppb;
list_for_each_entry(ppb, &pp->bufs, l)
if (vmsplice(ppb->p[1], ppb->iov, ppb->nr_segs,
SPLICE_F_GIFT | SPLICE_F_NONBLOCK) !=
ppb->pages_in * PAGE_SIZE) {
pr_perror("Can't get shmem into page-pipe");
return -1;
}
return page_xfer_dump_pages(xfer, pp, (unsigned long)addr);
}
// wrapper to write or vmsplice
void Write(int fd, void *msg, size_t length) {
int r;
#ifdef VMSPLICE
struct iovec iov;
iov.iov_base = msg;
iov.iov_len = length;
r = vmsplice(fd, &iov, 1, 0);
#else
r = write(fd, msg, length);
#endif
if (r < 0)
{
perror("Write");
}
}
static void vmsplice_verify(const struct test_case_t *tc)
{
TEST(vmsplice(*(tc->fd), tc->iov, tc->nr_segs, 0));
if (TEST_RETURN != -1) {
tst_resm(TFAIL, "vmsplice() returned %ld, "
"expected -1, errno:%d", TEST_RETURN,
tc->exp_errno);
return;
}
if (TEST_ERRNO == tc->exp_errno) {
tst_resm(TPASS | TTERRNO, "vmsplice() failed as expected");
} else {
tst_resm(TFAIL | TTERRNO,
"vmsplice() failed unexpectedly; expected: %d - %s",
tc->exp_errno, strerror(tc->exp_errno));
}
}
void SendData(ArgStruct *p)
{
long ret;
int fd = p->tr ? vmsplice_to_child[1] : vmsplice_to_parent[1];
int left = p->bufflen;
struct iovec v = {
.iov_base = p->s_ptr,
.iov_len = p->bufflen
};
/* Can only vmsplice 64k at a time -- don't know if this is a
hard-coded limit or not */
while (left > 0) {
v.iov_len = left > MAX_VMSPLICE_SIZE ? MAX_VMSPLICE_SIZE : left;
if (vmsplice(fd, &v, 1, 0) != v.iov_len) {
despair("Netpipe failed to vmsplice.");
}
left -= v.iov_len;
v.iov_base += v.iov_len;
}
}
void RecvData(ArgStruct *p)
{
int fd = p->tr ? vmsplice_to_parent[0] : vmsplice_to_child[0];
int len, left = p->bufflen;
char *dest = p->r_ptr;
/* Can only vmsplice 64k at a time */
while (left > 0) {
len = left > MAX_VMSPLICE_SIZE ? MAX_VMSPLICE_SIZE : left;
if (read(fd, dest, len) != len) {
despair("Netpipe failed to read bytes from the vmsplice.");
}
left -= len;
dest += len;
}
}
static void release_write_buffer(test_data* td, struct iovec* vecs, int n_vecs) {
pipe_state *ps = (pipe_state *)td->data;
assert(n_vecs == 1);
assert(vecs == &td->iov);
#ifdef VMSPLICE_COOP
ps->bytes_written += td->size;
while(ps->bytes_written >= coop_reporting_chunk_size) {
ps->chunks_written++;
ps->bytes_written -= coop_reporting_chunk_size;
}
#endif
ps->write_offset += td->size;
while(vecs[0].iov_len > 0) {
ssize_t this_write = vmsplice(ps->fds[1], vecs, n_vecs, 0);
if(this_write < 0)
err(1, "vmsplice");
else if(this_write == 0)
break;
vecs[0].iov_len -= this_write;
vecs[0].iov_base = ((char*)vecs[0].iov_base) + this_write;
}
}
/*
* For splice writing, we can vmsplice our data buffer directly into a
* pipe and then splice that to a file.
*/
static int fio_splice_write(struct thread_data *td, struct io_u *io_u)
{
struct spliceio_data *sd = td->io_ops->data;
struct iovec iov = {
.iov_base = io_u->xfer_buf,
.iov_len = io_u->xfer_buflen,
};
struct pollfd pfd = { .fd = sd->pipe[1], .events = POLLOUT, };
struct fio_file *f = io_u->file;
off_t off = io_u->offset;
int ret, ret2;
while (iov.iov_len) {
if (poll(&pfd, 1, -1) < 0)
return errno;
ret = vmsplice(sd->pipe[1], &iov, 1, SPLICE_F_NONBLOCK);
if (ret < 0)
return -errno;
iov.iov_len -= ret;
iov.iov_base += ret;
while (ret) {
ret2 = splice(sd->pipe[0], NULL, f->fd, &off, ret, 0);
if (ret2 < 0)
return -errno;
ret -= ret2;
}
}
return io_u->xfer_buflen;
}
static int fio_spliceio_queue(struct thread_data *td, struct io_u *io_u)
{
struct spliceio_data *sd = td->io_ops->data;
int uninitialized_var(ret);
fio_ro_check(td, io_u);
if (io_u->ddir == DDIR_READ) {
if (sd->vmsplice_to_user) {
ret = fio_splice_read(td, io_u);
/*
* This kernel doesn't support vmsplice to user
* space. Reset the vmsplice_to_user flag, so that
* we retry below and don't hit this path again.
*/
if (ret == -EBADF)
sd->vmsplice_to_user = 0;
}
if (!sd->vmsplice_to_user)
ret = fio_splice_read_old(td, io_u);
} else if (io_u->ddir == DDIR_WRITE)
ret = fio_splice_write(td, io_u);
else
ret = fsync(io_u->file->fd);
if (ret != (int) io_u->xfer_buflen) {
if (ret >= 0) {
io_u->resid = io_u->xfer_buflen - ret;
io_u->error = 0;
return FIO_Q_COMPLETED;
} else
io_u->error = errno;
}
if (io_u->error) {
td_verror(td, io_u->error, "xfer");
if (io_u->error == EINVAL)
log_err("fio: looks like splice doesn't work on this"
" file system\n");
}
return FIO_Q_COMPLETED;
}
static void fio_spliceio_cleanup(struct thread_data *td)
{
struct spliceio_data *sd = td->io_ops->data;
if (sd) {
close(sd->pipe[0]);
close(sd->pipe[1]);
free(sd);
}
}
static int fio_spliceio_init(struct thread_data *td)
{
struct spliceio_data *sd = malloc(sizeof(*sd));
if (pipe(sd->pipe) < 0) {
td_verror(td, errno, "pipe");
free(sd);
return 1;
}
/*
* Assume this work, we'll reset this if it doesn't
*/
sd->vmsplice_to_user = 1;
/*
* Works with "real" vmsplice to user, eg mapping pages directly.
* Reset if we fail.
*/
sd->vmsplice_to_user_map = 1;
/*
* And if vmsplice_to_user works, we definitely need aligned
* buffers. Just set ->odirect to force that.
*/
if (td_read(td))
td->o.odirect = 1;
td->io_ops->data = sd;
return 0;
}
static struct ioengine_ops ioengine = {
.name = "splice",
.version = FIO_IOOPS_VERSION,
.init = fio_spliceio_init,
.queue = fio_spliceio_queue,
.cleanup = fio_spliceio_cleanup,
.open_file = generic_open_file,
.close_file = generic_close_file,
.get_file_size = generic_get_file_size,
.flags = FIO_SYNCIO | FIO_PIPEIO,
};
#else /* FIO_HAVE_SPLICE */
/*
* When we have a proper configure system in place, we simply wont build
* and install this io engine. For now install a crippled version that
* just complains and fails to load.
*/
static int fio_spliceio_init(struct thread_data fio_unused *td)
{
fprintf(stderr, "fio: splice not available\n");
return 1;
}
static struct ioengine_ops ioengine = {
.name = "splice",
.version = FIO_IOOPS_VERSION,
.init = fio_spliceio_init,
};
#endif
static void fio_init fio_spliceio_register(void)
{
register_ioengine(&ioengine);
}
static void fio_exit fio_spliceio_unregister(void)
{
unregister_ioengine(&ioengine);
}
static int afalg_start_cipher_sk(afalg_ctx *actx, const unsigned char *in,
size_t inl, const unsigned char *iv,
unsigned int enc)
{
struct msghdr msg = { 0 };
struct cmsghdr *cmsg;
struct iovec iov;
ssize_t sbytes;
# ifdef ALG_ZERO_COPY
int ret;
# endif
char cbuf[CMSG_SPACE(ALG_IV_LEN(ALG_AES_IV_LEN)) + CMSG_SPACE(ALG_OP_LEN)];
memset(cbuf, 0, sizeof(cbuf));
msg.msg_control = cbuf;
msg.msg_controllen = sizeof(cbuf);
/*
* cipher direction (i.e. encrypt or decrypt) and iv are sent to the
* kernel as part of sendmsg()'s ancillary data
*/
cmsg = CMSG_FIRSTHDR(&msg);
afalg_set_op_sk(cmsg, enc);
cmsg = CMSG_NXTHDR(&msg, cmsg);
afalg_set_iv_sk(cmsg, iv, ALG_AES_IV_LEN);
/* iov that describes input data */
iov.iov_base = (unsigned char *)in;
iov.iov_len = inl;
msg.msg_flags = MSG_MORE;
# ifdef ALG_ZERO_COPY
/*
* ZERO_COPY mode
* Works best when buffer is 4k aligned
* OPENS: out of place processing (i.e. out != in)
*/
/* Input data is not sent as part of call to sendmsg() */
msg.msg_iovlen = 0;
msg.msg_iov = NULL;
/* Sendmsg() sends iv and cipher direction to the kernel */
sbytes = sendmsg(actx->sfd, &msg, 0);
if (sbytes < 0) {
ALG_PERR("%s: sendmsg failed for zero copy cipher operation : ",
__func__);
return 0;
}
/*
* vmsplice and splice are used to pin the user space input buffer for
* kernel space processing avoiding copys from user to kernel space
*/
ret = vmsplice(actx->zc_pipe[1], &iov, 1, SPLICE_F_GIFT);
if (ret < 0) {
ALG_PERR("%s: vmsplice failed : ", __func__);
return 0;
}
ret = splice(actx->zc_pipe[0], NULL, actx->sfd, NULL, inl, 0);
if (ret < 0) {
ALG_PERR("%s: splice failed : ", __func__);
return 0;
}
# else
msg.msg_iovlen = 1;
msg.msg_iov = &iov;
/* Sendmsg() sends iv, cipher direction and input data to the kernel */
sbytes = sendmsg(actx->sfd, &msg, 0);
if (sbytes < 0) {
ALG_PERR("%s: sendmsg failed for cipher operation : ", __func__);
return 0;
}
if (sbytes != (ssize_t) inl) {
ALG_WARN("Cipher operation send bytes %zd != inlen %zd\n", sbytes,
inl);
return 0;
}
# endif
return 1;
}
static u_int64_t transfer_mmappipe(int sock_fd, struct scope_parameter *param,
option_fields_t *options)
{
const int CHUNK = 16 * 4096;
u_int64_t transferred = 0ULL;
u_int64_t size = 1024ULL * options->kbytes_to_transfer;
unsigned long pos;
size_t len;
ssize_t slen;
unsigned long curr;
unsigned long *curr_addr;
unsigned long base;
void *mapped_base;
size_t buf_size;
int pipe_fd[2];
struct iovec iov;
curr_addr = param->mapped_io + (options->scope_chn ? 0x118 : 0x114);
base = *(unsigned long *)(param->mapped_io +
(options->scope_chn ? 0x10c : 0x104));
mapped_base = options->scope_chn ? param->mapped_buf_b
: param->mapped_buf_a;
buf_size = options->scope_chn ? param->buf_b_size : param->buf_a_size;
if (pipe(pipe_fd)) {
fprintf(stderr, "create pipe failed, %d\n", errno);
return 0ULL;
}
pos = 0UL;
while (!size || transferred < size) {
if (pos == buf_size)
pos = 0UL;
curr = *curr_addr - base;
if (pos + CHUNK <= curr) {
len = CHUNK;
} else if (pos > curr) {
if (pos + CHUNK <= buf_size) {
len = CHUNK;
} else {
len = buf_size - pos;
}
} else {
continue;
}
iov.iov_base = mapped_base + pos;
iov.iov_len = len;
slen = vmsplice(pipe_fd[1], &iov, 1, 0/*SPLICE_F_GIFT*/);
if (slen != len) {
fprintf(stderr,
"vmsplice failed, %d %d, on %p+%lx %d\n",
slen, errno, mapped_base, pos, len);
break;
}
slen = splice(pipe_fd[0], NULL, sock_fd, NULL, len,
SPLICE_F_MOVE | SPLICE_F_MORE);
if (slen != len) {
fprintf(stderr, "splice failed, %d %d\n", slen, errno);
break;
}
pos += len;
transferred += len;
}
close(pipe_fd[0]);
close(pipe_fd[1]);
return transferred;
}
int main(int argc, char **argv) {
int opt, i = 1, fd, sock;
struct stat sb;
char *addr, *p, *q;
char neterr[256];
int pfd[2];
/* FIXME */
ip = dstr_new("127.0.0.1");
port = 33330;
while ((opt = getopt(argc, argv, "h:p:?")) != -1)
switch (opt) {
case 'h':
dstr_free(ip);
ip = dstr_new(optarg);
i += 2;
break;
case 'p':
port = atoi(optarg);
i += 2;
break;
case '?':
default:
usage();
}
if ((fd = open(argv[i], O_RDONLY)) == -1) {
fprintf(stderr, "Error opening file: %s\n", strerror(errno));
exit(1);
}
fstat(fd, &sb);
if ((addr = mmap(NULL, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0)) == MAP_FAILED) {
fprintf(stderr, "Error mmaping file: %s\n", strerror(errno));
exit(1);
}
close(fd);
if ((sock = net_udp_nonblock_connect(ip, port, neterr, sizeof neterr)) == -1) {
fprintf(stderr, "Connecting %s:%d: %s\n", ip, port, neterr);
exit(1);
}
if (pipe(pfd) != 0) {
fprintf(stderr, "Error creating pipe: %s\n", strerror(errno));
exit(1);
}
p = addr;
q = memchr(p, '\n', sb.st_size);
while (q) {
int len = PAGEALIGN(q - p + 1);
char *line;
if ((line = POSIXALIGN(sysconf(_SC_PAGESIZE), len))) {
struct iovec iov;
memset(line, '\0', len);
strncpy(line, p, q - p);
iov.iov_base = line;
iov.iov_len = len;
/* zero copy */
if (vmsplice(pfd[1], &iov, 1, SPLICE_F_GIFT) == -1) {
fprintf(stderr, "Error vmsplicing: %s\n", strerror(errno));
exit(1);
}
again:
if (splice(pfd[0], NULL, sock, NULL, len, SPLICE_F_MOVE) == -1) {
if (errno == EAGAIN)
goto again;
else {
fprintf(stderr, "Error splicing: %s\n", strerror(errno));
exit(1);
}
}
FREE(line);
} else
fprintf(stderr, "Error allocating memory for line\n");
p = q + 1;
q = memchr(p, '\n', sb.st_size - (q - addr));
}
close(pfd[0]);
close(pfd[1]);
close(sock);
munmap(addr, sb.st_size);
return 0;
}
static void *Thread (void *data)
{
stream_t *stream = data;
stream_sys_t *p_sys = stream->p_sys;
#ifdef HAVE_VMSPLICE
const ssize_t page_mask = sysconf (_SC_PAGE_SIZE) - 1;
#endif
int fd = p_sys->write_fd;
bool error = false;
do
{
ssize_t len;
int canc = vlc_savecancel ();
#ifdef HAVE_VMSPLICE
unsigned char *buf = mmap (NULL, bufsize, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
if (unlikely(buf == MAP_FAILED))
break;
vlc_cleanup_push (cleanup_mmap, buf);
#else
unsigned char *buf = malloc (bufsize);
if (unlikely(buf == NULL))
break;
vlc_cleanup_push (free, buf);
#endif
vlc_mutex_lock (&p_sys->lock);
while (p_sys->paused) /* practically always false, but... */
vlc_cond_wait (&p_sys->wait, &p_sys->lock);
len = stream_Read (stream->p_source, buf, bufsize);
vlc_mutex_unlock (&p_sys->lock);
vlc_restorecancel (canc);
error = len <= 0;
for (ssize_t i = 0, j; i < len; i += j)
{
#ifdef HAVE_VMSPLICE
if ((len - i) <= page_mask) /* incomplete last page */
j = write (fd, buf + i, len - i);
else
{
struct iovec iov = { buf + i, (len - i) & ~page_mask, };
j = vmsplice (fd, &iov, 1, SPLICE_F_GIFT);
}
if (j == -1 && errno == ENOSYS) /* vmsplice() not supported */
#endif
j = write (fd, buf + i, len - i);
if (j <= 0)
{
if (j == 0)
errno = EPIPE;
msg_Err (stream, "cannot write data: %s",
vlc_strerror_c(errno));
error = true;
break;
}
}
vlc_cleanup_run (); /* free (buf) */
}
while (!error);
msg_Dbg (stream, "compressed stream at EOF");
/* Let child process know about EOF */
p_sys->write_fd = -1;
close (fd);
return NULL;
}
/*
* We can now vmsplice into userspace, so do the transfer by splicing into
* a pipe and vmsplicing that into userspace.
*/
static int fio_splice_read(struct thread_data *td, struct io_u *io_u)
{
struct spliceio_data *sd = td->io_ops->data;
struct fio_file *f = io_u->file;
struct iovec iov;
int ret , buflen, mmap_len;
off_t offset;
void *p, *map;
ret = 0;
offset = io_u->offset;
mmap_len = buflen = io_u->xfer_buflen;
if (sd->vmsplice_to_user_map) {
map = mmap(io_u->xfer_buf, buflen, PROT_READ, MAP_PRIVATE|OS_MAP_ANON, 0, 0);
if (map == MAP_FAILED) {
td_verror(td, errno, "mmap io_u");
return -1;
}
p = map;
} else {
map = NULL;
p = io_u->xfer_buf;
}
while (buflen) {
int this_len = buflen;
int flags = 0;
if (this_len > SPLICE_DEF_SIZE) {
this_len = SPLICE_DEF_SIZE;
flags = SPLICE_F_MORE;
}
ret = splice(f->fd, &offset, sd->pipe[1], NULL, this_len,flags);
if (ret < 0) {
if (errno == ENODATA || errno == EAGAIN)
continue;
td_verror(td, errno, "splice-from-fd");
break;
}
buflen -= ret;
iov.iov_base = p;
iov.iov_len = ret;
while (iov.iov_len) {
ret = vmsplice(sd->pipe[0], &iov, 1, SPLICE_F_MOVE);
if (ret < 0) {
if (errno == EFAULT &&
sd->vmsplice_to_user_map) {
sd->vmsplice_to_user_map = 0;
munmap(map, mmap_len);
map = NULL;
p = io_u->xfer_buf;
iov.iov_base = p;
continue;
}
if (errno == EBADF) {
ret = -EBADF;
break;
}
td_verror(td, errno, "vmsplice");
break;
} else if (!ret) {
td_verror(td, ENODATA, "vmsplice");
ret = -1;
break;
}
iov.iov_len -= ret;
iov.iov_base += ret;
p += ret;
}
if (ret < 0)
break;
}
if (sd->vmsplice_to_user_map && munmap(map, mmap_len) < 0) {
td_verror(td, errno, "munnap io_u");
return -1;
}
if (ret < 0)
return ret;
return io_u->xfer_buflen;
}
static int dump_one_shmem(struct shmem_info_dump *si)
{
struct iovec *iovs;
struct page_pipe *pp;
struct page_pipe_buf *ppb;
struct page_xfer xfer;
int err, ret = -1, fd;
unsigned char *map = NULL;
void *addr = NULL;
unsigned long pfn, nrpages;
pr_info("Dumping shared memory %ld\n", si->shmid);
nrpages = (si->size + PAGE_SIZE - 1) / PAGE_SIZE;
map = xmalloc(nrpages * sizeof(*map));
if (!map)
goto err;
fd = open_proc(si->pid, "map_files/%lx-%lx", si->start, si->end);
if (fd < 0)
goto err;
addr = mmap(NULL, si->size, PROT_READ, MAP_SHARED, fd, 0);
close(fd);
if (addr == MAP_FAILED) {
pr_err("Can't map shmem 0x%lx (0x%lx-0x%lx)\n",
si->shmid, si->start, si->end);
goto err;
}
/*
* We can't use pagemap here, because this vma is
* not mapped to us at all, but mincore reports the
* pagecache status of a file, which is correct in
* this case.
*/
err = mincore(addr, si->size, map);
if (err)
goto err_unmap;
iovs = xmalloc(((nrpages + 1) / 2) * sizeof(struct iovec));
if (!iovs)
goto err_unmap;
pp = create_page_pipe((nrpages + 1) / 2, iovs);
if (!pp)
goto err_iovs;
for (pfn = 0; pfn < nrpages; pfn++) {
if (!(map[pfn] & PAGE_RSS))
continue;
if (page_pipe_add_page(pp, (unsigned long)addr + pfn * PAGE_SIZE))
goto err_pp;
}
list_for_each_entry(ppb, &pp->bufs, l)
if (vmsplice(ppb->p[1], ppb->iov, ppb->nr_segs,
SPLICE_F_GIFT | SPLICE_F_NONBLOCK) !=
ppb->pages_in * PAGE_SIZE) {
pr_perror("Can't get shmem into page-pipe");
goto err_pp;
}
err = open_page_xfer(&xfer, CR_FD_SHMEM_PAGEMAP, si->shmid);
if (err)
goto err_pp;
ret = page_xfer_dump_pages(&xfer, pp, (unsigned long)addr);
xfer.close(&xfer);
err_pp:
destroy_page_pipe(pp);
err_iovs:
xfree(iovs);
err_unmap:
munmap(addr, si->size);
err:
xfree(map);
return ret;
}
/*
* stress_splice
* stress copying of /dev/zero to /dev/null
*/
int stress_vm_splice(
uint64_t *const counter,
const uint32_t instance,
const uint64_t max_ops,
const char *name)
{
int fd, fds[2];
uint8_t *buf;
const size_t page_size = stress_get_pagesize();
size_t sz;
(void)instance;
if (!set_vm_splice_bytes) {
if (opt_flags & OPT_FLAGS_MAXIMIZE)
opt_vm_splice_bytes = MAX_VM_SPLICE_BYTES;
if (opt_flags & OPT_FLAGS_MINIMIZE)
opt_vm_splice_bytes = MIN_VM_SPLICE_BYTES;
}
sz = opt_vm_splice_bytes & ~(page_size - 1);
buf = mmap(NULL, sz, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
if (buf == MAP_FAILED) {
pr_failed_dbg(name, "mmap");
return(EXIT_FAILURE);
}
if (pipe(fds) < 0) {
(void)munmap(buf, sz);
pr_failed_err(name, "pipe");
return EXIT_FAILURE;
}
if ((fd = open("/dev/null", O_WRONLY)) < 0) {
(void)munmap(buf, sz);
(void)close(fds[0]);
(void)close(fds[1]);
pr_failed_err(name, "open");
return EXIT_FAILURE;
}
do {
int ret;
ssize_t bytes;
struct iovec iov;
iov.iov_base = buf;
iov.iov_len = sz;
bytes = vmsplice(fds[1], &iov, 1, 0);
if (bytes < 0)
break;
ret = splice(fds[0], NULL, fd, NULL, opt_vm_splice_bytes, SPLICE_F_MOVE);
if (ret < 0)
break;
(*counter)++;
} while (opt_do_run && (!max_ops || *counter < max_ops));
(void)munmap(buf, sz);
(void)close(fd);
(void)close(fds[0]);
(void)close(fds[1]);
return EXIT_SUCCESS;
}
static int
write_splice(connection_t *conn,
jlong mmap_address,
int sublen)
{
struct iovec io;
int result;
int fd = conn->fd;
int write_len = 0;
if (fd < 0) {
return -1;
}
if (conn->ssl_sock) {
return conn->ops->write(conn,
(void*) (PTR) (mmap_address),
sublen);
}
io.iov_base = (void*) (mmap_address);
io.iov_len = sublen;
if (conn->pipe[0] <= 0) {
if (pipe(conn->pipe) < 0) {
fprintf(stderr, "BADPIPE\n");
}
}
sublen = vmsplice(conn->pipe[1], &io, 1, SPLICE_F_MOVE);
if (sublen < 0) {
if (errno != EAGAIN && errno != ECONNRESET && errno != EPIPE) {
fprintf(stderr, "vmsplice addr:%lx result:%d %d\n",
mmap_address,
sublen, errno);
}
return -1;
}
write_len = 0;
while (write_len < sublen) {
int delta = sublen - write_len;
result = splice(conn->pipe[0], 0, fd, 0, delta,
SPLICE_F_MOVE|SPLICE_F_MORE);
if (result <= 0) {
if (errno != EAGAIN && errno != ECONNRESET && errno != EPIPE) {
fprintf(stderr, "splice result:%d pipe:%d fd:%d addr:%lx errno:%d\n",
result, conn->pipe[0], fd, mmap_address, errno);
}
return -1;
}
write_len += result;
}
return sublen;
}
bool AF_ALGTest::do_operation(int op, int rounds)
{
int tfmfd, opfd;
struct sockaddr_alg sa = {
.salg_family = AF_ALG
};
struct msghdr msg = {};
struct iovec iov;
int pipes[2];
struct cmsghdr *cmsg;
char cbuf[CMSG_SPACE(4) + CMSG_SPACE(12)] = {};
struct af_alg_iv *iv;
int rc, i;
ssize_t r;
strcpy((char*)sa.salg_type, "skcipher");
strcpy((char*)sa.salg_name, "cbc(cast5)");
tfmfd = socket(AF_ALG, SOCK_SEQPACKET, 0);
assert(tfmfd >= 0);
rc = bind(tfmfd, (struct sockaddr *)&sa, sizeof(sa));
assert(rc == 0);
rc = setsockopt(tfmfd, SOL_ALG, ALG_SET_KEY,
"\x06\xa9\x21\x40\x36\xb8\xa1\x5b"
"\x51\x2e\x03\xd5\x34\x12\x00\x06", 8);
assert(rc == 0);
opfd = accept(tfmfd, NULL, 0);
assert(opfd != -1);
msg.msg_control = cbuf;
msg.msg_controllen = sizeof(cbuf);
cmsg = CMSG_FIRSTHDR(&msg);
cmsg->cmsg_level = SOL_ALG;
cmsg->cmsg_type = ALG_SET_OP;
cmsg->cmsg_len = CMSG_LEN(4);
*(__u32 *)CMSG_DATA(cmsg) = op;
cmsg = CMSG_NXTHDR(&msg, cmsg);
cmsg->cmsg_level = SOL_ALG;
cmsg->cmsg_type = ALG_SET_IV;
cmsg->cmsg_len = CMSG_LEN(12);
iv = (af_alg_iv*)CMSG_DATA(cmsg);
iv->ivlen = 8;
memcpy(iv->iv, "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
"\xb4\x22\xda\x80\x2c\x9f\xac\x41", 8);
#if 1
iov.iov_base = &in[0];
iov.iov_len = data_size();
msg.msg_iovlen = 1;
msg.msg_iov = &iov;
#endif
#if 0
iov.iov_base = in;
iov.iov_len = 4096;
msg.msg_iovlen = 0;
msg.msg_flags = MSG_MORE;
//msg.msg_iov = &iov;
pipe(pipes);
#endif
for (i = 0; i < rounds; i++) {
r = sendmsg(opfd, &msg, 0);
if (r < 0) {
perror("sendmsg");
return false;
}
//read(opfd, out, data_size());
#if 0
vmsplice(pipes[1], &iov, 1, SPLICE_F_GIFT);
splice(pipes[0], NULL, opfd, NULL, 4096, 0);
#endif
read(opfd, out, 4096);
}
return true;
}