static int afalg_setup_async_event_notification(afalg_aio *aio)
{
ASYNC_JOB *job;
ASYNC_WAIT_CTX *waitctx;
void *custom = NULL;
int ret;
if ((job = ASYNC_get_current_job()) != NULL) {
/* Async mode */
waitctx = ASYNC_get_wait_ctx(job);
if (waitctx == NULL) {
ALG_WARN("%s(%d): ASYNC_get_wait_ctx error", __FILE__, __LINE__);
return 0;
}
/* Get waitfd from ASYNC_WAIT_CTX if it is already set */
ret = ASYNC_WAIT_CTX_get_fd(waitctx, engine_afalg_id,
&aio->efd, &custom);
if (ret == 0) {
/*
* waitfd is not set in ASYNC_WAIT_CTX, create a new one
* and set it. efd will be signaled when AIO operation completes
*/
aio->efd = eventfd(0);
if (aio->efd == -1) {
ALG_PERR("%s(%d): Failed to get eventfd : ", __FILE__,
__LINE__);
AFALGerr(AFALG_F_AFALG_SETUP_ASYNC_EVENT_NOTIFICATION,
AFALG_R_EVENTFD_FAILED);
return 0;
}
ret = ASYNC_WAIT_CTX_set_wait_fd(waitctx, engine_afalg_id,
aio->efd, custom,
afalg_waitfd_cleanup);
if (ret == 0) {
ALG_WARN("%s(%d): Failed to set wait fd", __FILE__, __LINE__);
close(aio->efd);
return 0;
}
/* make fd non-blocking in async mode */
if (fcntl(aio->efd, F_SETFL, O_NONBLOCK) != 0) {
ALG_WARN("%s(%d): Failed to set event fd as NONBLOCKING",
__FILE__, __LINE__);
}
}
aio->mode = MODE_ASYNC;
} else {
/* Sync mode */
aio->efd = eventfd(0);
if (aio->efd == -1) {
ALG_PERR("%s(%d): Failed to get eventfd : ", __FILE__, __LINE__);
AFALGerr(AFALG_F_AFALG_SETUP_ASYNC_EVENT_NOTIFICATION,
AFALG_R_EVENTFD_FAILED);
return 0;
}
aio->mode = MODE_SYNC;
}
return 1;
}
static int afalg_create_sk(afalg_ctx *actx, const char *ciphertype,
const char *ciphername)
{
struct sockaddr_alg sa;
int r = -1;
actx->bfd = actx->sfd = -1;
memset(&sa, 0, sizeof(sa));
sa.salg_family = AF_ALG;
strncpy((char *) sa.salg_type, ciphertype, ALG_MAX_SALG_TYPE);
sa.salg_type[ALG_MAX_SALG_TYPE-1] = '\0';
strncpy((char *) sa.salg_name, ciphername, ALG_MAX_SALG_NAME);
sa.salg_name[ALG_MAX_SALG_NAME-1] = '\0';
actx->bfd = socket(AF_ALG, SOCK_SEQPACKET, 0);
if (actx->bfd == -1) {
ALG_PERR("%s: Failed to open socket : ", __func__);
AFALGerr(AFALG_F_AFALG_CREATE_SK, AFALG_R_SOCKET_CREATE_FAILED);
goto err;
}
r = bind(actx->bfd, (struct sockaddr *)&sa, sizeof(sa));
if (r < 0) {
ALG_PERR("%s: Failed to bind socket : ", __func__);
AFALGerr(AFALG_F_AFALG_CREATE_SK, AFALG_R_SOCKET_BIND_FAILED);
goto err;
}
actx->sfd = accept(actx->bfd, NULL, 0);
if (actx->sfd < 0) {
ALG_PERR("%s: Socket Accept Failed : ", __func__);
AFALGerr(AFALG_F_AFALG_CREATE_SK, AFALG_R_SOCKET_ACCEPT_FAILED);
goto err;
}
return 1;
err:
if (actx->bfd >= 0)
close(actx->bfd);
if (actx->sfd >= 0)
close(actx->sfd);
actx->bfd = actx->sfd = -1;
return 0;
}
static ossl_inline int afalg_set_key(afalg_ctx *actx, const unsigned char *key,
const int klen)
{
int ret;
ret = setsockopt(actx->bfd, SOL_ALG, ALG_SET_KEY, key, klen);
if (ret < 0) {
ALG_PERR("%s(%d): Failed to set socket option : ", __FILE__, __LINE__);
AFALGerr(AFALG_F_AFALG_SET_KEY, AFALG_R_SOCKET_SET_KEY_FAILED);
return 0;
}
return 1;
}
static int afalg_init_aio(afalg_aio *aio)
{
int r = -1;
/* Initialise for AIO */
aio->aio_ctx = 0;
r = io_setup(MAX_INFLIGHTS, &aio->aio_ctx);
if (r < 0) {
ALG_PERR("%s: io_setup error : ", __func__);
AFALGerr(AFALG_F_AFALG_INIT_AIO, AFALG_R_IO_SETUP_FAILED);
return 0;
}
memset(aio->cbt, 0, sizeof(aio->cbt));
aio->efd = -1;
aio->mode = MODE_UNINIT;
return 1;
}
static int afalg_start_cipher_sk(afalg_ctx *actx, const unsigned char *in,
size_t inl, const unsigned char *iv,
unsigned int enc)
{
struct msghdr msg = { 0 };
struct cmsghdr *cmsg;
struct iovec iov;
ssize_t sbytes;
# ifdef ALG_ZERO_COPY
int ret;
# endif
char cbuf[CMSG_SPACE(ALG_IV_LEN(ALG_AES_IV_LEN)) + CMSG_SPACE(ALG_OP_LEN)];
memset(cbuf, 0, sizeof(cbuf));
msg.msg_control = cbuf;
msg.msg_controllen = sizeof(cbuf);
/*
* cipher direction (i.e. encrypt or decrypt) and iv are sent to the
* kernel as part of sendmsg()'s ancillary data
*/
cmsg = CMSG_FIRSTHDR(&msg);
afalg_set_op_sk(cmsg, enc);
cmsg = CMSG_NXTHDR(&msg, cmsg);
afalg_set_iv_sk(cmsg, iv, ALG_AES_IV_LEN);
/* iov that describes input data */
iov.iov_base = (unsigned char *)in;
iov.iov_len = inl;
msg.msg_flags = MSG_MORE;
# ifdef ALG_ZERO_COPY
/*
* ZERO_COPY mode
* Works best when buffer is 4k aligned
* OPENS: out of place processing (i.e. out != in)
*/
/* Input data is not sent as part of call to sendmsg() */
msg.msg_iovlen = 0;
msg.msg_iov = NULL;
/* Sendmsg() sends iv and cipher direction to the kernel */
sbytes = sendmsg(actx->sfd, &msg, 0);
if (sbytes < 0) {
ALG_PERR("%s: sendmsg failed for zero copy cipher operation : ",
__func__);
return 0;
}
/*
* vmsplice and splice are used to pin the user space input buffer for
* kernel space processing avoiding copys from user to kernel space
*/
ret = vmsplice(actx->zc_pipe[1], &iov, 1, SPLICE_F_GIFT);
if (ret < 0) {
ALG_PERR("%s: vmsplice failed : ", __func__);
return 0;
}
ret = splice(actx->zc_pipe[0], NULL, actx->sfd, NULL, inl, 0);
if (ret < 0) {
ALG_PERR("%s: splice failed : ", __func__);
return 0;
}
# else
msg.msg_iovlen = 1;
msg.msg_iov = &iov;
/* Sendmsg() sends iv, cipher direction and input data to the kernel */
sbytes = sendmsg(actx->sfd, &msg, 0);
if (sbytes < 0) {
ALG_PERR("%s: sendmsg failed for cipher operation : ", __func__);
return 0;
}
if (sbytes != (ssize_t) inl) {
ALG_WARN("Cipher operation send bytes %zd != inlen %zd\n", sbytes,
inl);
return 0;
}
# endif
return 1;
}
static int afalg_fin_cipher_aio(afalg_aio *aio, int sfd, unsigned char *buf,
size_t len)
{
int r;
int retry = 0;
unsigned int done = 0;
struct iocb *cb;
struct timespec timeout;
struct io_event events[MAX_INFLIGHTS];
u_int64_t eval = 0;
timeout.tv_sec = 0;
timeout.tv_nsec = 0;
/* if efd has not been initialised yet do it here */
if (aio->mode == MODE_UNINIT) {
r = afalg_setup_async_event_notification(aio);
if (r == 0)
return 0;
}
cb = &(aio->cbt[0 % MAX_INFLIGHTS]);
memset(cb, '\0', sizeof(*cb));
cb->aio_fildes = sfd;
cb->aio_lio_opcode = IOCB_CMD_PREAD;
/*
* The pointer has to be converted to unsigned value first to avoid
* sign extension on cast to 64 bit value in 32-bit builds
*/
cb->aio_buf = (size_t)buf;
cb->aio_offset = 0;
cb->aio_data = 0;
cb->aio_nbytes = len;
cb->aio_flags = IOCB_FLAG_RESFD;
cb->aio_resfd = aio->efd;
/*
* Perform AIO read on AFALG socket, this in turn performs an async
* crypto operation in kernel space
*/
r = io_read(aio->aio_ctx, 1, &cb);
if (r < 0) {
ALG_PWARN("%s: io_read failed : ", __func__);
return 0;
}
do {
/* While AIO read is being performed pause job */
ASYNC_pause_job();
/* Check for completion of AIO read */
r = read(aio->efd, &eval, sizeof(eval));
if (r < 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK)
continue;
ALG_PERR("%s: read failed for event fd : ", __func__);
return 0;
} else if (r == 0 || eval <= 0) {
ALG_WARN("%s: eventfd read %d bytes, eval = %lu\n", __func__, r,
eval);
}
if (eval > 0) {
/* Get results of AIO read */
r = io_getevents(aio->aio_ctx, 1, MAX_INFLIGHTS,
events, &timeout);
if (r > 0) {
/*
* events.res indicates the actual status of the operation.
* Handle the error condition first.
*/
if (events[0].res < 0) {
/*
* Underlying operation cannot be completed at the time
* of previous submission. Resubmit for the operation.
*/
if (events[0].res == -EBUSY && retry++ < 3) {
r = io_read(aio->aio_ctx, 1, &cb);
if (r < 0) {
ALG_PERR("%s: retry %d for io_read failed : ",
__func__, retry);
return 0;
}
continue;
} else {
/*
* Retries exceed for -EBUSY or unrecoverable error
* condition for this instance of operation.
*/
ALG_WARN
("%s: Crypto Operation failed with code %lld\n",
__func__, events[0].res);
return 0;
}
}
/* Operation successful. */
done = 1;
} else if (r < 0) {
ALG_PERR("%s: io_getevents failed : ", __func__);
return 0;
} else {
ALG_WARN("%s: io_geteventd read 0 bytes\n", __func__);
}
}
} while (!done);
return 1;
}
