static int crypto842_sdecompress(struct crypto_scomp *tfm,
const u8 *src, unsigned int slen,
u8 *dst, unsigned int *dlen, void *ctx)
{
return sw842_decompress(src, slen, dst, dlen);
}
static int decompress(struct nx842_crypto_ctx *ctx,
struct nx842_crypto_param *p,
struct nx842_crypto_header_group *g,
struct nx842_constraints *c,
u16 ignore)
{
unsigned int slen = be32_to_cpu(g->compressed_length);
unsigned int required_len = be32_to_cpu(g->uncompressed_length);
unsigned int dlen = p->oremain, tmplen;
unsigned int adj_slen = slen;
u8 *src = p->in, *dst = p->out;
u16 padding = be16_to_cpu(g->padding);
int ret, spadding = 0;
ktime_t timeout;
if (!slen || !required_len)
return -EINVAL;
if (p->iremain <= 0 || padding + slen > p->iremain)
return -EOVERFLOW;
if (p->oremain <= 0 || required_len - ignore > p->oremain)
return -ENOSPC;
src += padding;
if (slen % c->multiple)
adj_slen = round_up(slen, c->multiple);
if (slen < c->minimum)
adj_slen = c->minimum;
if (slen > c->maximum)
goto usesw;
if (slen < adj_slen || (u64)src % c->alignment) {
/* we can append padding bytes because the 842 format defines
* an "end" template (see lib/842/842_decompress.c) and will
* ignore any bytes following it.
*/
if (slen < adj_slen)
memset(ctx->sbounce + slen, 0, adj_slen - slen);
memcpy(ctx->sbounce, src, slen);
src = ctx->sbounce;
spadding = adj_slen - slen;
slen = adj_slen;
pr_debug("using decomp sbounce buffer, len %x\n", slen);
}
if (dlen % c->multiple)
dlen = round_down(dlen, c->multiple);
if (dlen < required_len || (u64)dst % c->alignment) {
dst = ctx->dbounce;
dlen = min(required_len, BOUNCE_BUFFER_SIZE);
pr_debug("using decomp dbounce buffer, len %x\n", dlen);
}
if (dlen < c->minimum)
goto usesw;
if (dlen > c->maximum)
dlen = c->maximum;
tmplen = dlen;
timeout = ktime_add_ms(ktime_get(), DECOMP_BUSY_TIMEOUT);
do {
dlen = tmplen; /* reset dlen, if we're retrying */
ret = ctx->driver->decompress(src, slen, dst, &dlen, ctx->wmem);
} while (ret == -EBUSY && ktime_before(ktime_get(), timeout));
if (ret) {
usesw:
/* reset everything, sw doesn't have constraints */
src = p->in + padding;
slen = be32_to_cpu(g->compressed_length);
spadding = 0;
dst = p->out;
dlen = p->oremain;
if (dlen < required_len) { /* have ignore bytes */
dst = ctx->dbounce;
dlen = BOUNCE_BUFFER_SIZE;
}
pr_info_ratelimited("using software 842 decompression\n");
ret = sw842_decompress(src, slen, dst, &dlen);
}
if (ret)
return ret;
slen -= spadding;
dlen -= ignore;
if (ignore)
pr_debug("ignoring last %x bytes\n", ignore);
if (dst == ctx->dbounce)
memcpy(p->out, dst, dlen);
pr_debug("decompress slen %x padding %x dlen %x ignore %x\n",
slen, padding, dlen, ignore);
return update_param(p, slen + padding, dlen);
}
/**
* nx842_compress - Compress data using the 842 algorithm
*
* Compression provide by the NX842 coprocessor on IBM Power systems.
* The input buffer is compressed and the result is stored in the
* provided output buffer.
*
* Upon return from this function @outlen contains the length of the
* compressed data. If there is an error then @outlen will be 0 and an
* error will be specified by the return code from this function.
*
* @in: Pointer to input buffer, must be page aligned
* @inlen: Length of input buffer, must be PAGE_SIZE
* @out: Pointer to output buffer
* @outlen: Length of output buffer
* @wrkmem: ptr to buffer for working memory, size determined by
* nx842_get_workmem_size()
*
* Returns:
* 0 Success, output of length @outlen stored in the buffer at @out
* -ENOMEM Unable to allocate internal buffers
* -ENOSPC Output buffer is to small
* -EMSGSIZE XXX Difficult to describe this limitation
* -EIO Internal error
* -ENODEV Hardware unavailable
*/
int nx842_compress(const unsigned char *in, unsigned int inlen,
unsigned char *out, unsigned int *outlen, void *wmem)
{
struct nx842_header *hdr;
struct nx842_devdata *local_devdata;
struct device *dev = NULL;
struct nx842_workmem *workmem;
struct nx842_scatterlist slin, slout;
struct nx_csbcpb *csbcpb;
int ret = 0, max_sync_size, i, bytesleft, size, hdrsize;
unsigned long inbuf, outbuf, padding;
struct vio_pfo_op op = {
.done = NULL,
.handle = 0,
.timeout = 0,
};
unsigned long start_time = get_tb();
/*
* Make sure input buffer is 64k page aligned. This is assumed since
* this driver is designed for page compression only (for now). This
* is very nice since we can now use direct DDE(s) for the input and
* the alignment is guaranteed.
*/
inbuf = (unsigned long)in;
if (!IS_ALIGNED(inbuf, PAGE_SIZE) || inlen != PAGE_SIZE)
return -EINVAL;
rcu_read_lock();
local_devdata = rcu_dereference(devdata);
if (!local_devdata || !local_devdata->dev) {
rcu_read_unlock();
return -ENODEV;
}
max_sync_size = local_devdata->max_sync_size;
dev = local_devdata->dev;
/* Create the header */
hdr = (struct nx842_header *)out;
hdr->blocks_nr = PAGE_SIZE / max_sync_size;
hdrsize = nx842_header_size(hdr);
outbuf = (unsigned long)out + hdrsize;
bytesleft = *outlen - hdrsize;
/* Init scatterlist */
workmem = (struct nx842_workmem *)ALIGN((unsigned long)wmem,
NX842_HW_PAGE_SIZE);
slin.entries = (struct nx842_slentry *)workmem->slin;
slout.entries = (struct nx842_slentry *)workmem->slout;
/* Init operation */
op.flags = NX842_OP_COMPRESS;
csbcpb = &workmem->csbcpb;
memset(csbcpb, 0, sizeof(*csbcpb));
op.csbcpb = nx842_get_pa(csbcpb);
op.out = nx842_get_pa(slout.entries);
for (i = 0; i < hdr->blocks_nr; i++) {
/*
* Aligning the output blocks to 128 bytes does waste space,
* but it prevents the need for bounce buffers and memory
* copies. It also simplifies the code a lot. In the worst
* case (64k page, 4k max_sync_size), you lose up to
* (128*16)/64k = ~3% the compression factor. For 64k
* max_sync_size, the loss would be at most 128/64k = ~0.2%.
*/
padding = ALIGN(outbuf, IO_BUFFER_ALIGN) - outbuf;
outbuf += padding;
bytesleft -= padding;
if (i == 0)
/* save offset into first block in header */
hdr->offset = padding + hdrsize;
if (bytesleft <= 0) {
ret = -ENOSPC;
goto unlock;
}
/*
* NOTE: If the default max_sync_size is changed from 4k
* to 64k, remove the "likely" case below, since a
* scatterlist will always be needed.
*/
if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) {
/* Create direct DDE */
op.in = nx842_get_pa((void *)inbuf);
op.inlen = max_sync_size;
} else {
/* Create indirect DDE (scatterlist) */
nx842_build_scatterlist(inbuf, max_sync_size, &slin);
op.in = nx842_get_pa(slin.entries);
op.inlen = -nx842_get_scatterlist_size(&slin);
}
/*
* If max_sync_size != NX842_HW_PAGE_SIZE, an indirect
* DDE is required for the outbuf.
* If max_sync_size == NX842_HW_PAGE_SIZE, outbuf must
* also be page aligned (1 in 128/4k=32 chance) in order
* to use a direct DDE.
* This is unlikely, just use an indirect DDE always.
*/
nx842_build_scatterlist(outbuf,
min(bytesleft, max_sync_size), &slout);
/* op.out set before loop */
op.outlen = -nx842_get_scatterlist_size(&slout);
/* Send request to pHyp */
ret = vio_h_cop_sync(local_devdata->vdev, &op);
/* Check for pHyp error */
if (ret) {
dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
__func__, ret, op.hcall_err);
ret = -EIO;
goto unlock;
}
/* Check for hardware error */
ret = nx842_validate_result(dev, &csbcpb->csb);
if (ret && ret != -ENOSPC)
goto unlock;
/* Handle incompressible data */
if (unlikely(ret == -ENOSPC)) {
if (bytesleft < max_sync_size) {
/*
* Not enough space left in the output buffer
* to store uncompressed block
*/
goto unlock;
} else {
/* Store incompressible block */
memcpy((void *)outbuf, (void *)inbuf,
max_sync_size);
hdr->sizes[i] = -max_sync_size;
outbuf += max_sync_size;
bytesleft -= max_sync_size;
/* Reset ret, incompressible data handled */
ret = 0;
}
} else {
/* Normal case, compression was successful */
size = csbcpb->csb.processed_byte_count;
dev_dbg(dev, "%s: processed_bytes=%d\n",
__func__, size);
hdr->sizes[i] = size;
outbuf += size;
bytesleft -= size;
}
inbuf += max_sync_size;
}
*outlen = (unsigned int)(outbuf - (unsigned long)out);
unlock:
if (ret)
nx842_inc_comp_failed(local_devdata);
else {
nx842_inc_comp_complete(local_devdata);
ibm_nx842_incr_hist(local_devdata->counters->comp_times,
(get_tb() - start_time) / tb_ticks_per_usec);
}
rcu_read_unlock();
return ret;
}
EXPORT_SYMBOL_GPL(nx842_compress);
static int sw842_decompress(const unsigned char *, int, unsigned char *, int *,
const void *);
/**
* nx842_decompress - Decompress data using the 842 algorithm
*
* Decompression provide by the NX842 coprocessor on IBM Power systems.
* The input buffer is decompressed and the result is stored in the
* provided output buffer. The size allocated to the output buffer is
* provided by the caller of this function in @outlen. Upon return from
* this function @outlen contains the length of the decompressed data.
* If there is an error then @outlen will be 0 and an error will be
* specified by the return code from this function.
*
* @in: Pointer to input buffer, will use bounce buffer if not 128 byte
* aligned
* @inlen: Length of input buffer
* @out: Pointer to output buffer, must be page aligned
* @outlen: Length of output buffer, must be PAGE_SIZE
* @wrkmem: ptr to buffer for working memory, size determined by
* nx842_get_workmem_size()
*
* Returns:
* 0 Success, output of length @outlen stored in the buffer at @out
* -ENODEV Hardware decompression device is unavailable
* -ENOMEM Unable to allocate internal buffers
* -ENOSPC Output buffer is to small
* -EINVAL Bad input data encountered when attempting decompress
* -EIO Internal error
*/
int nx842_decompress(const unsigned char *in, unsigned int inlen,
unsigned char *out, unsigned int *outlen, void *wmem)
{
struct nx842_header *hdr;
struct nx842_devdata *local_devdata;
struct device *dev = NULL;
struct nx842_workmem *workmem;
struct nx842_scatterlist slin, slout;
struct nx_csbcpb *csbcpb;
int ret = 0, i, size, max_sync_size;
unsigned long inbuf, outbuf;
struct vio_pfo_op op = {
.done = NULL,
.handle = 0,
.timeout = 0,
};
unsigned long start_time = get_tb();
/* Ensure page alignment and size */
outbuf = (unsigned long)out;
if (!IS_ALIGNED(outbuf, PAGE_SIZE) || *outlen != PAGE_SIZE)
return -EINVAL;
rcu_read_lock();
local_devdata = rcu_dereference(devdata);
if (local_devdata)
dev = local_devdata->dev;
/* Get header */
hdr = (struct nx842_header *)in;
workmem = (struct nx842_workmem *)ALIGN((unsigned long)wmem,
NX842_HW_PAGE_SIZE);
inbuf = (unsigned long)in + hdr->offset;
if (likely(!IS_ALIGNED(inbuf, IO_BUFFER_ALIGN))) {
/* Copy block(s) into bounce buffer for alignment */
memcpy(workmem->bounce, in + hdr->offset, inlen - hdr->offset);
inbuf = (unsigned long)workmem->bounce;
}
/* Init scatterlist */
slin.entries = (struct nx842_slentry *)workmem->slin;
slout.entries = (struct nx842_slentry *)workmem->slout;
/* Init operation */
op.flags = NX842_OP_DECOMPRESS;
csbcpb = &workmem->csbcpb;
memset(csbcpb, 0, sizeof(*csbcpb));
op.csbcpb = nx842_get_pa(csbcpb);
/*
* max_sync_size may have changed since compression,
* so we can't read it from the device info. We need
* to derive it from hdr->blocks_nr.
*/
max_sync_size = PAGE_SIZE / hdr->blocks_nr;
for (i = 0; i < hdr->blocks_nr; i++) {
/* Skip padding */
inbuf = ALIGN(inbuf, IO_BUFFER_ALIGN);
if (hdr->sizes[i] < 0) {
/* Negative sizes indicate uncompressed data blocks */
size = abs(hdr->sizes[i]);
memcpy((void *)outbuf, (void *)inbuf, size);
outbuf += size;
inbuf += size;
continue;
}
if (!dev)
goto sw;
/*
* The better the compression, the more likely the "likely"
* case becomes.
*/
if (likely((inbuf & NX842_HW_PAGE_MASK) ==
((inbuf + hdr->sizes[i] - 1) & NX842_HW_PAGE_MASK))) {
/* Create direct DDE */
op.in = nx842_get_pa((void *)inbuf);
op.inlen = hdr->sizes[i];
} else {
/* Create indirect DDE (scatterlist) */
nx842_build_scatterlist(inbuf, hdr->sizes[i] , &slin);
op.in = nx842_get_pa(slin.entries);
op.inlen = -nx842_get_scatterlist_size(&slin);
}
/*
* NOTE: If the default max_sync_size is changed from 4k
* to 64k, remove the "likely" case below, since a
* scatterlist will always be needed.
*/
if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) {
/* Create direct DDE */
op.out = nx842_get_pa((void *)outbuf);
op.outlen = max_sync_size;
} else {
/* Create indirect DDE (scatterlist) */
nx842_build_scatterlist(outbuf, max_sync_size, &slout);
op.out = nx842_get_pa(slout.entries);
op.outlen = -nx842_get_scatterlist_size(&slout);
}
/* Send request to pHyp */
ret = vio_h_cop_sync(local_devdata->vdev, &op);
/* Check for pHyp error */
if (ret) {
dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
__func__, ret, op.hcall_err);
dev = NULL;
goto sw;
}
/* Check for hardware error */
ret = nx842_validate_result(dev, &csbcpb->csb);
if (ret) {
dev = NULL;
goto sw;
}
/* HW decompression success */
inbuf += hdr->sizes[i];
outbuf += csbcpb->csb.processed_byte_count;
continue;
sw:
/* software decompression */
size = max_sync_size;
ret = sw842_decompress(
(unsigned char *)inbuf, hdr->sizes[i],
(unsigned char *)outbuf, &size, wmem);
if (ret)
pr_debug("%s: sw842_decompress failed with %d\n",
__func__, ret);
if (ret) {
if (ret != -ENOSPC && ret != -EINVAL &&
ret != -EMSGSIZE)
ret = -EIO;
goto unlock;
}
/* SW decompression success */
inbuf += hdr->sizes[i];
outbuf += size;
}
*outlen = (unsigned int)(outbuf - (unsigned long)out);
unlock:
if (ret)
/* decompress fail */
nx842_inc_decomp_failed(local_devdata);
else {
if (!dev)
/* software decompress */
nx842_inc_swdecomp(local_devdata);
nx842_inc_decomp_complete(local_devdata);
ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
(get_tb() - start_time) / tb_ticks_per_usec);
}
rcu_read_unlock();
return ret;
}
EXPORT_SYMBOL_GPL(nx842_decompress);
/**
* nx842_OF_set_defaults -- Set default (disabled) values for devdata
*
* @devdata - struct nx842_devdata to update
*
* Returns:
* 0 on success
* -ENOENT if @devdata ptr is NULL
*/
static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
{
if (devdata) {
devdata->max_sync_size = 0;
devdata->max_sync_sg = 0;
devdata->max_sg_len = 0;
devdata->status = UNAVAILABLE;
return 0;
} else
return -ENOENT;
}
/**
* nx842_OF_upd_status -- Update the device info from OF status prop
*
* The status property indicates if the accelerator is enabled. If the
* device is in the OF tree it indicates that the hardware is present.
* The status field indicates if the device is enabled when the status
* is 'okay'. Otherwise the device driver will be disabled.
*
* @devdata - struct nx842_devdata to update
* @prop - struct property point containing the maxsyncop for the update
*
* Returns:
* 0 - Device is available
* -EINVAL - Device is not available
*/
static int nx842_OF_upd_status(struct nx842_devdata *devdata,
struct property *prop) {
int ret = 0;
const char *status = (const char *)prop->value;
if (!strncmp(status, "okay", (size_t)prop->length)) {
devdata->status = AVAILABLE;
} else {
dev_info(devdata->dev, "%s: status '%s' is not 'okay'\n",
__func__, status);
devdata->status = UNAVAILABLE;
}
return ret;
}
/**
* nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
*
* Definition of the 'ibm,max-sg-len' OF property:
* This field indicates the maximum byte length of a scatter list
* for the platform facility. It is a single cell encoded as with encode-int.
*
* Example:
* # od -x ibm,max-sg-len
* 0000000 0000 0ff0
*
* In this example, the maximum byte length of a scatter list is
* 0x0ff0 (4,080).
*
* @devdata - struct nx842_devdata to update
* @prop - struct property point containing the maxsyncop for the update
*
* Returns:
* 0 on success
* -EINVAL on failure
*/
static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
struct property *prop) {
int ret = 0;
const int *maxsglen = prop->value;
if (prop->length != sizeof(*maxsglen)) {
dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
prop->length, sizeof(*maxsglen));
ret = -EINVAL;
} else {
devdata->max_sg_len = (unsigned int)min(*maxsglen,
(int)NX842_HW_PAGE_SIZE);
}
return ret;
}
