/**
* nx_hcall_sync - make an H_COP_OP hcall for the passed in op structure
*
* @nx_ctx: the crypto context handle
* @op: PFO operation struct to pass in
* @may_sleep: flag indicating the request can sleep
*
* Make the hcall, retrying while the hardware is busy. If we cannot yield
* the thread, limit the number of retries to 10 here.
*/
int nx_hcall_sync(struct nx_crypto_ctx *nx_ctx,
struct vio_pfo_op *op,
u32 may_sleep)
{
int rc, retries = 10;
struct vio_dev *viodev = nx_driver.viodev;
atomic_inc(&(nx_ctx->stats->sync_ops));
do {
rc = vio_h_cop_sync(viodev, op);
} while ((rc == -EBUSY && !may_sleep && retries--) ||
(rc == -EBUSY && may_sleep && cond_resched()));
if (rc) {
dev_dbg(&viodev->dev, "vio_h_cop_sync failed: rc: %d "
"hcall rc: %ld\n", rc, op->hcall_err);
atomic_inc(&(nx_ctx->stats->errors));
atomic_set(&(nx_ctx->stats->last_error), op->hcall_err);
atomic_set(&(nx_ctx->stats->last_error_pid), current->pid);
}
return rc;
}
/**
* nx842_pseries_compress - Compress data using the 842 algorithm
*
* Compression provide by the NX842 coprocessor on IBM Power systems.
* The input buffer is compressed and the result is stored in the
* provided output buffer.
*
* Upon return from this function @outlen contains the length of the
* compressed data. If there is an error then @outlen will be 0 and an
* error will be specified by the return code from this function.
*
* @in: Pointer to input buffer
* @inlen: Length of input buffer
* @out: Pointer to output buffer
* @outlen: Length of output buffer
* @wrkmem: ptr to buffer for working memory, size determined by
* nx842_pseries_driver.workmem_size
*
* Returns:
* 0 Success, output of length @outlen stored in the buffer at @out
* -ENOMEM Unable to allocate internal buffers
* -ENOSPC Output buffer is to small
* -EIO Internal error
* -ENODEV Hardware unavailable
*/
static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen,
unsigned char *out, unsigned int *outlen,
void *wmem)
{
struct nx842_devdata *local_devdata;
struct device *dev = NULL;
struct nx842_workmem *workmem;
struct nx842_scatterlist slin, slout;
struct nx_csbcpb *csbcpb;
int ret = 0, max_sync_size;
unsigned long inbuf, outbuf;
struct vio_pfo_op op = {
.done = NULL,
.handle = 0,
.timeout = 0,
};
unsigned long start = get_tb();
inbuf = (unsigned long)in;
if (check_constraints(inbuf, &inlen, true))
return -EINVAL;
outbuf = (unsigned long)out;
if (check_constraints(outbuf, outlen, false))
return -EINVAL;
rcu_read_lock();
local_devdata = rcu_dereference(devdata);
if (!local_devdata || !local_devdata->dev) {
rcu_read_unlock();
return -ENODEV;
}
max_sync_size = local_devdata->max_sync_size;
dev = local_devdata->dev;
/* Init scatterlist */
workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
slin.entries = (struct nx842_slentry *)workmem->slin;
slout.entries = (struct nx842_slentry *)workmem->slout;
/* Init operation */
op.flags = NX842_OP_COMPRESS;
csbcpb = &workmem->csbcpb;
memset(csbcpb, 0, sizeof(*csbcpb));
op.csbcpb = nx842_get_pa(csbcpb);
if ((inbuf & NX842_HW_PAGE_MASK) ==
((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
/* Create direct DDE */
op.in = nx842_get_pa((void *)inbuf);
op.inlen = inlen;
} else {
/* Create indirect DDE (scatterlist) */
nx842_build_scatterlist(inbuf, inlen, &slin);
op.in = nx842_get_pa(slin.entries);
op.inlen = -nx842_get_scatterlist_size(&slin);
}
if ((outbuf & NX842_HW_PAGE_MASK) ==
((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
/* Create direct DDE */
op.out = nx842_get_pa((void *)outbuf);
op.outlen = *outlen;
} else {
/* Create indirect DDE (scatterlist) */
nx842_build_scatterlist(outbuf, *outlen, &slout);
op.out = nx842_get_pa(slout.entries);
op.outlen = -nx842_get_scatterlist_size(&slout);
}
dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
__func__, (unsigned long)op.in, (long)op.inlen,
(unsigned long)op.out, (long)op.outlen);
/* Send request to pHyp */
ret = vio_h_cop_sync(local_devdata->vdev, &op);
/* Check for pHyp error */
if (ret) {
dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
__func__, ret, op.hcall_err);
ret = -EIO;
goto unlock;
}
/* Check for hardware error */
ret = nx842_validate_result(dev, &csbcpb->csb);
if (ret)
goto unlock;
*outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen);
unlock:
if (ret)
nx842_inc_comp_failed(local_devdata);
else {
nx842_inc_comp_complete(local_devdata);
ibm_nx842_incr_hist(local_devdata->counters->comp_times,
(get_tb() - start) / tb_ticks_per_usec);
}
rcu_read_unlock();
return ret;
}
/**
* nx842_pseries_decompress - Decompress data using the 842 algorithm
*
* Decompression provide by the NX842 coprocessor on IBM Power systems.
* The input buffer is decompressed and the result is stored in the
* provided output buffer. The size allocated to the output buffer is
* provided by the caller of this function in @outlen. Upon return from
* this function @outlen contains the length of the decompressed data.
* If there is an error then @outlen will be 0 and an error will be
* specified by the return code from this function.
*
* @in: Pointer to input buffer
* @inlen: Length of input buffer
* @out: Pointer to output buffer
* @outlen: Length of output buffer
* @wrkmem: ptr to buffer for working memory, size determined by
* nx842_pseries_driver.workmem_size
*
* Returns:
* 0 Success, output of length @outlen stored in the buffer at @out
* -ENODEV Hardware decompression device is unavailable
* -ENOMEM Unable to allocate internal buffers
* -ENOSPC Output buffer is to small
* -EINVAL Bad input data encountered when attempting decompress
* -EIO Internal error
*/
static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen,
unsigned char *out, unsigned int *outlen,
void *wmem)
{
struct nx842_devdata *local_devdata;
struct device *dev = NULL;
struct nx842_workmem *workmem;
struct nx842_scatterlist slin, slout;
struct nx_csbcpb *csbcpb;
int ret = 0, max_sync_size;
unsigned long inbuf, outbuf;
struct vio_pfo_op op = {
.done = NULL,
.handle = 0,
.timeout = 0,
};
unsigned long start = get_tb();
/* Ensure page alignment and size */
inbuf = (unsigned long)in;
if (check_constraints(inbuf, &inlen, true))
return -EINVAL;
outbuf = (unsigned long)out;
if (check_constraints(outbuf, outlen, false))
return -EINVAL;
rcu_read_lock();
local_devdata = rcu_dereference(devdata);
if (!local_devdata || !local_devdata->dev) {
rcu_read_unlock();
return -ENODEV;
}
max_sync_size = local_devdata->max_sync_size;
dev = local_devdata->dev;
workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
/* Init scatterlist */
slin.entries = (struct nx842_slentry *)workmem->slin;
slout.entries = (struct nx842_slentry *)workmem->slout;
/* Init operation */
op.flags = NX842_OP_DECOMPRESS;
csbcpb = &workmem->csbcpb;
memset(csbcpb, 0, sizeof(*csbcpb));
op.csbcpb = nx842_get_pa(csbcpb);
if ((inbuf & NX842_HW_PAGE_MASK) ==
((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
/* Create direct DDE */
op.in = nx842_get_pa((void *)inbuf);
op.inlen = inlen;
} else {
/* Create indirect DDE (scatterlist) */
nx842_build_scatterlist(inbuf, inlen, &slin);
op.in = nx842_get_pa(slin.entries);
op.inlen = -nx842_get_scatterlist_size(&slin);
}
if ((outbuf & NX842_HW_PAGE_MASK) ==
((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
/* Create direct DDE */
op.out = nx842_get_pa((void *)outbuf);
op.outlen = *outlen;
} else {
/* Create indirect DDE (scatterlist) */
nx842_build_scatterlist(outbuf, *outlen, &slout);
op.out = nx842_get_pa(slout.entries);
op.outlen = -nx842_get_scatterlist_size(&slout);
}
dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
__func__, (unsigned long)op.in, (long)op.inlen,
(unsigned long)op.out, (long)op.outlen);
/* Send request to pHyp */
ret = vio_h_cop_sync(local_devdata->vdev, &op);
/* Check for pHyp error */
if (ret) {
dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
__func__, ret, op.hcall_err);
goto unlock;
}
/* Check for hardware error */
ret = nx842_validate_result(dev, &csbcpb->csb);
if (ret)
goto unlock;
*outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
unlock:
if (ret)
/* decompress fail */
nx842_inc_decomp_failed(local_devdata);
else {
nx842_inc_decomp_complete(local_devdata);
ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
(get_tb() - start) / tb_ticks_per_usec);
}
rcu_read_unlock();
return ret;
}
/**
* nx842_OF_set_defaults -- Set default (disabled) values for devdata
*
* @devdata - struct nx842_devdata to update
*
* Returns:
* 0 on success
* -ENOENT if @devdata ptr is NULL
*/
static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
{
if (devdata) {
devdata->max_sync_size = 0;
devdata->max_sync_sg = 0;
devdata->max_sg_len = 0;
return 0;
} else
return -ENOENT;
}
/**
* nx842_OF_upd_status -- Check the device info from OF status prop
*
* The status property indicates if the accelerator is enabled. If the
* device is in the OF tree it indicates that the hardware is present.
* The status field indicates if the device is enabled when the status
* is 'okay'. Otherwise the device driver will be disabled.
*
* @prop - struct property point containing the maxsyncop for the update
*
* Returns:
* 0 - Device is available
* -ENODEV - Device is not available
*/
static int nx842_OF_upd_status(struct property *prop)
{
const char *status = (const char *)prop->value;
if (!strncmp(status, "okay", (size_t)prop->length))
return 0;
if (!strncmp(status, "disabled", (size_t)prop->length))
return -ENODEV;
dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status);
return -EINVAL;
}
/**
* nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
*
* Definition of the 'ibm,max-sg-len' OF property:
* This field indicates the maximum byte length of a scatter list
* for the platform facility. It is a single cell encoded as with encode-int.
*
* Example:
* # od -x ibm,max-sg-len
* 0000000 0000 0ff0
*
* In this example, the maximum byte length of a scatter list is
* 0x0ff0 (4,080).
*
* @devdata - struct nx842_devdata to update
* @prop - struct property point containing the maxsyncop for the update
*
* Returns:
* 0 on success
* -EINVAL on failure
*/
static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
struct property *prop) {
int ret = 0;
const unsigned int maxsglen = of_read_number(prop->value, 1);
if (prop->length != sizeof(maxsglen)) {
dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
prop->length, sizeof(maxsglen));
ret = -EINVAL;
} else {
devdata->max_sg_len = min_t(unsigned int,
maxsglen, NX842_HW_PAGE_SIZE);
}
return ret;
}
/**
* nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop
*
* Definition of the 'ibm,max-sync-cop' OF property:
* Two series of cells. The first series of cells represents the maximums
* that can be synchronously compressed. The second series of cells
* represents the maximums that can be synchronously decompressed.
* 1. The first cell in each series contains the count of the number of
* data length, scatter list elements pairs that follow – each being
* of the form
* a. One cell data byte length
* b. One cell total number of scatter list elements
*
* Example:
* # od -x ibm,max-sync-cop
* 0000000 0000 0001 0000 1000 0000 01fe 0000 0001
* 0000020 0000 1000 0000 01fe
*
* In this example, compression supports 0x1000 (4,096) data byte length
* and 0x1fe (510) total scatter list elements. Decompression supports
* 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list
* elements.
*
* @devdata - struct nx842_devdata to update
* @prop - struct property point containing the maxsyncop for the update
*
* Returns:
* 0 on success
* -EINVAL on failure
*/
static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
struct property *prop) {
int ret = 0;
unsigned int comp_data_limit, decomp_data_limit;
unsigned int comp_sg_limit, decomp_sg_limit;
const struct maxsynccop_t {
__be32 comp_elements;
__be32 comp_data_limit;
__be32 comp_sg_limit;
__be32 decomp_elements;
__be32 decomp_data_limit;
__be32 decomp_sg_limit;
} *maxsynccop;
if (prop->length != sizeof(*maxsynccop)) {
dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);
dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,
sizeof(*maxsynccop));
ret = -EINVAL;
goto out;
}
maxsynccop = (const struct maxsynccop_t *)prop->value;
comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit);
comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit);
decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit);
decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit);
/* Use one limit rather than separate limits for compression and
* decompression. Set a maximum for this so as not to exceed the
* size that the header can support and round the value down to
* the hardware page size (4K) */
devdata->max_sync_size = min(comp_data_limit, decomp_data_limit);
devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,
65536);
if (devdata->max_sync_size < 4096) {
dev_err(devdata->dev, "%s: hardware max data size (%u) is "
"less than the driver minimum, unable to use "
"the hardware device\n",
__func__, devdata->max_sync_size);
ret = -EINVAL;
goto out;
}
nx842_pseries_constraints.maximum = devdata->max_sync_size;
devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit);
if (devdata->max_sync_sg < 1) {
dev_err(devdata->dev, "%s: hardware max sg size (%u) is "
"less than the driver minimum, unable to use "
"the hardware device\n",
__func__, devdata->max_sync_sg);
ret = -EINVAL;
goto out;
}
out:
return ret;
}
/**
* nx842_compress - Compress data using the 842 algorithm
*
* Compression provide by the NX842 coprocessor on IBM Power systems.
* The input buffer is compressed and the result is stored in the
* provided output buffer.
*
* Upon return from this function @outlen contains the length of the
* compressed data. If there is an error then @outlen will be 0 and an
* error will be specified by the return code from this function.
*
* @in: Pointer to input buffer, must be page aligned
* @inlen: Length of input buffer, must be PAGE_SIZE
* @out: Pointer to output buffer
* @outlen: Length of output buffer
* @wrkmem: ptr to buffer for working memory, size determined by
* nx842_get_workmem_size()
*
* Returns:
* 0 Success, output of length @outlen stored in the buffer at @out
* -ENOMEM Unable to allocate internal buffers
* -ENOSPC Output buffer is to small
* -EMSGSIZE XXX Difficult to describe this limitation
* -EIO Internal error
* -ENODEV Hardware unavailable
*/
int nx842_compress(const unsigned char *in, unsigned int inlen,
unsigned char *out, unsigned int *outlen, void *wmem)
{
struct nx842_header *hdr;
struct nx842_devdata *local_devdata;
struct device *dev = NULL;
struct nx842_workmem *workmem;
struct nx842_scatterlist slin, slout;
struct nx_csbcpb *csbcpb;
int ret = 0, max_sync_size, i, bytesleft, size, hdrsize;
unsigned long inbuf, outbuf, padding;
struct vio_pfo_op op = {
.done = NULL,
.handle = 0,
.timeout = 0,
};
unsigned long start_time = get_tb();
/*
* Make sure input buffer is 64k page aligned. This is assumed since
* this driver is designed for page compression only (for now). This
* is very nice since we can now use direct DDE(s) for the input and
* the alignment is guaranteed.
*/
inbuf = (unsigned long)in;
if (!IS_ALIGNED(inbuf, PAGE_SIZE) || inlen != PAGE_SIZE)
return -EINVAL;
rcu_read_lock();
local_devdata = rcu_dereference(devdata);
if (!local_devdata || !local_devdata->dev) {
rcu_read_unlock();
return -ENODEV;
}
max_sync_size = local_devdata->max_sync_size;
dev = local_devdata->dev;
/* Create the header */
hdr = (struct nx842_header *)out;
hdr->blocks_nr = PAGE_SIZE / max_sync_size;
hdrsize = nx842_header_size(hdr);
outbuf = (unsigned long)out + hdrsize;
bytesleft = *outlen - hdrsize;
/* Init scatterlist */
workmem = (struct nx842_workmem *)ALIGN((unsigned long)wmem,
NX842_HW_PAGE_SIZE);
slin.entries = (struct nx842_slentry *)workmem->slin;
slout.entries = (struct nx842_slentry *)workmem->slout;
/* Init operation */
op.flags = NX842_OP_COMPRESS;
csbcpb = &workmem->csbcpb;
memset(csbcpb, 0, sizeof(*csbcpb));
op.csbcpb = nx842_get_pa(csbcpb);
op.out = nx842_get_pa(slout.entries);
for (i = 0; i < hdr->blocks_nr; i++) {
/*
* Aligning the output blocks to 128 bytes does waste space,
* but it prevents the need for bounce buffers and memory
* copies. It also simplifies the code a lot. In the worst
* case (64k page, 4k max_sync_size), you lose up to
* (128*16)/64k = ~3% the compression factor. For 64k
* max_sync_size, the loss would be at most 128/64k = ~0.2%.
*/
padding = ALIGN(outbuf, IO_BUFFER_ALIGN) - outbuf;
outbuf += padding;
bytesleft -= padding;
if (i == 0)
/* save offset into first block in header */
hdr->offset = padding + hdrsize;
if (bytesleft <= 0) {
ret = -ENOSPC;
goto unlock;
}
/*
* NOTE: If the default max_sync_size is changed from 4k
* to 64k, remove the "likely" case below, since a
* scatterlist will always be needed.
*/
if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) {
/* Create direct DDE */
op.in = nx842_get_pa((void *)inbuf);
op.inlen = max_sync_size;
} else {
/* Create indirect DDE (scatterlist) */
nx842_build_scatterlist(inbuf, max_sync_size, &slin);
op.in = nx842_get_pa(slin.entries);
op.inlen = -nx842_get_scatterlist_size(&slin);
}
/*
* If max_sync_size != NX842_HW_PAGE_SIZE, an indirect
* DDE is required for the outbuf.
* If max_sync_size == NX842_HW_PAGE_SIZE, outbuf must
* also be page aligned (1 in 128/4k=32 chance) in order
* to use a direct DDE.
* This is unlikely, just use an indirect DDE always.
*/
nx842_build_scatterlist(outbuf,
min(bytesleft, max_sync_size), &slout);
/* op.out set before loop */
op.outlen = -nx842_get_scatterlist_size(&slout);
/* Send request to pHyp */
ret = vio_h_cop_sync(local_devdata->vdev, &op);
/* Check for pHyp error */
if (ret) {
dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
__func__, ret, op.hcall_err);
ret = -EIO;
goto unlock;
}
/* Check for hardware error */
ret = nx842_validate_result(dev, &csbcpb->csb);
if (ret && ret != -ENOSPC)
goto unlock;
/* Handle incompressible data */
if (unlikely(ret == -ENOSPC)) {
if (bytesleft < max_sync_size) {
/*
* Not enough space left in the output buffer
* to store uncompressed block
*/
goto unlock;
} else {
/* Store incompressible block */
memcpy((void *)outbuf, (void *)inbuf,
max_sync_size);
hdr->sizes[i] = -max_sync_size;
outbuf += max_sync_size;
bytesleft -= max_sync_size;
/* Reset ret, incompressible data handled */
ret = 0;
}
} else {
/* Normal case, compression was successful */
size = csbcpb->csb.processed_byte_count;
dev_dbg(dev, "%s: processed_bytes=%d\n",
__func__, size);
hdr->sizes[i] = size;
outbuf += size;
bytesleft -= size;
}
inbuf += max_sync_size;
}
*outlen = (unsigned int)(outbuf - (unsigned long)out);
unlock:
if (ret)
nx842_inc_comp_failed(local_devdata);
else {
nx842_inc_comp_complete(local_devdata);
ibm_nx842_incr_hist(local_devdata->counters->comp_times,
(get_tb() - start_time) / tb_ticks_per_usec);
}
rcu_read_unlock();
return ret;
}
EXPORT_SYMBOL_GPL(nx842_compress);
static int sw842_decompress(const unsigned char *, int, unsigned char *, int *,
const void *);
/**
* nx842_decompress - Decompress data using the 842 algorithm
*
* Decompression provide by the NX842 coprocessor on IBM Power systems.
* The input buffer is decompressed and the result is stored in the
* provided output buffer. The size allocated to the output buffer is
* provided by the caller of this function in @outlen. Upon return from
* this function @outlen contains the length of the decompressed data.
* If there is an error then @outlen will be 0 and an error will be
* specified by the return code from this function.
*
* @in: Pointer to input buffer, will use bounce buffer if not 128 byte
* aligned
* @inlen: Length of input buffer
* @out: Pointer to output buffer, must be page aligned
* @outlen: Length of output buffer, must be PAGE_SIZE
* @wrkmem: ptr to buffer for working memory, size determined by
* nx842_get_workmem_size()
*
* Returns:
* 0 Success, output of length @outlen stored in the buffer at @out
* -ENODEV Hardware decompression device is unavailable
* -ENOMEM Unable to allocate internal buffers
* -ENOSPC Output buffer is to small
* -EINVAL Bad input data encountered when attempting decompress
* -EIO Internal error
*/
int nx842_decompress(const unsigned char *in, unsigned int inlen,
unsigned char *out, unsigned int *outlen, void *wmem)
{
struct nx842_header *hdr;
struct nx842_devdata *local_devdata;
struct device *dev = NULL;
struct nx842_workmem *workmem;
struct nx842_scatterlist slin, slout;
struct nx_csbcpb *csbcpb;
int ret = 0, i, size, max_sync_size;
unsigned long inbuf, outbuf;
struct vio_pfo_op op = {
.done = NULL,
.handle = 0,
.timeout = 0,
};
unsigned long start_time = get_tb();
/* Ensure page alignment and size */
outbuf = (unsigned long)out;
if (!IS_ALIGNED(outbuf, PAGE_SIZE) || *outlen != PAGE_SIZE)
return -EINVAL;
rcu_read_lock();
local_devdata = rcu_dereference(devdata);
if (local_devdata)
dev = local_devdata->dev;
/* Get header */
hdr = (struct nx842_header *)in;
workmem = (struct nx842_workmem *)ALIGN((unsigned long)wmem,
NX842_HW_PAGE_SIZE);
inbuf = (unsigned long)in + hdr->offset;
if (likely(!IS_ALIGNED(inbuf, IO_BUFFER_ALIGN))) {
/* Copy block(s) into bounce buffer for alignment */
memcpy(workmem->bounce, in + hdr->offset, inlen - hdr->offset);
inbuf = (unsigned long)workmem->bounce;
}
/* Init scatterlist */
slin.entries = (struct nx842_slentry *)workmem->slin;
slout.entries = (struct nx842_slentry *)workmem->slout;
/* Init operation */
op.flags = NX842_OP_DECOMPRESS;
csbcpb = &workmem->csbcpb;
memset(csbcpb, 0, sizeof(*csbcpb));
op.csbcpb = nx842_get_pa(csbcpb);
/*
* max_sync_size may have changed since compression,
* so we can't read it from the device info. We need
* to derive it from hdr->blocks_nr.
*/
max_sync_size = PAGE_SIZE / hdr->blocks_nr;
for (i = 0; i < hdr->blocks_nr; i++) {
/* Skip padding */
inbuf = ALIGN(inbuf, IO_BUFFER_ALIGN);
if (hdr->sizes[i] < 0) {
/* Negative sizes indicate uncompressed data blocks */
size = abs(hdr->sizes[i]);
memcpy((void *)outbuf, (void *)inbuf, size);
outbuf += size;
inbuf += size;
continue;
}
if (!dev)
goto sw;
/*
* The better the compression, the more likely the "likely"
* case becomes.
*/
if (likely((inbuf & NX842_HW_PAGE_MASK) ==
((inbuf + hdr->sizes[i] - 1) & NX842_HW_PAGE_MASK))) {
/* Create direct DDE */
op.in = nx842_get_pa((void *)inbuf);
op.inlen = hdr->sizes[i];
} else {
/* Create indirect DDE (scatterlist) */
nx842_build_scatterlist(inbuf, hdr->sizes[i] , &slin);
op.in = nx842_get_pa(slin.entries);
op.inlen = -nx842_get_scatterlist_size(&slin);
}
/*
* NOTE: If the default max_sync_size is changed from 4k
* to 64k, remove the "likely" case below, since a
* scatterlist will always be needed.
*/
if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) {
/* Create direct DDE */
op.out = nx842_get_pa((void *)outbuf);
op.outlen = max_sync_size;
} else {
/* Create indirect DDE (scatterlist) */
nx842_build_scatterlist(outbuf, max_sync_size, &slout);
op.out = nx842_get_pa(slout.entries);
op.outlen = -nx842_get_scatterlist_size(&slout);
}
/* Send request to pHyp */
ret = vio_h_cop_sync(local_devdata->vdev, &op);
/* Check for pHyp error */
if (ret) {
dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
__func__, ret, op.hcall_err);
dev = NULL;
goto sw;
}
/* Check for hardware error */
ret = nx842_validate_result(dev, &csbcpb->csb);
if (ret) {
dev = NULL;
goto sw;
}
/* HW decompression success */
inbuf += hdr->sizes[i];
outbuf += csbcpb->csb.processed_byte_count;
continue;
sw:
/* software decompression */
size = max_sync_size;
ret = sw842_decompress(
(unsigned char *)inbuf, hdr->sizes[i],
(unsigned char *)outbuf, &size, wmem);
if (ret)
pr_debug("%s: sw842_decompress failed with %d\n",
__func__, ret);
if (ret) {
if (ret != -ENOSPC && ret != -EINVAL &&
ret != -EMSGSIZE)
ret = -EIO;
goto unlock;
}
/* SW decompression success */
inbuf += hdr->sizes[i];
outbuf += size;
}
*outlen = (unsigned int)(outbuf - (unsigned long)out);
unlock:
if (ret)
/* decompress fail */
nx842_inc_decomp_failed(local_devdata);
else {
if (!dev)
/* software decompress */
nx842_inc_swdecomp(local_devdata);
nx842_inc_decomp_complete(local_devdata);
ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
(get_tb() - start_time) / tb_ticks_per_usec);
}
rcu_read_unlock();
return ret;
}
EXPORT_SYMBOL_GPL(nx842_decompress);
/**
* nx842_OF_set_defaults -- Set default (disabled) values for devdata
*
* @devdata - struct nx842_devdata to update
*
* Returns:
* 0 on success
* -ENOENT if @devdata ptr is NULL
*/
static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
{
if (devdata) {
devdata->max_sync_size = 0;
devdata->max_sync_sg = 0;
devdata->max_sg_len = 0;
devdata->status = UNAVAILABLE;
return 0;
} else
return -ENOENT;
}
/**
* nx842_OF_upd_status -- Update the device info from OF status prop
*
* The status property indicates if the accelerator is enabled. If the
* device is in the OF tree it indicates that the hardware is present.
* The status field indicates if the device is enabled when the status
* is 'okay'. Otherwise the device driver will be disabled.
*
* @devdata - struct nx842_devdata to update
* @prop - struct property point containing the maxsyncop for the update
*
* Returns:
* 0 - Device is available
* -EINVAL - Device is not available
*/
static int nx842_OF_upd_status(struct nx842_devdata *devdata,
struct property *prop) {
int ret = 0;
const char *status = (const char *)prop->value;
if (!strncmp(status, "okay", (size_t)prop->length)) {
devdata->status = AVAILABLE;
} else {
dev_info(devdata->dev, "%s: status '%s' is not 'okay'\n",
__func__, status);
devdata->status = UNAVAILABLE;
}
return ret;
}
/**
* nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
*
* Definition of the 'ibm,max-sg-len' OF property:
* This field indicates the maximum byte length of a scatter list
* for the platform facility. It is a single cell encoded as with encode-int.
*
* Example:
* # od -x ibm,max-sg-len
* 0000000 0000 0ff0
*
* In this example, the maximum byte length of a scatter list is
* 0x0ff0 (4,080).
*
* @devdata - struct nx842_devdata to update
* @prop - struct property point containing the maxsyncop for the update
*
* Returns:
* 0 on success
* -EINVAL on failure
*/
static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
struct property *prop) {
int ret = 0;
const int *maxsglen = prop->value;
if (prop->length != sizeof(*maxsglen)) {
dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
prop->length, sizeof(*maxsglen));
ret = -EINVAL;
} else {
devdata->max_sg_len = (unsigned int)min(*maxsglen,
(int)NX842_HW_PAGE_SIZE);
}
return ret;
}