/* fetch the pages addr resides in into pg and initialise sg with them */ int __get_userbuf(uint8_t __user *addr, uint32_t len, int write, unsigned int pgcount, struct page **pg, struct scatterlist *sg, struct task_struct *task, struct mm_struct *mm) { int ret, pglen, i = 0; struct scatterlist *sgp; if (unlikely(!pgcount || !len || !addr)) { sg_mark_end(sg); return 0; } down_read(&mm->mmap_sem); ret = get_user_pages(task, mm, (unsigned long)addr, pgcount, write, 0, pg, NULL); up_read(&mm->mmap_sem); if (ret != pgcount) return -EINVAL; sg_init_table(sg, pgcount); pglen = min((ptrdiff_t)(PAGE_SIZE - PAGEOFFSET(addr)), (ptrdiff_t)len); sg_set_page(sg, pg[i++], pglen, PAGEOFFSET(addr)); len -= pglen; for (sgp = sg_next(sg); len; sgp = sg_next(sgp)) { pglen = min((uint32_t)PAGE_SIZE, len); sg_set_page(sgp, pg[i++], pglen, 0); len -= pglen; } sg_mark_end(sg_last(sg, pgcount)); return 0; }
/*---------------------------------------------------------------------------*/ static inline void xio_tbl_set_nents(struct sg_table *tbl, uint32_t nents) { struct scatterlist *sg; int i; #ifdef XIO_DEBUG_SG verify_tbl(tbl); #endif if (!tbl || tbl->orig_nents < nents) return; sg = tbl->sgl; /* tbl->nents is unsigned so if tbl->nents is ZERO then tbl->nents - 1 * is a huge number, so check this. */ if (tbl->nents && (tbl->nents < tbl->orig_nents)) { for (i = 0; i < tbl->nents - 1; i++) sg = sg_next(sg); sg_unmark_end(sg); } if (!nents) { tbl->nents = nents; return; } sg = tbl->sgl; for (i = 0; i < nents - 1; i++) sg = sg_next(sg); sg_mark_end(sg); tbl->nents = nents; }
/** * pack_sg_list_p - Just like pack_sg_list. Instead of taking a buffer, * this takes a list of pages. * @sg: scatter/gather list to pack into * @start: which segment of the sg_list to start at * @pdata: a list of pages to add into sg. * @nr_pages: number of pages to pack into the scatter/gather list * @data: data to pack into scatter/gather list * @count: amount of data to pack into the scatter/gather list */ static int pack_sg_list_p(struct scatterlist *sg, int start, int limit, struct page **pdata, int nr_pages, char *data, int count) { int i = 0, s; int data_off; int index = start; BUG_ON(nr_pages > (limit - start)); /* * if the first page doesn't start at * page boundary find the offset */ data_off = offset_in_page(data); while (nr_pages) { s = rest_of_page(data); if (s > count) s = count; /* Make sure we don't terminate early. */ sg_unmark_end(&sg[index]); sg_set_page(&sg[index++], pdata[i++], s, data_off); data_off = 0; data += s; count -= s; nr_pages--; } if (index-start) sg_mark_end(&sg[index - 1]); return index - start; }
static int tls_push_record(struct sock *sk, int flags, unsigned char record_type) { struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); int rc; sg_mark_end(ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem - 1); sg_mark_end(ctx->sg_encrypted_data + ctx->sg_encrypted_num_elem - 1); tls_make_aad(ctx->aad_space, ctx->sg_plaintext_size, tls_ctx->rec_seq, tls_ctx->rec_seq_size, record_type); tls_fill_prepend(tls_ctx, page_address(sg_page(&ctx->sg_encrypted_data[0])) + ctx->sg_encrypted_data[0].offset, ctx->sg_plaintext_size, record_type); tls_ctx->pending_open_record_frags = 0; set_bit(TLS_PENDING_CLOSED_RECORD, &tls_ctx->flags); rc = tls_do_encryption(tls_ctx, ctx, ctx->sg_plaintext_size, sk->sk_allocation); if (rc < 0) { /* If we are called from write_space and * we fail, we need to set this SOCK_NOSPACE * to trigger another write_space in the future. */ set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); return rc; } free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem, &ctx->sg_plaintext_size); ctx->sg_encrypted_num_elem = 0; ctx->sg_encrypted_size = 0; /* Only pass through MSG_DONTWAIT and MSG_NOSIGNAL flags */ rc = tls_push_sg(sk, tls_ctx, ctx->sg_encrypted_data, 0, flags); if (rc < 0 && rc != -EAGAIN) tls_err_abort(sk); tls_advance_record_sn(sk, tls_ctx); return rc; }
/** * sg_init_table - Initialize SG table * @sgl: The SG table * @nents: Number of entries in table * * Notes: * If this is part of a chained sg table, sg_mark_end() should be * used only on the last table part. * **/ void sg_init_table(struct scatterlist *sgl, unsigned int nents) { memset(sgl, 0, sizeof(*sgl) * nents); #ifdef CONFIG_DEBUG_SG { unsigned int i; for (i = 0; i < nents; i++) sgl[i].sg_magic = SG_MAGIC; } #endif sg_mark_end(&sgl[nents - 1]); }
int __sg_alloc_table(struct sg_table *table, unsigned int nents, unsigned int max_ents, gfp_t gfp_mask, sg_alloc_fn *alloc_fn) { struct scatterlist *sg, *prv; unsigned int left; #ifndef ARCH_HAS_SG_CHAIN BUG_ON(nents > max_ents); #endif memset(table, 0, sizeof(*table)); left = nents; prv = NULL; do { unsigned int sg_size, alloc_size = left; if (alloc_size > max_ents) { alloc_size = max_ents; sg_size = alloc_size - 1; } else sg_size = alloc_size; left -= sg_size; sg = alloc_fn(alloc_size, gfp_mask); if (unlikely(!sg)) { if (prv) table->nents = ++table->orig_nents; return -ENOMEM; } sg_init_table(sg, alloc_size); table->nents = table->orig_nents += sg_size; if (prv) sg_chain(prv, max_ents, sg); else table->sgl = sg; if (!left) sg_mark_end(&sg[sg_size - 1]); gfp_mask &= ~__GFP_WAIT; gfp_mask |= __GFP_HIGH; prv = sg; } while (left); return 0; }
static void eseqiv_chain(struct scatterlist *head, struct scatterlist *sg, int chain) { if (chain) { head->length += sg->length; sg = scatterwalk_sg_next(sg); } if (sg) scatterwalk_sg_chain(head, 2, sg); else sg_mark_end(head); }
static struct sg_table * huge_get_pages(struct drm_i915_gem_object *obj) { #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY) const unsigned long nreal = obj->scratch / PAGE_SIZE; const unsigned long npages = obj->base.size / PAGE_SIZE; struct scatterlist *sg, *src, *end; struct sg_table *pages; unsigned long n; pages = kmalloc(sizeof(*pages), GFP); if (!pages) return ERR_PTR(-ENOMEM); if (sg_alloc_table(pages, npages, GFP)) { kfree(pages); return ERR_PTR(-ENOMEM); } sg = pages->sgl; for (n = 0; n < nreal; n++) { struct page *page; page = alloc_page(GFP | __GFP_HIGHMEM); if (!page) { sg_mark_end(sg); goto err; } sg_set_page(sg, page, PAGE_SIZE, 0); sg = __sg_next(sg); } if (nreal < npages) { for (end = sg, src = pages->sgl; sg; sg = __sg_next(sg)) { sg_set_page(sg, sg_page(src), PAGE_SIZE, 0); src = __sg_next(src); if (src == end) src = pages->sgl; } } if (i915_gem_gtt_prepare_pages(obj, pages)) goto err; return pages; err: huge_free_pages(obj, pages); return ERR_PTR(-ENOMEM); #undef GFP }
/** * blk_bio_map_sg - map a bio to a scatterlist * @q: request_queue in question * @bio: bio being mapped * @sglist: scatterlist being mapped * * Note: * Caller must make sure sg can hold bio->bi_phys_segments entries * * Will return the number of sg entries setup */ int blk_bio_map_sg(struct request_queue *q, struct bio *bio, struct scatterlist *sglist) { struct scatterlist *sg = NULL; int nsegs; struct bio *next = bio->bi_next; bio->bi_next = NULL; nsegs = __blk_bios_map_sg(q, bio, sglist, &sg); bio->bi_next = next; if (sg) sg_mark_end(sg); BUG_ON(bio->bi_phys_segments && nsegs > bio->bi_phys_segments); return nsegs; }
static int sahara_walk_and_recalc(struct scatterlist *sg, unsigned int nbytes) { if (!sg || !sg->length) return nbytes; while (nbytes && sg) { if (nbytes <= sg->length) { sg->length = nbytes; sg_mark_end(sg); break; } nbytes -= sg->length; sg = sg_next(sg); } return nbytes; }
/** * blk_rq_map_integrity_sg - Map integrity metadata into a scatterlist * @q: request queue * @bio: bio with integrity metadata attached * @sglist: target scatterlist * * Description: Map the integrity vectors in request into a * scatterlist. The scatterlist must be big enough to hold all * elements. I.e. sized using blk_rq_count_integrity_sg(). */ int blk_rq_map_integrity_sg(struct request_queue *q, struct bio *bio, struct scatterlist *sglist) { struct bio_vec iv, ivprv = { NULL }; struct scatterlist *sg = NULL; unsigned int segments = 0; struct bvec_iter iter; int prev = 0; bio_for_each_integrity_vec(iv, bio, iter) { if (prev) { if (!BIOVEC_PHYS_MERGEABLE(&ivprv, &iv)) goto new_segment; if (!BIOVEC_SEG_BOUNDARY(q, &ivprv, &iv)) goto new_segment; if (sg->length + iv.bv_len > queue_max_segment_size(q)) goto new_segment; sg->length += iv.bv_len; } else { new_segment: if (!sg) sg = sglist; else { sg_unmark_end(sg); sg = sg_next(sg); } sg_set_page(sg, iv.bv_page, iv.bv_len, iv.bv_offset); segments++; } prev = 1; ivprv = iv; } if (sg) sg_mark_end(sg); return segments; }
static int map_skb(struct device *dev, const struct sk_buff *skb, struct mpodp_tx *tx) { const skb_frag_t *fp, *end; const struct skb_shared_info *si; int count = 1; dma_addr_t handler; sg_init_table(tx->sg, MAX_SKB_FRAGS + 1); handler = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE); if (dma_mapping_error(dev, handler)) goto out_err; sg_dma_address(&tx->sg[0]) = handler; sg_dma_len(&tx->sg[0]) = skb_headlen(skb); si = skb_shinfo(skb); end = &si->frags[si->nr_frags]; for (fp = si->frags; fp < end; fp++, count++) { handler = skb_frag_dma_map(dev, fp, 0, skb_frag_size(fp), DMA_TO_DEVICE); if (dma_mapping_error(dev, handler)) goto unwind; sg_dma_address(&tx->sg[count]) = handler; sg_dma_len(&tx->sg[count]) = skb_frag_size(fp); } sg_mark_end(&tx->sg[count - 1]); tx->sg_len = count; return 0; unwind: while (fp-- > si->frags) dma_unmap_page(dev, sg_dma_address(&tx->sg[--count]), skb_frag_size(fp), DMA_TO_DEVICE); dma_unmap_single(dev, sg_dma_address(&tx->sg[0]), skb_headlen(skb), DMA_TO_DEVICE); out_err: return -ENOMEM; }
/** * blk_rq_map_integrity_sg - Map integrity metadata into a scatterlist * @q: request queue * @bio: bio with integrity metadata attached * @sglist: target scatterlist * * Description: Map the integrity vectors in request into a * scatterlist. The scatterlist must be big enough to hold all * elements. I.e. sized using blk_rq_count_integrity_sg(). */ int blk_rq_map_integrity_sg(struct request_queue *q, struct bio *bio, struct scatterlist *sglist) { struct bio_vec *iv, *ivprv = NULL; struct scatterlist *sg = NULL; unsigned int segments = 0; unsigned int i = 0; bio_for_each_integrity_vec(iv, bio, i) { if (ivprv) { if (!BIOVEC_PHYS_MERGEABLE(ivprv, iv)) goto new_segment; if (!BIOVEC_SEG_BOUNDARY(q, ivprv, iv)) goto new_segment; if (sg->length + iv->bv_len > queue_max_segment_size(q)) goto new_segment; sg->length += iv->bv_len; } else { new_segment: if (!sg) sg = sglist; else { sg->page_link &= ~0x02; sg = sg_next(sg); } sg_set_page(sg, iv->bv_page, iv->bv_len, iv->bv_offset); segments++; } ivprv = iv; } if (sg) sg_mark_end(sg); return segments; }
static int pack_sg_list(struct scatterlist *sg, int start, int limit, char *data, int count) { int s; int index = start; while (count) { s = rest_of_page(data); if (s > count) s = count; BUG_ON(index > limit); /* Make sure we don't terminate early. */ sg_unmark_end(&sg[index]); sg_set_buf(&sg[index++], data, s); count -= s; data += s; } if (index-start) sg_mark_end(&sg[index - 1]); return index-start; }
/** * blk_rq_map_integrity_sg - Map integrity metadata into a scatterlist * @rq: request with integrity metadata attached * @sglist: target scatterlist * * Description: Map the integrity vectors in request into a * scatterlist. The scatterlist must be big enough to hold all * elements. I.e. sized using blk_rq_count_integrity_sg(). */ int blk_rq_map_integrity_sg(struct request *rq, struct scatterlist *sglist) { struct bio_vec *iv, *ivprv; struct req_iterator iter; struct scatterlist *sg; unsigned int segments; ivprv = NULL; sg = NULL; segments = 0; rq_for_each_integrity_segment(iv, rq, iter) { if (ivprv) { if (!BIOVEC_PHYS_MERGEABLE(ivprv, iv)) goto new_segment; sg->length += iv->bv_len; } else { new_segment: if (!sg) sg = sglist; else { sg->page_link &= ~0x02; sg = sg_next(sg); } sg_set_page(sg, iv->bv_page, iv->bv_len, iv->bv_offset); segments++; } ivprv = iv; } if (sg) sg_mark_end(sg); return segments; }
static void digest_data(struct hash_desc *hash, struct iscsi_cmnd *cmnd, struct tio *tio, u32 offset, u8 *crc) { struct scatterlist *sg = cmnd->conn->hash_sg; u32 size, length; int i, idx, count; unsigned int nbytes; size = cmnd->pdu.datasize; nbytes = size = (size + 3) & ~3; offset += tio->offset; idx = offset >> PAGE_CACHE_SHIFT; offset &= ~PAGE_CACHE_MASK; count = get_pgcnt(size, offset); assert(idx + count <= tio->pg_cnt); assert(count <= ISCSI_CONN_IOV_MAX); sg_init_table(sg, ARRAY_SIZE(cmnd->conn->hash_sg)); crypto_hash_init(hash); for (i = 0; size; i++) { if (offset + size > PAGE_CACHE_SIZE) length = PAGE_CACHE_SIZE - offset; else length = size; sg_set_page(&sg[i], tio->pvec[idx + i], length, offset); size -= length; offset = 0; } sg_mark_end(&sg[i - 1]); crypto_hash_update(hash, sg, nbytes); crypto_hash_final(hash, crc); }
static int omap_crypto_copy_sg_lists(int total, int bs, struct scatterlist **sg, struct scatterlist *new_sg, u16 flags) { int n = sg_nents(*sg); struct scatterlist *tmp; if (!(flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY)) { new_sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL); if (!new_sg) return -ENOMEM; sg_init_table(new_sg, n); } tmp = new_sg; while (*sg && total) { int len = (*sg)->length; if (total < len) len = total; if (len > 0) { total -= len; sg_set_page(tmp, sg_page(*sg), len, (*sg)->offset); if (total <= 0) sg_mark_end(tmp); tmp = sg_next(tmp); } *sg = sg_next(*sg); } *sg = new_sg; return 0; }
/* * map a request to scatterlist, return number of sg entries setup. Caller * must make sure sg can hold rq->nr_phys_segments entries */ int blk_rq_map_sg(struct request_queue *q, struct request *rq, struct scatterlist *sglist) { struct scatterlist *sg = NULL; int nsegs = 0; if (rq->bio) nsegs = __blk_bios_map_sg(q, rq->bio, sglist, &sg); if (unlikely(rq->cmd_flags & REQ_COPY_USER) && (blk_rq_bytes(rq) & q->dma_pad_mask)) { unsigned int pad_len = (q->dma_pad_mask & ~blk_rq_bytes(rq)) + 1; sg->length += pad_len; rq->extra_len += pad_len; } if (q->dma_drain_size && q->dma_drain_needed(rq)) { if (rq->cmd_flags & REQ_WRITE) memset(q->dma_drain_buffer, 0, q->dma_drain_size); sg->page_link &= ~0x02; sg = sg_next(sg); sg_set_page(sg, virt_to_page(q->dma_drain_buffer), q->dma_drain_size, ((unsigned long)q->dma_drain_buffer) & (PAGE_SIZE - 1)); nsegs++; rq->extra_len += q->dma_drain_size; } if (sg) sg_mark_end(sg); return nsegs; }
static int qce_ahash_update(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct qce_sha_reqctx *rctx = ahash_request_ctx(req); struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm); struct qce_device *qce = tmpl->qce; struct scatterlist *sg_last, *sg; unsigned int total, len; unsigned int hash_later; unsigned int nbytes; unsigned int blocksize; blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); rctx->count += req->nbytes; /* check for buffer from previous updates and append it */ total = req->nbytes + rctx->buflen; if (total <= blocksize) { scatterwalk_map_and_copy(rctx->buf + rctx->buflen, req->src, 0, req->nbytes, 0); rctx->buflen += req->nbytes; return 0; } /* save the original req structure fields */ rctx->src_orig = req->src; rctx->nbytes_orig = req->nbytes; /* * if we have data from previous update copy them on buffer. The old * data will be combined with current request bytes. */ if (rctx->buflen) memcpy(rctx->tmpbuf, rctx->buf, rctx->buflen); /* calculate how many bytes will be hashed later */ hash_later = total % blocksize; if (hash_later) { unsigned int src_offset = req->nbytes - hash_later; scatterwalk_map_and_copy(rctx->buf, req->src, src_offset, hash_later, 0); } /* here nbytes is multiple of blocksize */ nbytes = total - hash_later; len = rctx->buflen; sg = sg_last = req->src; while (len < nbytes && sg) { if (len + sg_dma_len(sg) > nbytes) break; len += sg_dma_len(sg); sg_last = sg; sg = sg_next(sg); } if (!sg_last) return -EINVAL; sg_mark_end(sg_last); if (rctx->buflen) { sg_init_table(rctx->sg, 2); sg_set_buf(rctx->sg, rctx->tmpbuf, rctx->buflen); sg_chain(rctx->sg, 2, req->src); req->src = rctx->sg; } req->nbytes = nbytes; rctx->buflen = hash_later; return qce->async_req_enqueue(tmpl->qce, &req->base); }
/** * __sg_alloc_table - Allocate and initialize an sg table with given allocator * @table: The sg table header to use * @nents: Number of entries in sg list * @max_ents: The maximum number of entries the allocator returns per call * @gfp_mask: GFP allocation mask * @alloc_fn: Allocator to use * * Description: * This function returns a @table @nents long. The allocator is * defined to return scatterlist chunks of maximum size @max_ents. * Thus if @nents is bigger than @max_ents, the scatterlists will be * chained in units of @max_ents. * * Notes: * If this function returns non-0 (eg failure), the caller must call * __sg_free_table() to cleanup any leftover allocations. * **/ int __sg_alloc_table(struct sg_table *table, unsigned int nents, unsigned int max_ents, gfp_t gfp_mask, sg_alloc_fn *alloc_fn) { struct scatterlist *sg, *prv; unsigned int left; unsigned int total_alloc = 0; #ifndef ARCH_HAS_SG_CHAIN BUG_ON(nents > max_ents); #endif memset(table, 0, sizeof(*table)); left = nents; prv = NULL; do { unsigned int sg_size, alloc_size = left; if (alloc_size > max_ents) { alloc_size = max_ents; sg_size = alloc_size - 1; } else sg_size = alloc_size; left -= sg_size; sg = alloc_fn(alloc_size, gfp_mask); if (unlikely(!sg)) { table->orig_nents = total_alloc; /* mark the end of previous entry */ sg_mark_end(&prv[alloc_size - 1]); return -ENOMEM; } total_alloc += alloc_size; sg_init_table(sg, alloc_size); table->nents = table->orig_nents += sg_size; /* * If this is the first mapping, assign the sg table header. * If this is not the first mapping, chain previous part. */ if (prv) sg_chain(prv, max_ents, sg); else table->sgl = sg; /* * If no more entries after this one, mark the end */ if (!left) sg_mark_end(&sg[sg_size - 1]); /* * only really needed for mempool backed sg allocations (like * SCSI), a possible improvement here would be to pass the * table pointer into the allocator and let that clear these * flags */ gfp_mask &= ~__GFP_WAIT; gfp_mask |= __GFP_HIGH; prv = sg; } while (left); return 0; }
static struct sg_table * i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) { struct drm_i915_private *i915 = to_i915(obj->base.dev); unsigned int npages = obj->base.size / PAGE_SIZE; struct sg_table *st; struct scatterlist *sg; int max_order; gfp_t gfp; st = kmalloc(sizeof(*st), GFP_KERNEL); if (!st) return ERR_PTR(-ENOMEM); if (sg_alloc_table(st, npages, GFP_KERNEL)) { kfree(st); return ERR_PTR(-ENOMEM); } sg = st->sgl; st->nents = 0; max_order = MAX_ORDER; #ifdef CONFIG_SWIOTLB if (swiotlb_nr_tbl()) /* minimum max swiotlb size is IO_TLB_SEGSIZE */ max_order = min(max_order, ilog2(IO_TLB_SEGPAGES)); #endif gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_RECLAIMABLE; if (IS_CRESTLINE(i915) || IS_BROADWATER(i915)) { /* 965gm cannot relocate objects above 4GiB. */ gfp &= ~__GFP_HIGHMEM; gfp |= __GFP_DMA32; } do { int order = min(fls(npages) - 1, max_order); struct page *page; do { page = alloc_pages(gfp | (order ? QUIET : 0), order); if (page) break; if (!order--) goto err; /* Limit subsequent allocations as well */ max_order = order; } while (1); sg_set_page(sg, page, PAGE_SIZE << order, 0); st->nents++; npages -= 1 << order; if (!npages) { sg_mark_end(sg); break; } sg = __sg_next(sg); } while (1); if (i915_gem_gtt_prepare_pages(obj, st)) goto err; /* Mark the pages as dontneed whilst they are still pinned. As soon * as they are unpinned they are allowed to be reaped by the shrinker, * and the caller is expected to repopulate - the contents of this * object are only valid whilst active and pinned. */ obj->mm.madv = I915_MADV_DONTNEED; return st; err: sg_mark_end(sg); internal_free_pages(st); return ERR_PTR(-ENOMEM); }
static int qce_ablkcipher_async_req_handle(struct crypto_async_request *async_req) { struct ablkcipher_request *req = ablkcipher_request_cast(async_req); struct qce_cipher_reqctx *rctx = ablkcipher_request_ctx(req); struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); struct qce_alg_template *tmpl = to_cipher_tmpl(async_req->tfm); struct qce_device *qce = tmpl->qce; enum dma_data_direction dir_src, dir_dst; struct scatterlist *sg; bool diff_dst; gfp_t gfp; int ret; rctx->iv = req->info; rctx->ivsize = crypto_ablkcipher_ivsize(ablkcipher); rctx->cryptlen = req->nbytes; diff_dst = (req->src != req->dst) ? true : false; dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL; rctx->src_nents = sg_nents_for_len(req->src, req->nbytes); if (diff_dst) rctx->dst_nents = sg_nents_for_len(req->dst, req->nbytes); else rctx->dst_nents = rctx->src_nents; if (rctx->src_nents < 0) { dev_err(qce->dev, "Invalid numbers of src SG.\n"); return rctx->src_nents; } if (rctx->dst_nents < 0) { dev_err(qce->dev, "Invalid numbers of dst SG.\n"); return -rctx->dst_nents; } rctx->dst_nents += 1; gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; ret = sg_alloc_table(&rctx->dst_tbl, rctx->dst_nents, gfp); if (ret) return ret; sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ); sg = qce_sgtable_add(&rctx->dst_tbl, req->dst); if (IS_ERR(sg)) { ret = PTR_ERR(sg); goto error_free; } sg = qce_sgtable_add(&rctx->dst_tbl, &rctx->result_sg); if (IS_ERR(sg)) { ret = PTR_ERR(sg); goto error_free; } sg_mark_end(sg); rctx->dst_sg = rctx->dst_tbl.sgl; ret = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); if (ret < 0) goto error_free; if (diff_dst) { ret = dma_map_sg(qce->dev, req->src, rctx->src_nents, dir_src); if (ret < 0) goto error_unmap_dst; rctx->src_sg = req->src; } else { rctx->src_sg = rctx->dst_sg; } ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, rctx->src_nents, rctx->dst_sg, rctx->dst_nents, qce_ablkcipher_done, async_req); if (ret) goto error_unmap_src; qce_dma_issue_pending(&qce->dma); ret = qce_start(async_req, tmpl->crypto_alg_type, req->nbytes, 0); if (ret) goto error_terminate; return 0; error_terminate: qce_dma_terminate_all(&qce->dma); error_unmap_src: if (diff_dst) dma_unmap_sg(qce->dev, req->src, rctx->src_nents, dir_src); error_unmap_dst: dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); error_free: sg_free_table(&rctx->dst_tbl); return ret; }
/* * Cipher algorithm self tests */ int _fips_qcrypto_cipher_selftest(struct fips_selftest_data *selftest_d) { int rc = 0, err, tv_index, num_tv; struct crypto_ablkcipher *tfm; struct ablkcipher_request *ablkcipher_req; struct _fips_completion fips_completion; char *k_align_src = NULL; struct scatterlist fips_sg; struct _fips_test_vector_cipher tv_cipher; num_tv = (sizeof(fips_test_vector_cipher)) / (sizeof(struct _fips_test_vector_cipher)); /* One-by-one testing */ for (tv_index = 0; tv_index < num_tv; tv_index++) { memcpy(&tv_cipher, &fips_test_vector_cipher[tv_index], (sizeof(struct _fips_test_vector_cipher))); /* Single buffer allocation for in place operation */ k_align_src = kzalloc(tv_cipher.pln_txt_len, GFP_KERNEL); if (k_align_src == NULL) { pr_err("qcrypto:, Failed to allocate memory for k_align_src %ld\n", PTR_ERR(k_align_src)); return -ENOMEM; } memcpy(&k_align_src[0], tv_cipher.pln_txt, tv_cipher.pln_txt_len); /* use_sw flags are set in dtsi file which makes default Linux API calls to go to s/w crypto instead of h/w crypto. This code makes sure that all selftests calls always go to h/w, independent of DTSI flags. */ if (!strcmp(tv_cipher.mod_alg, "xts(aes)")) { if (selftest_d->prefix_aes_xts_algo) if (_fips_get_alg_cra_name( tv_cipher.mod_alg, selftest_d->algo_prefix, strlen(tv_cipher.mod_alg))) { rc = -1; pr_err("Algo Name is too long for tv %d\n", tv_index); goto clr_buf; } } else { if (selftest_d->prefix_aes_cbc_ecb_ctr_algo) if (_fips_get_alg_cra_name( tv_cipher.mod_alg, selftest_d->algo_prefix, strlen(tv_cipher.mod_alg))) { rc = -1; pr_err("Algo Name is too long for tv %d\n", tv_index); goto clr_buf; } } tfm = crypto_alloc_ablkcipher(tv_cipher.mod_alg, 0, 0); if (IS_ERR(tfm)) { pr_err("qcrypto: %s algorithm not found\n", tv_cipher.mod_alg); rc = -ENOMEM; goto clr_buf; } ablkcipher_req = ablkcipher_request_alloc(tfm, GFP_KERNEL); if (!ablkcipher_req) { pr_err("qcrypto: ablkcipher_request_alloc failed\n"); rc = -ENOMEM; goto clr_tfm; } rc = qcrypto_cipher_set_device(ablkcipher_req, selftest_d->ce_device); if (rc != 0) { pr_err("%s qcrypto_cipher_set_device failed with err %d\n", __func__, rc); goto clr_ablkcipher_req; } ablkcipher_request_set_callback(ablkcipher_req, CRYPTO_TFM_REQ_MAY_BACKLOG, _fips_cb, &fips_completion); crypto_ablkcipher_clear_flags(tfm, ~0); rc = crypto_ablkcipher_setkey(tfm, tv_cipher.key, tv_cipher.klen); if (rc) { pr_err("qcrypto: crypto_ablkcipher_setkey failed\n"); goto clr_ablkcipher_req; } sg_set_buf(&fips_sg, k_align_src, tv_cipher.enc_txt_len); sg_mark_end(&fips_sg); ablkcipher_request_set_crypt(ablkcipher_req, &fips_sg, &fips_sg, tv_cipher.pln_txt_len, tv_cipher.iv); /**** Encryption Test ****/ init_completion(&fips_completion.completion); rc = crypto_ablkcipher_encrypt(ablkcipher_req); if (rc == -EINPROGRESS || rc == -EBUSY) { rc = wait_for_completion_interruptible( &fips_completion.completion); err = fips_completion.err; if (!rc && !err) { INIT_COMPLETION(fips_completion.completion); } else { pr_err("qcrypto:cipher:ENC, wait_for_completion failed\n"); goto clr_ablkcipher_req; } } if (memcmp(k_align_src, tv_cipher.enc_txt, tv_cipher.enc_txt_len)) { rc = -1; goto clr_ablkcipher_req; } /**** Decryption test ****/ init_completion(&fips_completion.completion); rc = crypto_ablkcipher_decrypt(ablkcipher_req); if (rc == -EINPROGRESS || rc == -EBUSY) { rc = wait_for_completion_interruptible( &fips_completion.completion); err = fips_completion.err; if (!rc && !err) { INIT_COMPLETION(fips_completion.completion); } else { pr_err("qcrypto:cipher:DEC, wait_for_completion failed\n"); goto clr_ablkcipher_req; } } if (memcmp(k_align_src, tv_cipher.pln_txt, tv_cipher.pln_txt_len)) rc = -1; clr_ablkcipher_req: ablkcipher_request_free(ablkcipher_req); clr_tfm: crypto_free_ablkcipher(tfm); clr_buf: kzfree(k_align_src); if (rc) return rc; } return rc; }
static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) { struct drm_i915_private *i915 = to_i915(obj->base.dev); struct sg_table *st; struct scatterlist *sg; unsigned int sg_page_sizes; unsigned int npages; int max_order; gfp_t gfp; max_order = MAX_ORDER; #ifdef CONFIG_SWIOTLB if (swiotlb_nr_tbl()) { unsigned int max_segment; max_segment = swiotlb_max_segment(); if (max_segment) { max_segment = max_t(unsigned int, max_segment, PAGE_SIZE) >> PAGE_SHIFT; max_order = min(max_order, ilog2(max_segment)); } } #endif gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_RECLAIMABLE; if (IS_I965GM(i915) || IS_I965G(i915)) { /* 965gm cannot relocate objects above 4GiB. */ gfp &= ~__GFP_HIGHMEM; gfp |= __GFP_DMA32; } create_st: st = kmalloc(sizeof(*st), GFP_KERNEL); if (!st) return -ENOMEM; npages = obj->base.size / PAGE_SIZE; if (sg_alloc_table(st, npages, GFP_KERNEL)) { kfree(st); return -ENOMEM; } sg = st->sgl; st->nents = 0; sg_page_sizes = 0; do { int order = min(fls(npages) - 1, max_order); struct page *page; do { page = alloc_pages(gfp | (order ? QUIET : MAYFAIL), order); if (page) break; if (!order--) goto err; /* Limit subsequent allocations as well */ max_order = order; } while (1); sg_set_page(sg, page, PAGE_SIZE << order, 0); sg_page_sizes |= PAGE_SIZE << order; st->nents++; npages -= 1 << order; if (!npages) { sg_mark_end(sg); break; } sg = __sg_next(sg); } while (1); if (i915_gem_gtt_prepare_pages(obj, st)) { /* Failed to dma-map try again with single page sg segments */ if (get_order(st->sgl->length)) { internal_free_pages(st); max_order = 0; goto create_st; } goto err; } /* Mark the pages as dontneed whilst they are still pinned. As soon * as they are unpinned they are allowed to be reaped by the shrinker, * and the caller is expected to repopulate - the contents of this * object are only valid whilst active and pinned. */ obj->mm.madv = I915_MADV_DONTNEED; __i915_gem_object_set_pages(obj, st, sg_page_sizes); return 0; err: sg_set_page(sg, NULL, 0, 0); sg_mark_end(sg); internal_free_pages(st); return -ENOMEM; }
/** * __sg_alloc_table - Allocate and initialize an sg table with given allocator * @table: The sg table header to use * @nents: Number of entries in sg list * @max_ents: The maximum number of entries the allocator returns per call * @gfp_mask: GFP allocation mask * @alloc_fn: Allocator to use * * Description: * This function returns a @table @nents long. The allocator is * defined to return scatterlist chunks of maximum size @max_ents. * Thus if @nents is bigger than @max_ents, the scatterlists will be * chained in units of @max_ents. * * Notes: * If this function returns non-0 (eg failure), the caller must call * __sg_free_table() to cleanup any leftover allocations. * **/ int __sg_alloc_table(struct sg_table *table, unsigned int nents, unsigned int max_ents, gfp_t gfp_mask, sg_alloc_fn *alloc_fn) { struct scatterlist *sg, *prv; unsigned int left; #ifndef ARCH_HAS_SG_CHAIN if (WARN_ON_ONCE(nents > max_ents)) return -EINVAL; #endif memset(table, 0, sizeof(*table)); left = nents; prv = NULL; do { unsigned int sg_size, alloc_size = left; if (alloc_size > max_ents) { alloc_size = max_ents; sg_size = alloc_size - 1; } else sg_size = alloc_size; left -= sg_size; sg = alloc_fn(alloc_size, gfp_mask); if (unlikely(!sg)) { /* * Adjust entry count to reflect that the last * entry of the previous table won't be used for * linkage. Without this, sg_kfree() may get * confused. */ if (prv) table->nents = ++table->orig_nents; return -ENOMEM; } sg_init_table(sg, alloc_size); table->nents = table->orig_nents += sg_size; /* * If this is the first mapping, assign the sg table header. * If this is not the first mapping, chain previous part. */ if (prv) sg_chain(prv, max_ents, sg); else table->sgl = sg; /* * If no more entries after this one, mark the end */ if (!left) sg_mark_end(&sg[sg_size - 1]); prv = sg; } while (left); return 0; }