void kasan_kfree_large(const void *ptr)
{
struct page *page = virt_to_page(ptr);
kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page),
KASAN_FREE_PAGE);
}
static int check_slab(struct kmem_cache *s, struct page *page)
{
int maxobj;
VM_BUG_ON(!irqs_disabled());
if (!PageSlab(page)) {
slab_err(s, page, "Not a valid slab page");
return 0;
}
maxobj = order_objects(compound_order(page), s->size, s->reserved);
if (page->objects > maxobj) {
slab_err(s, page, "objects %u > max %u",
s->name, page->objects, maxobj);
return 0;
}
if (page->inuse > page->objects) {
slab_err(s, page, "inuse %u > max %u",
s->name, page->inuse, page->objects);
return 0;
}
/* Slab_pad_check fixes things up after itself */
slab_pad_check(s, page);
return 1;
}
/* Check the pad bytes at the end of a slab page */
static int slab_pad_check(struct kmem_cache *s, struct page *page)
{
u8 *start;
u8 *fault;
u8 *end;
int length;
int remainder;
if (!(s->flags & SLAB_POISON))
return 1;
start = page_address(page);
length = (PAGE_SIZE << compound_order(page)) - s->reserved;
end = start + length;
remainder = length % s->size;
if (!remainder)
return 1;
fault = check_bytes(end - remainder, POISON_INUSE, remainder);
if (!fault)
return 1;
while (end > fault && end[-1] == POISON_INUSE)
end--;
slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
print_section("Padding", end - remainder, remainder);
restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end);
return 0;
}
static void kgsl_page_pool_free(struct kgsl_page_pool *pool, struct page *page)
{
int ret;
BUG_ON(pool->order != compound_order(page));
ret = kgsl_page_pool_add(pool, page);
if (ret)
kgsl_page_pool_free_pages(pool, page);
}
void __sync_icache_dcache(pte_t pte)
{
struct page *page = pte_page(pte);
if (!test_and_set_bit(PG_dcache_clean, &page->flags))
sync_icache_aliases(page_address(page),
PAGE_SIZE << compound_order(page));
}
int oo_iobufset_resource_remap_bt(struct oo_iobufset *iobrs, uint64_t *hw_addrs)
{
return efrm_pd_dma_remap_bt(iobrs->pd, iobrs->pages->n_bufs,
compound_order(iobrs->pages->pages[0]),
&iobrs->dma_addrs[0], sizeof(iobrs->dma_addrs[0]),
hw_addrs, sizeof(hw_addrs[0]),
put_user_fake,
&iobrs->buf_tbl_alloc);
}
void __sync_icache_dcache(pte_t pte, unsigned long addr)
{
struct page *page = pte_page(pte);
if (!test_and_set_bit(PG_dcache_clean, &page->flags))
sync_icache_aliases(page_address(page),
PAGE_SIZE << compound_order(page));
else if (icache_is_aivivt())
__flush_icache_all();
}
int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp_mask)
{
int error;
error = radix_tree_maybe_preload_order(gfp_mask, compound_order(page));
if (!error) {
error = __add_to_swap_cache(page, entry);
radix_tree_preload_end();
}
return error;
}
void kasan_poison_kfree(void *ptr)
{
struct page *page;
page = virt_to_head_page(ptr);
if (unlikely(!PageSlab(page)))
kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page),
KASAN_FREE_PAGE);
else
kasan_poison_slab_free(page->slab_cache, ptr);
}
void __sync_icache_dcache(pte_t pte, unsigned long addr)
{
struct page *page = pte_page(pte);
/* no flushing needed for anonymous pages */
if (!page_mapping(page))
return;
if (!test_and_set_bit(PG_dcache_clean, &page->flags))
sync_icache_aliases(page_address(page),
PAGE_SIZE << compound_order(page));
else if (icache_is_aivivt())
__flush_icache_all();
}
/*
* Determine if a certain object on a page is on the freelist. Must hold the
* slab lock to guarantee that the chains are in a consistent state.
*/
static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
{
int nr = 0;
void *fp;
void *object = NULL;
unsigned long max_objects;
fp = page->freelist;
while (fp && nr <= page->objects) {
if (fp == search)
return 1;
if (!check_valid_pointer(s, page, fp)) {
if (object) {
object_err(s, page, object,
"Freechain corrupt");
set_freepointer(s, object, NULL);
break;
} else {
slab_err(s, page, "Freepointer corrupt");
page->freelist = NULL;
page->inuse = page->objects;
slab_fix(s, "Freelist cleared");
return 0;
}
break;
}
object = fp;
fp = get_freepointer(s, object);
nr++;
}
max_objects = order_objects(compound_order(page), s->size, s->reserved);
if (max_objects > MAX_OBJS_PER_PAGE)
max_objects = MAX_OBJS_PER_PAGE;
if (page->objects != max_objects) {
slab_err(s, page, "Wrong number of objects. Found %d but "
"should be %d", page->objects, max_objects);
page->objects = max_objects;
slab_fix(s, "Number of objects adjusted.");
}
if (page->inuse != page->objects - nr) {
slab_err(s, page, "Wrong object count. Counter is %d but "
"counted were %d", page->inuse, page->objects - nr);
page->inuse = page->objects - nr;
slab_fix(s, "Object count adjusted.");
}
return search == NULL;
}
static void oo_iobufset_free_pages(struct oo_buffer_pages *pages)
{
#ifdef OO_DO_HUGE_PAGES
if( pages->shmid >= 0 )
oo_bufpage_huge_free(pages);
else
#endif
{
int i;
for (i = 0; i < pages->n_bufs; ++i)
__free_pages(pages->pages[i], compound_order(pages->pages[i]));
oo_iobufset_kfree(pages);
}
}
static void
oo_iobufset_resource_free(struct oo_iobufset *rs, int reset_pending)
{
efrm_pd_dma_unmap(rs->pd, rs->pages->n_bufs,
EFHW_GFP_ORDER_TO_NIC_ORDER(
compound_order(rs->pages->pages[0])),
&rs->dma_addrs[0], sizeof(rs->dma_addrs[0]),
&rs->buf_tbl_alloc, reset_pending);
if (rs->pd != NULL)
efrm_pd_release(rs->pd);
oo_iobufset_pages_release(rs->pages);
oo_iobufset_free_memory(rs);
}
void __page_frag_cache_drain(struct page *page, unsigned int count)
{
VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
if (page_ref_sub_and_test(page, count)) {
unsigned int order = compound_order(page);
/*
* __free_pages_ok() is not exported so call
* __free_pages() which decrements the ref counter
* and increment the ref counter before.
*/
page_ref_inc(page);
__free_pages(page, order);
}
}
void kasan_poison_kfree(void *ptr, unsigned long ip)
{
struct page *page;
page = virt_to_head_page(ptr);
if (unlikely(!PageSlab(page))) {
if (ptr != page_address(page)) {
kasan_report_invalid_free(ptr, ip);
return;
}
kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page),
KASAN_FREE_PAGE);
} else {
__kasan_slab_free(page->slab_cache, ptr, ip, false);
}
}
void kfree(const void *block)
{
struct page *sp;
trace_kfree(_RET_IP_, block);
if (unlikely(ZERO_OR_NULL_PTR(block)))
return;
kmemleak_free(block);
sp = virt_to_page(block);
if (PageSlab(sp)) {
int align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
unsigned int *m = (unsigned int *)(block - align);
slob_free(m, *m + align);
} else
__free_pages(sp, compound_order(sp));
}
/* can't use ksize for kmem_cache_alloc memory, only kmalloc */
size_t ksize(const void *block)
{
struct page *sp;
int align;
unsigned int *m;
BUG_ON(!block);
if (unlikely(block == ZERO_SIZE_PTR))
return 0;
sp = virt_to_page(block);
if (unlikely(!PageSlab(sp)))
return PAGE_SIZE << compound_order(sp);
align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
m = (unsigned int *)(block - align);
return SLOB_UNITS(*m) * SLOB_UNIT;
}
void kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
{
struct page *page;
unsigned long redzone_start;
unsigned long redzone_end;
if (gfpflags_allow_blocking(flags))
quarantine_reduce();
if (unlikely(ptr == NULL))
return;
page = virt_to_page(ptr);
redzone_start = round_up((unsigned long)(ptr + size),
KASAN_SHADOW_SCALE_SIZE);
redzone_end = (unsigned long)ptr + (PAGE_SIZE << compound_order(page));
kasan_unpoison_shadow(ptr, size);
kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start,
KASAN_PAGE_REDZONE);
}
/*
* Return the total memory allocated for this pointer, not
* just what the caller asked for.
*
* Doesn't have to be accurate, i.e. may have races.
*/
unsigned int kobjsize(const void *objp)
{
struct page *page;
/*
* If the object we have should not have ksize performed on it,
* return size of 0
*/
if (!objp || !virt_addr_valid(objp))
return 0;
page = virt_to_head_page(objp);
/*
* If the allocator sets PageSlab, we know the pointer came from
* kmalloc().
*/
if (PageSlab(page))
return ksize(objp);
/*
* If it's not a compound page, see if we have a matching VMA
* region. This test is intentionally done in reverse order,
* so if there's no VMA, we still fall through and hand back
* PAGE_SIZE for 0-order pages.
*/
if (!PageCompound(page)) {
struct vm_area_struct *vma;
vma = find_vma(current->mm, (unsigned long)objp);
if (vma)
return vma->vm_end - vma->vm_start;
}
/*
* The ksize() function is only guaranteed to work for pointers
* returned by kmalloc(). So handle arbitrary pointers here.
*/
return PAGE_SIZE << compound_order(page);
}
void kgsl_heap_free(struct page *page)
{
unsigned int order = compound_order(page);
free_buffer_page(page, order);
}
/**
* page_vma_mapped_walk - check if @pvmw->page is mapped in @pvmw->vma at
* @pvmw->address
* @pvmw: pointer to struct page_vma_mapped_walk. page, vma, address and flags
* must be set. pmd, pte and ptl must be NULL.
*
* Returns true if the page is mapped in the vma. @pvmw->pmd and @pvmw->pte point
* to relevant page table entries. @pvmw->ptl is locked. @pvmw->address is
* adjusted if needed (for PTE-mapped THPs).
*
* If @pvmw->pmd is set but @pvmw->pte is not, you have found PMD-mapped page
* (usually THP). For PTE-mapped THP, you should run page_vma_mapped_walk() in
* a loop to find all PTEs that map the THP.
*
* For HugeTLB pages, @pvmw->pte is set to the relevant page table entry
* regardless of which page table level the page is mapped at. @pvmw->pmd is
* NULL.
*
* Retruns false if there are no more page table entries for the page in
* the vma. @pvmw->ptl is unlocked and @pvmw->pte is unmapped.
*
* If you need to stop the walk before page_vma_mapped_walk() returned false,
* use page_vma_mapped_walk_done(). It will do the housekeeping.
*/
bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw)
{
struct mm_struct *mm = pvmw->vma->vm_mm;
struct page *page = pvmw->page;
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t pmde;
/* The only possible pmd mapping has been handled on last iteration */
if (pvmw->pmd && !pvmw->pte)
return not_found(pvmw);
if (pvmw->pte)
goto next_pte;
if (unlikely(PageHuge(pvmw->page))) {
/* when pud is not present, pte will be NULL */
pvmw->pte = huge_pte_offset(mm, pvmw->address,
PAGE_SIZE << compound_order(page));
if (!pvmw->pte)
return false;
pvmw->ptl = huge_pte_lockptr(page_hstate(page), mm, pvmw->pte);
spin_lock(pvmw->ptl);
if (!check_pte(pvmw))
return not_found(pvmw);
return true;
}
restart:
pgd = pgd_offset(mm, pvmw->address);
if (!pgd_present(*pgd))
return false;
p4d = p4d_offset(pgd, pvmw->address);
if (!p4d_present(*p4d))
return false;
pud = pud_offset(p4d, pvmw->address);
if (!pud_present(*pud))
return false;
pvmw->pmd = pmd_offset(pud, pvmw->address);
/*
* Make sure the pmd value isn't cached in a register by the
* compiler and used as a stale value after we've observed a
* subsequent update.
*/
pmde = READ_ONCE(*pvmw->pmd);
if (pmd_trans_huge(pmde) || is_pmd_migration_entry(pmde)) {
pvmw->ptl = pmd_lock(mm, pvmw->pmd);
if (likely(pmd_trans_huge(*pvmw->pmd))) {
if (pvmw->flags & PVMW_MIGRATION)
return not_found(pvmw);
if (pmd_page(*pvmw->pmd) != page)
return not_found(pvmw);
return true;
} else if (!pmd_present(*pvmw->pmd)) {
if (thp_migration_supported()) {
if (!(pvmw->flags & PVMW_MIGRATION))
return not_found(pvmw);
if (is_migration_entry(pmd_to_swp_entry(*pvmw->pmd))) {
swp_entry_t entry = pmd_to_swp_entry(*pvmw->pmd);
if (migration_entry_to_page(entry) != page)
return not_found(pvmw);
return true;
}
}
return not_found(pvmw);
} else {
/* THP pmd was split under us: handle on pte level */
spin_unlock(pvmw->ptl);
pvmw->ptl = NULL;
}
} else if (!pmd_present(pmde)) {
return false;
}
if (!map_pte(pvmw))
goto next_pte;
while (1) {
if (check_pte(pvmw))
return true;
next_pte:
/* Seek to next pte only makes sense for THP */
if (!PageTransHuge(pvmw->page) || PageHuge(pvmw->page))
return not_found(pvmw);
do {
pvmw->address += PAGE_SIZE;
if (pvmw->address >= pvmw->vma->vm_end ||
pvmw->address >=
__vma_address(pvmw->page, pvmw->vma) +
hpage_nr_pages(pvmw->page) * PAGE_SIZE)
return not_found(pvmw);
/* Did we cross page table boundary? */
if (pvmw->address % PMD_SIZE == 0) {
pte_unmap(pvmw->pte);
if (pvmw->ptl) {
spin_unlock(pvmw->ptl);
pvmw->ptl = NULL;
}
goto restart;
} else {
pvmw->pte++;
}
} while (pte_none(*pvmw->pte));
if (!pvmw->ptl) {
pvmw->ptl = pte_lockptr(mm, pvmw->pmd);
spin_lock(pvmw->ptl);
}
}
}
/*
* Send read data back to initiator.
*/
int ft_send_read_data(struct scst_cmd *cmd)
{
struct ft_cmd *fcmd;
struct fc_frame *fp = NULL;
struct fc_exch *ep;
struct fc_lport *lport;
size_t remaining;
u32 fh_off = 0;
u32 frame_off;
size_t frame_len = 0;
size_t mem_len;
u32 mem_off;
size_t tlen;
struct page *page;
int use_sg;
int error;
void *to = NULL;
u8 *from = NULL;
int loop_limit = 10000;
fcmd = scst_cmd_get_tgt_priv(cmd);
ep = fc_seq_exch(fcmd->seq);
lport = ep->lp;
frame_off = fcmd->read_data_len;
tlen = scst_cmd_get_resp_data_len(cmd);
FT_IO_DBG("oid %x oxid %x resp_len %zd frame_off %u\n",
ep->oid, ep->oxid, tlen, frame_off);
if (tlen <= frame_off)
return SCST_TGT_RES_SUCCESS;
remaining = tlen - frame_off;
if (remaining > UINT_MAX)
FT_ERR("oid %x oxid %x resp_len %zd frame_off %u\n",
ep->oid, ep->oxid, tlen, frame_off);
mem_len = scst_get_buf_first(cmd, &from);
mem_off = 0;
if (!mem_len) {
FT_IO_DBG("mem_len 0\n");
return SCST_TGT_RES_SUCCESS;
}
FT_IO_DBG("sid %x oxid %x mem_len %zd frame_off %u remaining %zd\n",
ep->sid, ep->oxid, mem_len, frame_off, remaining);
/*
* If we've already transferred some of the data, skip through
* the buffer over the data already sent and continue with the
* same sequence. Otherwise, get a new sequence for the data.
*/
if (frame_off) {
tlen = frame_off;
while (mem_len <= tlen) {
tlen -= mem_len;
scst_put_buf(cmd, from);
mem_len = scst_get_buf_next(cmd, &from);
if (!mem_len)
return SCST_TGT_RES_SUCCESS;
}
mem_len -= tlen;
mem_off = tlen;
} else
fcmd->seq = lport->tt.seq_start_next(fcmd->seq);
/* no scatter/gather in skb for odd word length due to fc_seq_send() */
use_sg = !(remaining % 4) && lport->sg_supp;
while (remaining) {
if (!loop_limit) {
FT_ERR("hit loop limit. remaining %zx mem_len %zx "
"frame_len %zx tlen %zx\n",
remaining, mem_len, frame_len, tlen);
break;
}
loop_limit--;
if (!mem_len) {
scst_put_buf(cmd, from);
mem_len = scst_get_buf_next(cmd, &from);
mem_off = 0;
if (!mem_len) {
FT_ERR("mem_len 0 from get_buf_next\n");
break;
}
}
if (!frame_len) {
frame_len = fcmd->max_lso_payload;
frame_len = min(frame_len, remaining);
fp = fc_frame_alloc(lport, use_sg ? 0 : frame_len);
if (!fp) {
FT_IO_DBG("frame_alloc failed. "
"use_sg %d frame_len %zd\n",
use_sg, frame_len);
break;
}
fr_max_payload(fp) = fcmd->max_payload;
to = fc_frame_payload_get(fp, 0);
fh_off = frame_off;
}
tlen = min(mem_len, frame_len);
BUG_ON(!tlen);
BUG_ON(tlen > remaining);
BUG_ON(tlen > mem_len);
BUG_ON(tlen > frame_len);
if (use_sg) {
page = virt_to_page(from + mem_off);
get_page(page);
tlen = min_t(size_t, tlen,
PAGE_SIZE - (mem_off & ~PAGE_MASK));
skb_fill_page_desc(fp_skb(fp),
skb_shinfo(fp_skb(fp))->nr_frags,
page, offset_in_page(from + mem_off),
tlen);
fr_len(fp) += tlen;
fp_skb(fp)->data_len += tlen;
fp_skb(fp)->truesize +=
PAGE_SIZE << compound_order(page);
frame_len -= tlen;
if (skb_shinfo(fp_skb(fp))->nr_frags >= FC_FRAME_SG_LEN)
frame_len = 0;
} else {
memcpy(to, from + mem_off, tlen);
to += tlen;
frame_len -= tlen;
}
mem_len -= tlen;
mem_off += tlen;
remaining -= tlen;
frame_off += tlen;
if (frame_len)
continue;
fc_fill_fc_hdr(fp, FC_RCTL_DD_SOL_DATA, ep->did, ep->sid,
FC_TYPE_FCP,
remaining ? (FC_FC_EX_CTX | FC_FC_REL_OFF) :
(FC_FC_EX_CTX | FC_FC_REL_OFF | FC_FC_END_SEQ),
fh_off);
error = lport->tt.seq_send(lport, fcmd->seq, fp);
if (error) {
WARN_ON(1);
/* XXX For now, initiator will retry */
} else
fcmd->read_data_len = frame_off;
}
if (mem_len)
scst_put_buf(cmd, from);
if (remaining) {
FT_IO_DBG("remaining read data %zd\n", remaining);
return SCST_TGT_RES_QUEUE_FULL;
}
return SCST_TGT_RES_SUCCESS;
}
int
oo_iobufset_resource_alloc(struct oo_buffer_pages * pages, struct efrm_pd *pd,
struct oo_iobufset **iobrs_out, uint64_t *hw_addrs,
int reset_pending)
{
struct oo_iobufset *iobrs;
int rc;
int gfp_flag = (in_atomic() || in_interrupt()) ? GFP_ATOMIC : GFP_KERNEL;
int size = sizeof(struct oo_iobufset) + pages->n_bufs * sizeof(dma_addr_t);
int nic_order;
void **addrs;
unsigned int i;
ci_assert(iobrs_out);
ci_assert(pd);
if( size <= PAGE_SIZE ) {
iobrs = kmalloc(size, gfp_flag);
if( iobrs == NULL )
return -ENOMEM;
iobrs->dma_addrs = (void *)(iobrs + 1);
}
else {
/* Avoid multi-page allocations */
iobrs = kmalloc(sizeof(struct oo_iobufset), gfp_flag);
if( iobrs == NULL )
return -ENOMEM;
ci_assert_le(pages->n_bufs * sizeof(dma_addr_t), PAGE_SIZE);
iobrs->dma_addrs = kmalloc(pages->n_bufs * sizeof(dma_addr_t), gfp_flag);
if( iobrs->dma_addrs == NULL ) {
kfree(iobrs);
return -ENOMEM;
}
}
oo_atomic_set(&iobrs->ref_count, 1);
iobrs->pd = pd;
iobrs->pages = pages;
nic_order = EFHW_GFP_ORDER_TO_NIC_ORDER(compound_order(pages->pages[0]));
ci_assert_le(sizeof(void *) * pages->n_bufs, PAGE_SIZE);
addrs = kmalloc(sizeof(void *) * pages->n_bufs, gfp_flag);
if (addrs == NULL)
{
rc = -ENOMEM;
goto fail;
}
for (i = 0; i < pages->n_bufs; i++) {
addrs[i] = page_address(pages->pages[i]);
}
rc = efrm_pd_dma_map(iobrs->pd, pages->n_bufs,
nic_order,
addrs, sizeof(addrs[0]),
&iobrs->dma_addrs[0], sizeof(iobrs->dma_addrs[0]),
hw_addrs, sizeof(hw_addrs[0]),
put_user_fake, &iobrs->buf_tbl_alloc, reset_pending);
kfree(addrs);
if( rc < 0 )
goto fail;
OO_DEBUG_VERB(ci_log("%s: [%p] %d pages", __FUNCTION__,
iobrs, iobrs->pages->n_bufs));
efrm_resource_ref(efrm_pd_to_resource(pd));
oo_atomic_inc(&pages->ref_count);
*iobrs_out = iobrs;
return 0;
fail:
oo_iobufset_free_memory(iobrs);
return rc;
}
/*
* Set up the grant operations for this fragment. If it's a flipping
* interface, we also set up the unmap request from here.
*/
static void xenvif_gop_frag_copy(struct xenvif *vif, struct sk_buff *skb,
struct netrx_pending_operations *npo,
struct page *page, unsigned long size,
unsigned long offset, int *head,
struct xenvif *foreign_vif,
grant_ref_t foreign_gref)
{
struct gnttab_copy *copy_gop;
struct xenvif_rx_meta *meta;
unsigned long bytes;
int gso_type = XEN_NETIF_GSO_TYPE_NONE;
/* Data must not cross a page boundary. */
BUG_ON(size + offset > PAGE_SIZE<<compound_order(page));
meta = npo->meta + npo->meta_prod - 1;
/* Skip unused frames from start of page */
page += offset >> PAGE_SHIFT;
offset &= ~PAGE_MASK;
while (size > 0) {
BUG_ON(offset >= PAGE_SIZE);
BUG_ON(npo->copy_off > MAX_BUFFER_OFFSET);
bytes = PAGE_SIZE - offset;
if (bytes > size)
bytes = size;
if (start_new_rx_buffer(npo->copy_off, bytes, *head)) {
/*
* Netfront requires there to be some data in the head
* buffer.
*/
BUG_ON(*head);
meta = get_next_rx_buffer(vif, npo);
}
if (npo->copy_off + bytes > MAX_BUFFER_OFFSET)
bytes = MAX_BUFFER_OFFSET - npo->copy_off;
copy_gop = npo->copy + npo->copy_prod++;
copy_gop->flags = GNTCOPY_dest_gref;
copy_gop->len = bytes;
if (foreign_vif) {
copy_gop->source.domid = foreign_vif->domid;
copy_gop->source.u.ref = foreign_gref;
copy_gop->flags |= GNTCOPY_source_gref;
} else {
copy_gop->source.domid = DOMID_SELF;
copy_gop->source.u.gmfn =
virt_to_mfn(page_address(page));
}
copy_gop->source.offset = offset;
copy_gop->dest.domid = vif->domid;
copy_gop->dest.offset = npo->copy_off;
copy_gop->dest.u.ref = npo->copy_gref;
npo->copy_off += bytes;
meta->size += bytes;
offset += bytes;
size -= bytes;
/* Next frame */
if (offset == PAGE_SIZE && size) {
BUG_ON(!PageCompound(page));
page++;
offset = 0;
}
/* Leave a gap for the GSO descriptor. */
if (skb_is_gso(skb)) {
if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
gso_type = XEN_NETIF_GSO_TYPE_TCPV4;
else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
gso_type = XEN_NETIF_GSO_TYPE_TCPV6;
}
if (*head && ((1 << gso_type) & vif->gso_mask))
vif->rx.req_cons++;
*head = 0; /* There must be something in this buffer now. */
}
}
void kasan_poison_slab(struct page *page)
{
kasan_poison_shadow(page_address(page),
PAGE_SIZE << compound_order(page),
KASAN_KMALLOC_REDZONE);
}
/*
* Deliver read data back to initiator.
* XXX TBD handle resource problems later.
*/
int ft_queue_data_in(struct se_cmd *se_cmd)
{
struct ft_cmd *cmd = container_of(se_cmd, struct ft_cmd, se_cmd);
struct fc_frame *fp = NULL;
struct fc_exch *ep;
struct fc_lport *lport;
struct scatterlist *sg = NULL;
size_t remaining;
u32 f_ctl = FC_FC_EX_CTX | FC_FC_REL_OFF;
u32 mem_off = 0;
u32 fh_off = 0;
u32 frame_off = 0;
size_t frame_len = 0;
size_t mem_len = 0;
size_t tlen;
size_t off_in_page;
struct page *page = NULL;
int use_sg;
int error;
void *page_addr;
void *from;
void *to = NULL;
if (cmd->aborted)
return 0;
if (se_cmd->scsi_status == SAM_STAT_TASK_SET_FULL)
goto queue_status;
ep = fc_seq_exch(cmd->seq);
lport = ep->lp;
cmd->seq = lport->tt.seq_start_next(cmd->seq);
remaining = se_cmd->data_length;
/*
* Setup to use first mem list entry, unless no data.
*/
BUG_ON(remaining && !se_cmd->t_data_sg);
if (remaining) {
sg = se_cmd->t_data_sg;
mem_len = sg->length;
mem_off = sg->offset;
page = sg_page(sg);
}
/* no scatter/gather in skb for odd word length due to fc_seq_send() */
use_sg = !(remaining % 4);
while (remaining) {
struct fc_seq *seq = cmd->seq;
if (!seq) {
pr_debug("%s: Command aborted, xid 0x%x\n",
__func__, ep->xid);
break;
}
if (!mem_len) {
sg = sg_next(sg);
mem_len = min((size_t)sg->length, remaining);
mem_off = sg->offset;
page = sg_page(sg);
}
if (!frame_len) {
/*
* If lport's has capability of Large Send Offload LSO)
* , then allow 'frame_len' to be as big as 'lso_max'
* if indicated transfer length is >= lport->lso_max
*/
frame_len = (lport->seq_offload) ? lport->lso_max :
cmd->sess->max_frame;
frame_len = min(frame_len, remaining);
fp = fc_frame_alloc(lport, use_sg ? 0 : frame_len);
if (!fp)
return -ENOMEM;
to = fc_frame_payload_get(fp, 0);
fh_off = frame_off;
frame_off += frame_len;
/*
* Setup the frame's max payload which is used by base
* driver to indicate HW about max frame size, so that
* HW can do fragmentation appropriately based on
* "gso_max_size" of underline netdev.
*/
fr_max_payload(fp) = cmd->sess->max_frame;
}
tlen = min(mem_len, frame_len);
if (use_sg) {
off_in_page = mem_off;
BUG_ON(!page);
get_page(page);
skb_fill_page_desc(fp_skb(fp),
skb_shinfo(fp_skb(fp))->nr_frags,
page, off_in_page, tlen);
fr_len(fp) += tlen;
fp_skb(fp)->data_len += tlen;
fp_skb(fp)->truesize +=
PAGE_SIZE << compound_order(page);
} else {
BUG_ON(!page);
from = kmap_atomic(page + (mem_off >> PAGE_SHIFT));
page_addr = from;
from += mem_off & ~PAGE_MASK;
tlen = min(tlen, (size_t)(PAGE_SIZE -
(mem_off & ~PAGE_MASK)));
memcpy(to, from, tlen);
kunmap_atomic(page_addr);
to += tlen;
}
mem_off += tlen;
mem_len -= tlen;
frame_len -= tlen;
remaining -= tlen;
if (frame_len &&
(skb_shinfo(fp_skb(fp))->nr_frags < FC_FRAME_SG_LEN))
continue;
if (!remaining)
f_ctl |= FC_FC_END_SEQ;
fc_fill_fc_hdr(fp, FC_RCTL_DD_SOL_DATA, ep->did, ep->sid,
FC_TYPE_FCP, f_ctl, fh_off);
error = lport->tt.seq_send(lport, seq, fp);
if (error) {
pr_info_ratelimited("%s: Failed to send frame %p, "
"xid <0x%x>, remaining %zu, "
"lso_max <0x%x>\n",
__func__, fp, ep->xid,
remaining, lport->lso_max);
/*
* Go ahead and set TASK_SET_FULL status ignoring the
* rest of the DataIN, and immediately attempt to
* send the response via ft_queue_status() in order
* to notify the initiator that it should reduce it's
* per LUN queue_depth.
*/
se_cmd->scsi_status = SAM_STAT_TASK_SET_FULL;
break;
}
}
queue_status:
return ft_queue_status(se_cmd);
}
static int ept_set_epte(struct vmx_vcpu *vcpu, int make_write,
unsigned long gpa, unsigned long hva)
{
int ret;
epte_t *epte, flags;
struct page *page;
unsigned huge_shift;
int level;
ret = get_user_pages_fast(hva, 1, make_write, &page);
if (ret != 1) {
ret = ept_set_pfnmap_epte(vcpu, make_write, gpa, hva);
if (ret)
printk(KERN_ERR "ept: failed to get user page %lx\n", hva);
return ret;
}
spin_lock(&vcpu->ept_lock);
huge_shift = compound_order(compound_head(page)) + PAGE_SHIFT;
level = 0;
if (huge_shift == 30)
level = 2;
else if (huge_shift == 21)
level = 1;
ret = ept_lookup_gpa(vcpu, (void *) gpa,
level, 1, &epte);
if (ret) {
spin_unlock(&vcpu->ept_lock);
put_page(page);
printk(KERN_ERR "ept: failed to lookup EPT entry\n");
return ret;
}
if (epte_present(*epte)) {
if (!epte_big(*epte) && level == 2)
ept_clear_l2_epte(epte);
else if (!epte_big(*epte) && level == 1)
ept_clear_l1_epte(epte);
else
ept_clear_epte(epte);
}
flags = __EPTE_READ | __EPTE_EXEC |
__EPTE_TYPE(EPTE_TYPE_WB) | __EPTE_IPAT;
if (make_write)
flags |= __EPTE_WRITE;
if (vcpu->ept_ad_enabled) {
/* premark A/D to avoid extra memory references */
flags |= __EPTE_A;
if (make_write)
flags |= __EPTE_D;
}
if (level) {
struct page *tmp = page;
page = compound_head(page);
get_page(page);
put_page(tmp);
flags |= __EPTE_SZ;
}
*epte = epte_addr(page_to_phys(page)) | flags;
spin_unlock(&vcpu->ept_lock);
return 0;
}
/*
* Set up the grant operations for this fragment. If it's a flipping
* interface, we also set up the unmap request from here.
*/
static void netbk_gop_frag_copy(struct xenvif *vif, struct sk_buff *skb,
struct netrx_pending_operations *npo,
struct page *page, unsigned long size,
unsigned long offset, int *head)
{
struct gnttab_copy *copy_gop;
struct netbk_rx_meta *meta;
/*
* These variables are used iff get_page_ext returns true,
* in which case they are guaranteed to be initialized.
*/
unsigned int uninitialized_var(group), uninitialized_var(idx);
int foreign = get_page_ext(page, &group, &idx);
unsigned long bytes;
/* Data must not cross a page boundary. */
BUG_ON(size + offset > PAGE_SIZE<<compound_order(page));
meta = npo->meta + npo->meta_prod - 1;
/* Skip unused frames from start of page */
page += offset >> PAGE_SHIFT;
offset &= ~PAGE_MASK;
while (size > 0) {
BUG_ON(offset >= PAGE_SIZE);
BUG_ON(npo->copy_off > MAX_BUFFER_OFFSET);
bytes = PAGE_SIZE - offset;
if (bytes > size)
bytes = size;
if (start_new_rx_buffer(npo->copy_off, bytes, *head)) {
/*
* Netfront requires there to be some data in the head
* buffer.
*/
BUG_ON(*head);
meta = get_next_rx_buffer(vif, npo);
}
if (npo->copy_off + bytes > MAX_BUFFER_OFFSET)
bytes = MAX_BUFFER_OFFSET - npo->copy_off;
copy_gop = npo->copy + npo->copy_prod++;
copy_gop->flags = GNTCOPY_dest_gref;
if (foreign) {
struct xen_netbk *netbk = &xen_netbk[group];
struct pending_tx_info *src_pend;
src_pend = &netbk->pending_tx_info[idx];
copy_gop->source.domid = src_pend->vif->domid;
copy_gop->source.u.ref = src_pend->req.gref;
copy_gop->flags |= GNTCOPY_source_gref;
} else {
void *vaddr = page_address(page);
copy_gop->source.domid = DOMID_SELF;
copy_gop->source.u.gmfn = virt_to_mfn(vaddr);
}
copy_gop->source.offset = offset;
copy_gop->dest.domid = vif->domid;
copy_gop->dest.offset = npo->copy_off;
copy_gop->dest.u.ref = npo->copy_gref;
copy_gop->len = bytes;
npo->copy_off += bytes;
meta->size += bytes;
offset += bytes;
size -= bytes;
/* Next frame */
if (offset == PAGE_SIZE && size) {
BUG_ON(!PageCompound(page));
page++;
offset = 0;
}
/* Leave a gap for the GSO descriptor. */
if (*head && skb_shinfo(skb)->gso_size && !vif->gso_prefix)
vif->rx.req_cons++;
*head = 0; /* There must be something in this buffer now. */
}
}
static void xennet_make_frags(struct sk_buff *skb, struct net_device *dev,
struct xen_netif_tx_request *tx)
{
struct netfront_info *np = netdev_priv(dev);
char *data = skb->data;
unsigned long mfn;
RING_IDX prod = np->tx.req_prod_pvt;
int frags = skb_shinfo(skb)->nr_frags;
unsigned int offset = offset_in_page(data);
unsigned int len = skb_headlen(skb);
unsigned int id;
grant_ref_t ref;
int i;
/* While the header overlaps a page boundary (including being
larger than a page), split it it into page-sized chunks. */
while (len > PAGE_SIZE - offset) {
tx->size = PAGE_SIZE - offset;
tx->flags |= XEN_NETTXF_more_data;
len -= tx->size;
data += tx->size;
offset = 0;
id = get_id_from_freelist(&np->tx_skb_freelist, np->tx_skbs);
np->tx_skbs[id].skb = skb_get(skb);
tx = RING_GET_REQUEST(&np->tx, prod++);
tx->id = id;
ref = gnttab_claim_grant_reference(&np->gref_tx_head);
BUG_ON((signed short)ref < 0);
mfn = virt_to_mfn(data);
gnttab_grant_foreign_access_ref(ref, np->xbdev->otherend_id,
mfn, GNTMAP_readonly);
tx->gref = np->grant_tx_ref[id] = ref;
tx->offset = offset;
tx->size = len;
tx->flags = 0;
}
/* Grant backend access to each skb fragment page. */
for (i = 0; i < frags; i++) {
skb_frag_t *frag = skb_shinfo(skb)->frags + i;
struct page *page = skb_frag_page(frag);
len = skb_frag_size(frag);
offset = frag->page_offset;
/* Data must not cross a page boundary. */
BUG_ON(len + offset > PAGE_SIZE<<compound_order(page));
/* Skip unused frames from start of page */
page += offset >> PAGE_SHIFT;
offset &= ~PAGE_MASK;
while (len > 0) {
unsigned long bytes;
BUG_ON(offset >= PAGE_SIZE);
bytes = PAGE_SIZE - offset;
if (bytes > len)
bytes = len;
tx->flags |= XEN_NETTXF_more_data;
id = get_id_from_freelist(&np->tx_skb_freelist,
np->tx_skbs);
np->tx_skbs[id].skb = skb_get(skb);
tx = RING_GET_REQUEST(&np->tx, prod++);
tx->id = id;
ref = gnttab_claim_grant_reference(&np->gref_tx_head);
BUG_ON((signed short)ref < 0);
mfn = pfn_to_mfn(page_to_pfn(page));
gnttab_grant_foreign_access_ref(ref,
np->xbdev->otherend_id,
mfn, GNTMAP_readonly);
tx->gref = np->grant_tx_ref[id] = ref;
tx->offset = offset;
tx->size = bytes;
tx->flags = 0;
offset += bytes;
len -= bytes;
/* Next frame */
if (offset == PAGE_SIZE && len) {
BUG_ON(!PageCompound(page));
page++;
offset = 0;
}
}
}
np->tx.req_prod_pvt = prod;
}
