VCHIQ_STATUS_T vchiq_prepare_bulk_data(VCHIQ_BULK_T *bulk, VCHI_MEM_HANDLE_T memhandle, void *offset, int size, int dir) { PAGELIST_T *pagelist; int ret; WARN_ON(memhandle != VCHI_MEM_HANDLE_INVALID); ret = create_pagelist((char __user *)offset, size, (dir == VCHIQ_BULK_RECEIVE) ? PAGELIST_READ : PAGELIST_WRITE, current, &pagelist); if (ret != 0) return VCHIQ_ERROR; bulk->handle = memhandle; bulk->data = VCHIQ_ARM_ADDRESS(pagelist); /* Store the pagelist address in remote_data, which isn't used by the slave. */ bulk->remote_data = pagelist; return VCHIQ_SUCCESS; }
VCHIQ_STATUS_T vchiq_prepare_bulk_data(VCHIQ_BULK_T *bulk, VCHI_MEM_HANDLE_T memhandle, void *offset, int size, int dir) { PAGELIST_T *pagelist; BULKINFO_T *bi; int ret; WARN_ON(memhandle != VCHI_MEM_HANDLE_INVALID); bi = malloc(sizeof(*bi), M_VCPAGELIST, M_WAITOK | M_ZERO); if (bi == NULL) return VCHIQ_ERROR; ret = create_pagelist((char __user *)offset, size, (dir == VCHIQ_BULK_RECEIVE) ? PAGELIST_READ : PAGELIST_WRITE, current, bi); if (ret != 0) return VCHIQ_ERROR; bulk->handle = memhandle; bulk->data = VCHIQ_ARM_ADDRESS(bi->pagelist); /* Store the pagelist address in remote_data, which isn't used by the slave. */ bulk->remote_data = bi; return VCHIQ_SUCCESS; }
static int create_pagelist(char __user *buf, size_t count, unsigned short type, struct task_struct *task, PAGELIST_T ** ppagelist) { PAGELIST_T *pagelist; struct page **pages; struct page *page; unsigned long *addrs; unsigned int num_pages, offset, i; char *addr, *base_addr, *next_addr; int run, addridx, actual_pages; unsigned long *need_release; offset = (unsigned int)buf & (PAGE_SIZE - 1); num_pages = (count + offset + PAGE_SIZE - 1) / PAGE_SIZE; *ppagelist = NULL; /* Allocate enough storage to hold the page pointers and the page ** list */ pagelist = kmalloc(sizeof(PAGELIST_T) + (num_pages * sizeof(unsigned long)) + sizeof(unsigned long) + (num_pages * sizeof(pages[0])), GFP_KERNEL); vchiq_log_trace(vchiq_arm_log_level, "create_pagelist - %x", (unsigned int)pagelist); if (!pagelist) return -ENOMEM; addrs = pagelist->addrs; need_release = (unsigned long *)(addrs + num_pages); pages = (struct page **)(addrs + num_pages + 1); if (is_vmalloc_addr(buf)) { for (actual_pages = 0; actual_pages < num_pages; actual_pages++) { pages[actual_pages] = vmalloc_to_page(buf + (actual_pages * PAGE_SIZE)); } *need_release = 0; /* do not try and release vmalloc pages */ } else { down_read(&task->mm->mmap_sem); actual_pages = get_user_pages(task, task->mm, (unsigned long)buf & ~(PAGE_SIZE - 1), num_pages, (type == PAGELIST_READ) /*Write */ , 0 /*Force */ , pages, NULL /*vmas */); up_read(&task->mm->mmap_sem); if (actual_pages != num_pages) { vchiq_log_info(vchiq_arm_log_level, "create_pagelist - only %d/%d pages locked", actual_pages, num_pages); /* This is probably due to the process being killed */ while (actual_pages > 0) { actual_pages--; page_cache_release(pages[actual_pages]); } kfree(pagelist); if (actual_pages == 0) actual_pages = -ENOMEM; return actual_pages; } *need_release = 1; /* release user pages */ } pagelist->length = count; pagelist->type = type; pagelist->offset = offset; /* Group the pages into runs of contiguous pages */ base_addr = VCHIQ_ARM_ADDRESS(page_address(pages[0])); next_addr = base_addr + PAGE_SIZE; addridx = 0; run = 0; for (i = 1; i < num_pages; i++) { addr = VCHIQ_ARM_ADDRESS(page_address(pages[i])); if ((addr == next_addr) && (run < (PAGE_SIZE - 1))) { next_addr += PAGE_SIZE; run++; } else { addrs[addridx] = (unsigned long)base_addr + run; addridx++; base_addr = addr; next_addr = addr + PAGE_SIZE; run = 0; } } addrs[addridx] = (unsigned long)base_addr + run; addridx++; /* Partial cache lines (fragments) require special measures */ if ((type == PAGELIST_READ) && ((pagelist->offset & (CACHE_LINE_SIZE - 1)) || ((pagelist->offset + pagelist->length) & (CACHE_LINE_SIZE - 1)))) { FRAGMENTS_T *fragments; if (down_interruptible(&g_free_fragments_sema) != 0) { kfree(pagelist); return -EINTR; } WARN_ON(g_free_fragments == NULL); down(&g_free_fragments_mutex); fragments = (FRAGMENTS_T *) g_free_fragments; WARN_ON(fragments == NULL); g_free_fragments = *(FRAGMENTS_T **) g_free_fragments; up(&g_free_fragments_mutex); pagelist->type = PAGELIST_READ_WITH_FRAGMENTS + (fragments - g_fragments_base); } for (page = virt_to_page(pagelist); page <= virt_to_page(addrs + num_pages - 1); page++) { flush_dcache_page(page); } *ppagelist = pagelist; return 0; }