static void free_pages_to_dynamic_pool(void *pool, struct hmm_page_object *page_obj) { struct hmm_page *hmm_page; unsigned long flags; int ret; struct hmm_dynamic_pool_info *dypool_info = pool; if (!dypool_info) return; spin_lock_irqsave(&dypool_info->list_lock, flags); if (!dypool_info->initialized) { spin_unlock_irqrestore(&dypool_info->list_lock, flags); return; } spin_unlock_irqrestore(&dypool_info->list_lock, flags); if (page_obj->type == HMM_PAGE_TYPE_RESERVED) return; if (dypool_info->pgnr >= dypool_info->pool_size) { /* free page directly back to system */ ret = set_pages_wb(page_obj->page, 1); if (ret) dev_err(atomisp_dev, "set page to WB err ...\n"); __free_pages(page_obj->page, 0); return; } #ifdef USE_KMEM_CACHE hmm_page = kmem_cache_zalloc(dypool_info->pgptr_cache, GFP_KERNEL); #else hmm_page = atomisp_kernel_malloc(sizeof(struct hmm_page)); #endif if (!hmm_page) { dev_err(atomisp_dev, "out of memory for hmm_page.\n"); /* free page directly */ ret = set_pages_wb(page_obj->page, 1); if (ret) dev_err(atomisp_dev, "set page to WB err ...\n"); __free_pages(page_obj->page, 0); return; } hmm_page->page = page_obj->page; /* * add to pages_list of pages_pool */ spin_lock_irqsave(&dypool_info->list_lock, flags); list_add_tail(&hmm_page->list, &dypool_info->pages_list); dypool_info->pgnr++; spin_unlock_irqrestore(&dypool_info->list_lock, flags); }
static inline int ttm_tt_set_page_caching(struct page *p, enum ttm_caching_state c_old, enum ttm_caching_state c_new) { int ret = 0; if (PageHighMem(p)) return 0; if (c_old != tt_cached) { /* p isn't in the default caching state, set it to * writeback first to free its current memtype. */ ret = set_pages_wb(p, 1); if (ret) return ret; } if (c_new == tt_wc) ret = set_memory_wc((unsigned long) page_address(p), 1); else if (c_new == tt_uncached) ret = set_pages_uc(p, 1); return ret; }
static void __dma_free(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_addr, struct dma_attrs *attrs, bool is_coherent) { unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; struct page *page = virt_to_page(vaddr); if (is_coherent == false) set_pages_wb(page, count); if (!dma_release_from_contiguous(dev, page, count)) free_pages((unsigned long)vaddr, get_order(size)); }
static void hmm_dynamic_pool_exit(void **priv_data) { struct hmm_dynamic_pool_info *dypool_info; struct hmm_page *hmm_page; unsigned long flags; int ret; if (*priv_data != NULL) dypool_info = *priv_data; else return; spin_lock_irqsave(&dypool_info->list_lock, flags); if (dypool_info->flag != HMM_DYNAMIC_POOL_INITED) { spin_unlock_irqrestore(&dypool_info->list_lock, flags); return; } dypool_info->flag &= ~HMM_DYNAMIC_POOL_INITED; while (!list_empty(&dypool_info->pages_list)) { hmm_page = list_entry(dypool_info->pages_list.next, struct hmm_page, list); list_del(&hmm_page->list); spin_unlock_irqrestore(&dypool_info->list_lock, flags); /* can cause thread sleep, so cannot be put into spin_lock */ ret = set_pages_wb(hmm_page->page, 1); if (ret) v4l2_err(&atomisp_dev, "set page to WB err...\n"); __free_pages(hmm_page->page, 0); #ifdef USE_KMEM_CACHE kmem_cache_free(dypool_info->pgptr_cache, hmm_page); #else atomisp_kernel_free(hmm_page); #endif spin_lock_irqsave(&dypool_info->list_lock, flags); } spin_unlock_irqrestore(&dypool_info->list_lock, flags); #ifdef USE_KMEM_CACHE kmem_cache_destroy(dypool_info->pgptr_cache); #endif atomisp_kernel_free(dypool_info); *priv_data = NULL; }
static void free_private_bo_pages(struct hmm_buffer_object *bo, struct hmm_pool *dypool, struct hmm_pool *repool, int free_pgnr) { int i, ret; for (i = 0; i < free_pgnr; i++) { switch (bo->page_obj[i].type) { case HMM_PAGE_TYPE_RESERVED: if (repool->pops && repool->pops->pool_free_pages) { repool->pops->pool_free_pages(repool->pool_info, &bo->page_obj[i]); hmm_mem_stat.res_cnt--; } break; /* * HMM_PAGE_TYPE_GENERAL indicates that pages are from system * memory, so when free them, they should be put into dynamic * pool. */ case HMM_PAGE_TYPE_DYNAMIC: case HMM_PAGE_TYPE_GENERAL: if (dypool->pops && dypool->pops->pool_inited && dypool->pops->pool_inited(dypool->pool_info)) { if (dypool->pops->pool_free_pages) dypool->pops->pool_free_pages( dypool->pool_info, &bo->page_obj[i]); break; } /* * if dynamic memory pool doesn't exist, need to free * pages to system directly. */ default: ret = set_pages_wb(bo->page_obj[i].page, 1); if (ret) dev_err(atomisp_dev, "set page to WB err ...\n"); __free_pages(bo->page_obj[i].page, 0); hmm_mem_stat.sys_size--; break; } } return; }
static void free_pages_to_dynamic_pool(void *priv_data, struct hmm_page_object *page_obj) { struct hmm_page *hmm_page; unsigned long flags; int ret; struct hmm_dynamic_pool_info *dypool_info; if (priv_data != NULL) dypool_info = priv_data; else return; spin_lock_irqsave(&dypool_info->list_lock, flags); if (dypool_info->flag != HMM_DYNAMIC_POOL_INITED) { spin_unlock_irqrestore(&dypool_info->list_lock, flags); return; } spin_unlock_irqrestore(&dypool_info->list_lock, flags); if (page_obj->type == HMM_PAGE_TYPE_RESERVED) return; #ifdef USE_KMEM_CACHE hmm_page = kmem_cache_zalloc(dypool_info->pgptr_cache, GFP_KERNEL); #else hmm_page = atomisp_kernel_malloc(sizeof(struct hmm_page)); #endif if (!hmm_page) { v4l2_err(&atomisp_dev, "out of memory for hmm_page.\n"); /* free page directly */ ret = set_pages_wb(page_obj->page, 1); if (ret) v4l2_err(&atomisp_dev, "set page to WB err ...\n"); __free_pages(page_obj->page, 0); return; } hmm_page->page = page_obj->page; /* * add to pages_list of pages_pool */ spin_lock_irqsave(&dypool_info->list_lock, flags); list_add_tail(&hmm_page->list, &dypool_info->pages_list); spin_unlock_irqrestore(&dypool_info->list_lock, flags); }
static inline int ttm_tt_set_page_caching(struct page *p, enum ttm_caching_state c_state) { if (PageHighMem(p)) return 0; switch (c_state) { case tt_cached: return set_pages_wb(p, 1); case tt_wc: return set_memory_wc((unsigned long) page_address(p), 1); default: return set_pages_uc(p, 1); } }
static void hmm_dynamic_pool_exit(void **pool) { struct hmm_dynamic_pool_info *dypool_info = *pool; struct hmm_page *hmm_page; unsigned long flags; int ret; if (!dypool_info) return; spin_lock_irqsave(&dypool_info->list_lock, flags); if (!dypool_info->initialized) { spin_unlock_irqrestore(&dypool_info->list_lock, flags); return; } dypool_info->initialized = false; while (!list_empty(&dypool_info->pages_list)) { hmm_page = list_entry(dypool_info->pages_list.next, struct hmm_page, list); list_del(&hmm_page->list); spin_unlock_irqrestore(&dypool_info->list_lock, flags); /* can cause thread sleep, so cannot be put into spin_lock */ ret = set_pages_wb(hmm_page->page, 1); if (ret) dev_err(atomisp_dev, "set page to WB err...\n"); __free_pages(hmm_page->page, 0); #ifdef USE_KMEM_CACHE kmem_cache_free(dypool_info->pgptr_cache, hmm_page); #else atomisp_kernel_free(hmm_page); #endif spin_lock_irqsave(&dypool_info->list_lock, flags); } spin_unlock_irqrestore(&dypool_info->list_lock, flags); #ifdef USE_KMEM_CACHE kmem_cache_destroy(dypool_info->pgptr_cache); #endif atomisp_kernel_free(dypool_info); *pool = NULL; }
static void hmm_reserved_pool_exit(void **pool) { unsigned long flags; int i, ret; unsigned int pgnr; struct hmm_reserved_pool_info *repool_info = *pool; if (!repool_info) return; spin_lock_irqsave(&repool_info->list_lock, flags); if (!repool_info->initialized) { spin_unlock_irqrestore(&repool_info->list_lock, flags); return; } pgnr = repool_info->pgnr; repool_info->index = 0; repool_info->pgnr = 0; repool_info->initialized = false; spin_unlock_irqrestore(&repool_info->list_lock, flags); for (i = 0; i < pgnr; i++) { ret = set_pages_wb(repool_info->pages[i], 1); if (ret) dev_err(atomisp_dev, "set page to WB err...ret=%d\n", ret); /* W/A: set_pages_wb seldom return value = -EFAULT indicate that address of page is not in valid range(0xffff880000000000~0xffffc7ffffffffff) then, _free_pages would panic; Do not know why page address be valid, it maybe memory corruption by lowmemory */ if (!ret) __free_pages(repool_info->pages[i], 0); } kfree(repool_info->pages); kfree(repool_info); *pool = NULL; }
static void free_private_pages(struct hmm_buffer_object *bo) { struct page_block *pgblk; int ret; while (!list_empty(&bo->pgblocks)) { pgblk = list_first_entry(&bo->pgblocks, struct page_block, list); list_del(&pgblk->list); ret = set_pages_wb(pgblk->pages, order_to_nr(pgblk->order)); if (ret) v4l2_err(&atomisp_dev, "set page to WB err...\n"); __free_pages(pgblk->pages, pgblk->order); kfree(pgblk); } atomisp_kernel_free(bo->pages); }
static void hmm_reserved_pool_exit(void **priv_data) { struct hmm_reserved_pool_info *repool_info; unsigned long flags; int i, ret; unsigned int pgnr; if (*priv_data != NULL) repool_info = *priv_data; else return; spin_lock_irqsave(&repool_info->list_lock, flags); if (repool_info->flag != HMM_RESERVED_POOL_INITED) { spin_unlock_irqrestore(&repool_info->list_lock, flags); return; } pgnr = repool_info->pgnr; repool_info->index = 0; repool_info->pgnr = 0; repool_info->flag &= ~HMM_RESERVED_POOL_INITED; spin_unlock_irqrestore(&repool_info->list_lock, flags); for (i = 0; i < pgnr; i++) { ret = set_pages_wb(repool_info->pages[i], 1); if (ret) v4l2_err(&atomisp_dev, "set page to WB err...\n"); __free_pages(repool_info->pages[i], 0); } atomisp_kernel_free(repool_info->pages); atomisp_kernel_free(repool_info); *priv_data = NULL; }
/*Allocate pages which will be used only by ISP*/ static int alloc_private_pages(struct hmm_buffer_object *bo, int from_highmem, bool cached) { int ret; unsigned int pgnr, order, blk_pgnr; struct page *pages; struct page_block *pgblk; gfp_t gfp = GFP_NOWAIT | __GFP_NOWARN; /* REVISIT: need __GFP_FS too? */ int i, j; int failure_number = 0; bool reduce_order = false; bool lack_mem = true; if (from_highmem) gfp |= __GFP_HIGHMEM; pgnr = bo->pgnr; bo->pages = atomisp_kernel_malloc(sizeof(struct page *) * pgnr); if (unlikely(!bo->pages)) { v4l2_err(&atomisp_dev, "out of memory for bo->pages\n"); return -ENOMEM; } i = 0; while (pgnr) { order = nr_to_order_bottom(pgnr); /* * if be short of memory, we will set order to 0 * everytime. */ if (lack_mem) order = HMM_MIN_ORDER; else if (order > HMM_MAX_ORDER) order = HMM_MAX_ORDER; retry: /* * When order > HMM_MIN_ORDER, for performance reasons we don't * want alloc_pages() to sleep. In case it fails and fallbacks * to HMM_MIN_ORDER or in case the requested order is originally * the minimum value, we can allow alloc_pages() to sleep for * robustness purpose. * * REVISIT: why __GFP_FS is necessary? */ if (order == HMM_MIN_ORDER) { gfp &= ~GFP_NOWAIT; gfp |= __GFP_WAIT | __GFP_FS; } pages = alloc_pages(gfp, order); if (unlikely(!pages)) { /* * in low memory case, if allocation page fails, * we turn to try if order=0 allocation could * succeed. if order=0 fails too, that means there is * no memory left. */ if (order == HMM_MIN_ORDER) { v4l2_err(&atomisp_dev, "%s: cannot allocate pages\n", __func__); goto cleanup; } order = HMM_MIN_ORDER; failure_number++; reduce_order = true; /* * if fail two times continuously, we think be short * of memory now. */ if (failure_number == 2) { lack_mem = true; failure_number = 0; } goto retry; } else { blk_pgnr = order_to_nr(order); pgblk = kzalloc(sizeof(*pgblk), GFP_KERNEL); if (unlikely(!pgblk)) { v4l2_err(&atomisp_dev, "out of memory for pgblk\n"); goto out_of_mem; } INIT_LIST_HEAD(&pgblk->list); pgblk->pages = pages; pgblk->order = order; list_add_tail(&pgblk->list, &bo->pgblocks); for (j = 0; j < blk_pgnr; j++) bo->pages[i++] = pages + j; pgnr -= blk_pgnr; if (!cached) { /* * set memory to uncacheable -- UC_MINUS */ ret = set_pages_uc(pages, blk_pgnr); if (ret) { v4l2_err(&atomisp_dev, "set page uncacheable" "failed.\n"); goto cleanup; } } /* * if order is not reduced this time, clear * failure_number. */ if (reduce_order) reduce_order = false; else failure_number = 0; } } return 0; out_of_mem: __free_pages(pages, order); cleanup: while (!list_empty(&bo->pgblocks)) { pgblk = list_first_entry(&bo->pgblocks, struct page_block, list); list_del(&pgblk->list); ret = set_pages_wb(pgblk->pages, order_to_nr(pgblk->order)); if (ret) v4l2_err(&atomisp_dev, "set page to WB err...\n"); __free_pages(pgblk->pages, pgblk->order); kfree(pgblk); } atomisp_kernel_free(bo->pages); return -ENOMEM; }