void sptlrpc_enc_pool_fini(void)
{
unsigned long cleaned, npools;
LASSERT(page_pools.epp_pools);
LASSERT(page_pools.epp_total_pages == page_pools.epp_free_pages);
unregister_shrinker(&pools_shrinker);
npools = npages_to_npools(page_pools.epp_total_pages);
cleaned = enc_pools_cleanup(page_pools.epp_pools, npools);
LASSERT(cleaned == page_pools.epp_total_pages);
enc_pools_free();
if (page_pools.epp_st_access > 0) {
CDEBUG(D_SEC,
"max pages %lu, grows %u, grow fails %u, shrinks %u, access %lu, missing %lu, max qlen %u, max wait %ld/%d\n",
page_pools.epp_st_max_pages, page_pools.epp_st_grows,
page_pools.epp_st_grow_fails,
page_pools.epp_st_shrinks, page_pools.epp_st_access,
page_pools.epp_st_missings, page_pools.epp_st_max_wqlen,
page_pools.epp_st_max_wait, HZ);
}
}
void sptlrpc_enc_pool_fini(void)
{
unsigned long cleaned, npools;
LASSERT(pools_shrinker);
LASSERT(page_pools.epp_pools);
LASSERT(page_pools.epp_total_pages == page_pools.epp_free_pages);
cfs_remove_shrinker(pools_shrinker);
npools = npages_to_npools(page_pools.epp_total_pages);
cleaned = enc_pools_cleanup(page_pools.epp_pools, npools);
LASSERT(cleaned == page_pools.epp_total_pages);
enc_pools_free();
if (page_pools.epp_st_access > 0) {
CWARN("max pages %lu, grows %u, grow fails %u, shrinks %u, "
"access %lu, missing %lu, max qlen %u, max wait "
CFS_TIME_T"/%d\n",
page_pools.epp_st_max_pages, page_pools.epp_st_grows,
page_pools.epp_st_grow_fails,
page_pools.epp_st_shrinks, page_pools.epp_st_access,
page_pools.epp_st_missings, page_pools.epp_st_max_wqlen,
page_pools.epp_st_max_wait, CFS_HZ);
}
}
void sptlrpc_enc_pool_fini(void)
{
unsigned long cleaned, npools;
LASSERT(pools_shrinker);
LASSERT(page_pools.epp_pools);
LASSERT(page_pools.epp_total_pages == page_pools.epp_free_pages);
remove_shrinker(pools_shrinker);
npools = npages_to_npools(page_pools.epp_total_pages);
cleaned = enc_pools_cleanup(page_pools.epp_pools, npools);
LASSERT(cleaned == page_pools.epp_total_pages);
enc_pools_free();
if (page_pools.epp_st_access > 0) {
CDEBUG(D_SEC,
"max pages %lu, grows %u, grow fails %u, shrinks %u, "
"access %lu, missing %lu, max qlen %u, max wait "
CFS_TIME_T"/%lu\n",
page_pools.epp_st_max_pages, page_pools.epp_st_grows,
page_pools.epp_st_grow_fails,
page_pools.epp_st_shrinks, page_pools.epp_st_access,
page_pools.epp_st_missings, page_pools.epp_st_max_wqlen,
page_pools.epp_st_max_wait,
msecs_to_jiffies(MSEC_PER_SEC));
}
}
static int enc_pools_add_pages(int npages)
{
static CFS_DECLARE_MUTEX(sem_add_pages);
cfs_page_t ***pools;
int npools, alloced = 0;
int i, j, rc = -ENOMEM;
if (npages < PTLRPC_MAX_BRW_PAGES)
npages = PTLRPC_MAX_BRW_PAGES;
cfs_down(&sem_add_pages);
if (npages + page_pools.epp_total_pages > page_pools.epp_max_pages)
npages = page_pools.epp_max_pages - page_pools.epp_total_pages;
LASSERT(npages > 0);
page_pools.epp_st_grows++;
npools = npages_to_npools(npages);
OBD_ALLOC(pools, npools * sizeof(*pools));
if (pools == NULL)
goto out;
for (i = 0; i < npools; i++) {
OBD_ALLOC(pools[i], CFS_PAGE_SIZE);
if (pools[i] == NULL)
goto out_pools;
for (j = 0; j < PAGES_PER_POOL && alloced < npages; j++) {
pools[i][j] = cfs_alloc_page(CFS_ALLOC_IO |
CFS_ALLOC_HIGH);
if (pools[i][j] == NULL)
goto out_pools;
alloced++;
}
}
LASSERT(alloced == npages);
enc_pools_insert(pools, npools, npages);
CDEBUG(D_SEC, "added %d pages into pools\n", npages);
rc = 0;
out_pools:
enc_pools_cleanup(pools, npools);
OBD_FREE(pools, npools * sizeof(*pools));
out:
if (rc) {
page_pools.epp_st_grow_fails++;
CERROR("Failed to allocate %d enc pages\n", npages);
}
cfs_up(&sem_add_pages);
return rc;
}
static int enc_pools_add_pages(int npages)
{
static DEFINE_MUTEX(add_pages_mutex);
struct page ***pools;
int npools, alloced = 0;
int i, j, rc = -ENOMEM;
if (npages < PTLRPC_MAX_BRW_PAGES)
npages = PTLRPC_MAX_BRW_PAGES;
mutex_lock(&add_pages_mutex);
if (npages + page_pools.epp_total_pages > page_pools.epp_max_pages)
npages = page_pools.epp_max_pages - page_pools.epp_total_pages;
LASSERT(npages > 0);
page_pools.epp_st_grows++;
npools = npages_to_npools(npages);
OBD_ALLOC(pools, npools * sizeof(*pools));
if (pools == NULL)
goto out;
for (i = 0; i < npools; i++) {
OBD_ALLOC(pools[i], PAGE_CACHE_SIZE);
if (pools[i] == NULL)
goto out_pools;
for (j = 0; j < PAGES_PER_POOL && alloced < npages; j++) {
pools[i][j] = alloc_page(GFP_NOFS |
__GFP_HIGHMEM);
if (pools[i][j] == NULL)
goto out_pools;
alloced++;
}
}
LASSERT(alloced == npages);
enc_pools_insert(pools, npools, npages);
CDEBUG(D_SEC, "added %d pages into pools\n", npages);
rc = 0;
out_pools:
enc_pools_cleanup(pools, npools);
OBD_FREE(pools, npools * sizeof(*pools));
out:
if (rc) {
page_pools.epp_st_grow_fails++;
CERROR("Failed to allocate %d enc pages\n", npages);
}
mutex_unlock(&add_pages_mutex);
return rc;
}
int sptlrpc_enc_pool_init(void)
{
DEF_SHRINKER_VAR(shvar, enc_pools_shrink,
enc_pools_shrink_count, enc_pools_shrink_scan);
/*
* maximum capacity is 1/8 of total physical memory.
* is the 1/8 a good number?
*/
page_pools.epp_max_pages = totalram_pages / 8;
page_pools.epp_max_pools = npages_to_npools(page_pools.epp_max_pages);
init_waitqueue_head(&page_pools.epp_waitq);
page_pools.epp_waitqlen = 0;
page_pools.epp_pages_short = 0;
page_pools.epp_growing = 0;
page_pools.epp_idle_idx = 0;
page_pools.epp_last_shrink = cfs_time_current_sec();
page_pools.epp_last_access = cfs_time_current_sec();
spin_lock_init(&page_pools.epp_lock);
page_pools.epp_total_pages = 0;
page_pools.epp_free_pages = 0;
page_pools.epp_st_max_pages = 0;
page_pools.epp_st_grows = 0;
page_pools.epp_st_grow_fails = 0;
page_pools.epp_st_shrinks = 0;
page_pools.epp_st_access = 0;
page_pools.epp_st_missings = 0;
page_pools.epp_st_lowfree = 0;
page_pools.epp_st_max_wqlen = 0;
page_pools.epp_st_max_wait = 0;
enc_pools_alloc();
if (page_pools.epp_pools == NULL)
return -ENOMEM;
pools_shrinker = set_shrinker(pools_shrinker_seeks, &shvar);
if (pools_shrinker == NULL) {
enc_pools_free();
return -ENOMEM;
}
return 0;
}
int sptlrpc_enc_pool_init(void)
{
/*
* maximum capacity is 1/8 of total physical memory.
* is the 1/8 a good number?
*/
page_pools.epp_max_pages = cfs_num_physpages / 8;
page_pools.epp_max_pools = npages_to_npools(page_pools.epp_max_pages);
cfs_waitq_init(&page_pools.epp_waitq);
page_pools.epp_waitqlen = 0;
page_pools.epp_pages_short = 0;
page_pools.epp_growing = 0;
page_pools.epp_idle_idx = 0;
page_pools.epp_last_shrink = cfs_time_current_sec();
page_pools.epp_last_access = cfs_time_current_sec();
cfs_spin_lock_init(&page_pools.epp_lock);
page_pools.epp_total_pages = 0;
page_pools.epp_free_pages = 0;
page_pools.epp_st_max_pages = 0;
page_pools.epp_st_grows = 0;
page_pools.epp_st_grow_fails = 0;
page_pools.epp_st_shrinks = 0;
page_pools.epp_st_access = 0;
page_pools.epp_st_missings = 0;
page_pools.epp_st_lowfree = 0;
page_pools.epp_st_max_wqlen = 0;
page_pools.epp_st_max_wait = 0;
enc_pools_alloc();
if (page_pools.epp_pools == NULL)
return -ENOMEM;
pools_shrinker = cfs_set_shrinker(pools_shrinker_seeks,
enc_pools_shrink);
if (pools_shrinker == NULL) {
enc_pools_free();
return -ENOMEM;
}
return 0;
}
int sptlrpc_enc_pool_init(void)
{
/*
* maximum capacity is 1/8 of total physical memory.
* is the 1/8 a good number?
*/
page_pools.epp_max_pages = totalram_pages / 8;
page_pools.epp_max_pools = npages_to_npools(page_pools.epp_max_pages);
init_waitqueue_head(&page_pools.epp_waitq);
page_pools.epp_waitqlen = 0;
page_pools.epp_pages_short = 0;
page_pools.epp_growing = 0;
page_pools.epp_idle_idx = 0;
page_pools.epp_last_shrink = ktime_get_seconds();
page_pools.epp_last_access = ktime_get_seconds();
spin_lock_init(&page_pools.epp_lock);
page_pools.epp_total_pages = 0;
page_pools.epp_free_pages = 0;
page_pools.epp_st_max_pages = 0;
page_pools.epp_st_grows = 0;
page_pools.epp_st_grow_fails = 0;
page_pools.epp_st_shrinks = 0;
page_pools.epp_st_access = 0;
page_pools.epp_st_missings = 0;
page_pools.epp_st_lowfree = 0;
page_pools.epp_st_max_wqlen = 0;
page_pools.epp_st_max_wait = 0;
page_pools.epp_st_outofmem = 0;
enc_pools_alloc();
if (!page_pools.epp_pools)
return -ENOMEM;
register_shrinker(&pools_shrinker);
return 0;
}
/*
* merge @npools pointed by @pools which contains @npages new pages
* into current pools.
*
* we have options to avoid most memory copy with some tricks. but we choose
* the simplest way to avoid complexity. It's not frequently called.
*/
static void enc_pools_insert(cfs_page_t ***pools, int npools, int npages)
{
int freeslot;
int op_idx, np_idx, og_idx, ng_idx;
int cur_npools, end_npools;
LASSERT(npages > 0);
LASSERT(page_pools.epp_total_pages+npages <= page_pools.epp_max_pages);
LASSERT(npages_to_npools(npages) == npools);
LASSERT(page_pools.epp_growing);
cfs_spin_lock(&page_pools.epp_lock);
/*
* (1) fill all the free slots of current pools.
*/
/* free slots are those left by rent pages, and the extra ones with
* index >= total_pages, locate at the tail of last pool. */
freeslot = page_pools.epp_total_pages % PAGES_PER_POOL;
if (freeslot != 0)
freeslot = PAGES_PER_POOL - freeslot;
freeslot += page_pools.epp_total_pages - page_pools.epp_free_pages;
op_idx = page_pools.epp_free_pages / PAGES_PER_POOL;
og_idx = page_pools.epp_free_pages % PAGES_PER_POOL;
np_idx = npools - 1;
ng_idx = (npages - 1) % PAGES_PER_POOL;
while (freeslot) {
LASSERT(page_pools.epp_pools[op_idx][og_idx] == NULL);
LASSERT(pools[np_idx][ng_idx] != NULL);
page_pools.epp_pools[op_idx][og_idx] = pools[np_idx][ng_idx];
pools[np_idx][ng_idx] = NULL;
freeslot--;
if (++og_idx == PAGES_PER_POOL) {
op_idx++;
og_idx = 0;
}
if (--ng_idx < 0) {
if (np_idx == 0)
break;
np_idx--;
ng_idx = PAGES_PER_POOL - 1;
}
}
/*
* (2) add pools if needed.
*/
cur_npools = (page_pools.epp_total_pages + PAGES_PER_POOL - 1) /
PAGES_PER_POOL;
end_npools = (page_pools.epp_total_pages + npages + PAGES_PER_POOL -1) /
PAGES_PER_POOL;
LASSERT(end_npools <= page_pools.epp_max_pools);
np_idx = 0;
while (cur_npools < end_npools) {
LASSERT(page_pools.epp_pools[cur_npools] == NULL);
LASSERT(np_idx < npools);
LASSERT(pools[np_idx] != NULL);
page_pools.epp_pools[cur_npools++] = pools[np_idx];
pools[np_idx++] = NULL;
}
page_pools.epp_total_pages += npages;
page_pools.epp_free_pages += npages;
page_pools.epp_st_lowfree = page_pools.epp_free_pages;
if (page_pools.epp_total_pages > page_pools.epp_st_max_pages)
page_pools.epp_st_max_pages = page_pools.epp_total_pages;
CDEBUG(D_SEC, "add %d pages to total %lu\n", npages,
page_pools.epp_total_pages);
cfs_spin_unlock(&page_pools.epp_lock);
}