static void per_cpu_remove_cache(void *arg) { struct kmem_cache *cache = arg; struct qlist_head to_free = QLIST_INIT; struct qlist_head *q; q = this_cpu_ptr(&cpu_quarantine); qlist_move_cache(q, &to_free, cache); qlist_free_all(&to_free, cache); }
void quarantine_remove_cache(struct kmem_cache *cache) { unsigned long flags; struct qlist_head to_free = QLIST_INIT; on_each_cpu(per_cpu_remove_cache, cache, 1); spin_lock_irqsave(&quarantine_lock, flags); qlist_move_cache(&global_quarantine, &to_free, cache); spin_unlock_irqrestore(&quarantine_lock, flags); qlist_free_all(&to_free, cache); }
static void per_cpu_remove_cache(void *arg) { struct kmem_cache *cache = arg; struct qlist to_free = QLIST_INIT; struct qlist *q; unsigned long flags; local_irq_save(flags); q = this_cpu_ptr(&cpu_quarantine); qlist_move_cache(q, &to_free, cache); local_irq_restore(flags); qlist_free_all(&to_free, cache); }
void quarantine_reduce(void) { size_t new_quarantine_size, percpu_quarantines; unsigned long flags; struct qlist_head to_free = QLIST_INIT; size_t size_to_free = 0; struct qlist_node *last; if (likely(READ_ONCE(global_quarantine.bytes) <= READ_ONCE(quarantine_size))) return; spin_lock_irqsave(&quarantine_lock, flags); /* * Update quarantine size in case of hotplug. Allocate a fraction of * the installed memory to quarantine minus per-cpu queue limits. */ new_quarantine_size = (READ_ONCE(totalram_pages) << PAGE_SHIFT) / QUARANTINE_FRACTION; percpu_quarantines = QUARANTINE_PERCPU_SIZE * num_online_cpus(); new_quarantine_size = (new_quarantine_size < percpu_quarantines) ? 0 : new_quarantine_size - percpu_quarantines; WRITE_ONCE(quarantine_size, new_quarantine_size); last = global_quarantine.head; while (last) { struct kmem_cache *cache = qlink_to_cache(last); size_to_free += cache->size; if (!last->next || size_to_free > global_quarantine.bytes - QUARANTINE_LOW_SIZE) break; last = last->next; } qlist_move(&global_quarantine, last, &to_free, size_to_free); spin_unlock_irqrestore(&quarantine_lock, flags); qlist_free_all(&to_free, NULL); }
void quarantine_reduce(void) { size_t new_quarantine_size; unsigned long flags; struct qlist to_free = QLIST_INIT; size_t size_to_free = 0; void **last; /* smp_load_acquire() here pairs with smp_store_release() below. */ if (likely(ACCESS_ONCE(global_quarantine.bytes) <= smp_load_acquire(&quarantine_size))) return; spin_lock_irqsave(&quarantine_lock, flags); /* Update quarantine size in case of hotplug. Allocate a fraction of * the installed memory to quarantine minus per-cpu queue limits. */ new_quarantine_size = (ACCESS_ONCE(totalram_pages) << PAGE_SHIFT) / QUARANTINE_FRACTION; new_quarantine_size -= QUARANTINE_PERCPU_SIZE * num_online_cpus(); /* Pairs with smp_load_acquire() above and in QUARANTINE_LOW_SIZE. */ smp_store_release(&quarantine_size, new_quarantine_size); last = global_quarantine.head; while (last) { struct kmem_cache *cache = qlink_to_cache(last); size_to_free += cache->size; if (!*last || size_to_free > global_quarantine.bytes - QUARANTINE_LOW_SIZE) break; last = (void **) *last; } qlist_move(&global_quarantine, last, &to_free, size_to_free); spin_unlock_irqrestore(&quarantine_lock, flags); qlist_free_all(&to_free, NULL); }