int
vmap_page_range (struct vm_area_struct *vma, unsigned long from, unsigned long size, unsigned long vaddr)
{
int error = 0;
pgd_t * dir;
unsigned long beg = from;
unsigned long end = from + size;
vaddr -= from;
dir = pgd_offset(current->mm, from);
flush_cache_range(vma, beg, end);
while (from < end) {
pmd_t *pmd = pmd_alloc(current->mm, dir, from);
error = -ENOMEM;
if (!pmd)
break;
error = vmap_pmd_range(pmd, from, end - from, vaddr + from);
if (error)
break;
from = (from + PGDIR_SIZE) & PGDIR_MASK;
dir++;
}
local_flush_tlb_range(current->mm, beg, end);
return error;
}
void smp_flush_tlb_range(struct mm_struct *mm, unsigned long start,
unsigned long end)
{
if(mm->context != NO_CONTEXT) {
if(mm->cpu_vm_mask != (1 << smp_processor_id()))
xc3((smpfunc_t) BTFIXUP_CALL(local_flush_tlb_range), (unsigned long) mm, start, end);
local_flush_tlb_range(mm, start, end);
}
}
void flush_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
if (tlb_ops_need_broadcast()) {
struct tlb_args ta;
ta.ta_vma = vma;
ta.ta_start = start;
ta.ta_end = end;
on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, mm_cpumask(vma->vm_mm));
} else
local_flush_tlb_range(vma, start, end);
}
/*
* Flush all the (user) entries for the address space described
* by mm. We can't rely on mm->mmap describing all the entries
* that might be in the hash table.
*/
void
local_flush_tlb_mm(struct mm_struct *mm)
{
if (Hash == 0) {
_tlbia();
return;
}
if (mm->map_count) {
struct vm_area_struct *mp;
for (mp = mm->mmap; mp != NULL; mp = mp->vm_next)
local_flush_tlb_range(mp, mp->vm_start, mp->vm_end);
} else {
struct vm_area_struct vma;
vma.vm_mm = mm;
local_flush_tlb_range(&vma, 0, TASK_SIZE);
}
#ifdef CONFIG_SMP
smp_send_tlb_invalidate(0);
#endif
}
void smp_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end)
{
struct mm_struct *mm = vma->vm_mm;
if (mm->context != NO_CONTEXT) {
cpumask_t cpu_mask = mm->cpu_vm_mask;
cpu_clear(smp_processor_id(), cpu_mask);
if (!cpus_empty(cpu_mask))
xc3((smpfunc_t) BTFIXUP_CALL(local_flush_tlb_range), (unsigned long) vma, start, end);
local_flush_tlb_range(vma, start, end);
}
}
void flush_tlb_range(struct mm_struct *mm, unsigned long start, unsigned long end)
{
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
struct flush_tlb_data fd;
fd.mm = mm;
fd.addr1 = start;
fd.addr2 = end;
smp_call_function(flush_tlb_range_ipi, (void *)&fd, 1, 1);
} else {
int i;
for (i = 0; i < smp_num_cpus; i++)
if (smp_processor_id() != i)
cpu_context(i, mm) = 0;
}
local_flush_tlb_range(mm, start, end);
}
/*
* This routine is called from the page fault handler to remove a
* range of active mappings at this point
*/
void
remove_mapping (struct task_struct *task, unsigned long start, unsigned long end)
{
unsigned long beg = start;
pgd_t *dir;
down_write (&task->mm->mmap_sem);
dir = pgd_offset (task->mm, start);
flush_cache_range (task->mm->mmap, beg, end);
while (start < end){
remove_mapping_pmd_range (dir, start, end - start);
start = (start + PGDIR_SIZE) & PGDIR_MASK;
dir++;
}
local_flush_tlb_range (task->mm, beg, end);
up_write (&task->mm->mmap_sem);
}
/*
* Flush all tlb/hash table entries (except perhaps for those
* mapping RAM starting at PAGE_OFFSET, since they never change).
*/
void
local_flush_tlb_all(void)
{
struct vm_area_struct vma;
/* aargh!!! */
/*
* Just flush the kernel part of the address space, that's
* all that the current callers of this require.
* Eventually I hope to persuade the powers that be that
* we can and should dispense with flush_tlb_all().
* -- paulus.
*/
vma.vm_mm = &init_mm;
local_flush_tlb_range(&vma, TASK_SIZE, ~0UL);
#ifdef CONFIG_SMP
smp_send_tlb_invalidate(0);
#endif /* CONFIG_SMP */
}
void flush_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
if (IS_ENABLED(CONFIG_L4)) {
l4x_unmap_sync_mm(vma->vm_mm);
l4x_unmap_page_range(vma->vm_mm, start, end);
return;
}
if (tlb_ops_need_broadcast()) {
struct tlb_args ta;
ta.ta_vma = vma;
ta.ta_start = start;
ta.ta_end = end;
on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_range,
&ta, 1);
} else
local_flush_tlb_range(vma, start, end);
broadcast_tlb_mm_a15_erratum(vma->vm_mm);
}
void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
{
struct mm_struct *mm = vma->vm_mm;
preempt_disable();
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
struct flush_tlb_data fd;
fd.vma = vma;
fd.addr1 = start;
fd.addr2 = end;
smp_on_other_tlbs(flush_tlb_range_ipi, (void *)&fd);
} else {
int i;
for (i = 0; i < num_online_cpus(); i++)
if (smp_processor_id() != i)
cpu_context(i, mm) = 0;
}
local_flush_tlb_range(vma, start, end);
preempt_enable();
}
static void flush_tlb_range_ipi(void *info)
{
struct flush_tlb_data *fd = (struct flush_tlb_data *)info;
local_flush_tlb_range(fd->mm, fd->addr1, fd->addr2);
}
static inline void ipi_flush_tlb_range(void *arg)
{
struct tlb_args *ta = (struct tlb_args *)arg;
local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
}