/** * 建立临时内核映射 * type和CPU共同确定用哪个固定映射的线性地址映射请求页。 */ void *kmap_atomic(struct page *page, enum km_type type) { enum fixed_addresses idx; unsigned long vaddr; /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */ inc_preempt_count(); /** * 如果被映射的页不属于高端内存,当然用不着映射。直接返回线性地址就行了。 */ if (!PageHighMem(page)) return page_address(page); /** * 通过type和CPU确定线性地址。 */ idx = type + KM_TYPE_NR*smp_processor_id(); vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); #ifdef CONFIG_DEBUG_HIGHMEM if (!pte_none(*(kmap_pte-idx))) BUG(); #endif /** * 将线性地址与页表项建立映射。 */ set_pte(kmap_pte-idx, mk_pte(page, kmap_prot)); /** * 当然,最后必须刷新一下TLB。然后才能返回线性地址。 */ __flush_tlb_one(vaddr); return (void*) vaddr; }
/* * copy_user_page * @to: kernel logical address * @from: kernel logical address * @address: U0 address to be mapped * @page: page (virt_to_page(to)) */ void copy_user_page(void *to, void *from, unsigned long address, struct page *page) { extern void __copy_page_wb(void *to, void *from); if (((address ^ (unsigned long)to) & CACHE_ALIAS) == 0) __copy_page_wb(to, from); else { pgprot_t pgprot = __pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_CACHABLE | _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_HW_SHARED | _PAGE_FLAGS_HARD); unsigned long phys_addr = virt_to_phys(to); unsigned long p3_addr = P3SEG + (address & CACHE_ALIAS); pgd_t *pgd = pgd_offset_k(p3_addr); pud_t *pud = pud_offset(pgd, p3_addr); pmd_t *pmd = pmd_offset(pud, p3_addr); pte_t *pte = pte_offset_kernel(pmd, p3_addr); pte_t entry; unsigned long flags; entry = pfn_pte(phys_addr >> PAGE_SHIFT, pgprot); inc_preempt_count(); BUG_ON(atomic_inc_return(&concurreny_check[(address & CACHE_ALIAS)>>12]) != 1); set_pte(pte, entry); local_irq_save(flags); flush_tlb_one(get_asid(), p3_addr); local_irq_restore(flags); update_mmu_cache(NULL, p3_addr, entry); __copy_user_page((void *)p3_addr, from, to); pte_clear(&init_mm, p3_addr, pte); atomic_dec(&concurreny_check[(address & CACHE_ALIAS)>>12]); dec_preempt_count(); } }
void *kmap_atomic(struct page *page, enum km_type type) { unsigned long idx; unsigned long vaddr; /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */ inc_preempt_count(); if (page < highmem_start_page) return page_address(page); idx = type + KM_TYPE_NR*smp_processor_id(); vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); /* XXX Fix - Anton */ #if 0 __flush_cache_one(vaddr); #else flush_cache_all(); #endif #ifdef CONFIG_DEBUG_HIGHMEM BUG_ON(!pte_none(*(kmap_pte-idx))); #endif set_pte(kmap_pte-idx, mk_pte(page, kmap_prot)); /* XXX Fix - Anton */ #if 0 __flush_tlb_one(vaddr); #else flush_tlb_all(); #endif return (void*) vaddr; }
/* * This is the same as kmap_atomic() but can map memory that doesn't * have a struct page associated with it. */ void *kmap_atomic_pfn(unsigned long pfn, enum km_type type) { enum fixed_addresses idx; unsigned long vaddr; inc_preempt_count(); idx = type + KM_TYPE_NR*smp_processor_id(); vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); set_pte(kmap_pte-idx, pfn_pte(pfn, kmap_prot)); flush_tlb_one(vaddr); return (void*) vaddr; }
/* * kmap_atomic/kunmap_atomic is significantly faster than kmap/kunmap because * no global lock is needed and because the kmap code must perform a global TLB * invalidation when the kmap pool wraps. * * However when holding an atomic kmap is is not legal to sleep, so atomic * kmaps are appropriate for short, tight code paths only. */ static void *__kmap_atomic_xen(struct page *page, enum km_type type, pgprot_t prot) { enum fixed_addresses idx; unsigned long vaddr; /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */ inc_preempt_count(); if (page < highmem_start_page) return page_address(page); idx = type + KM_TYPE_NR*smp_processor_id(); vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); #ifdef CONFIG_DEBUG_HIGHMEM if (!pte_none(*(kmap_pte-idx))) BUG(); #endif set_pte_at_sync(&init_mm, vaddr, kmap_pte-idx, mk_pte(page,prot)); return (void*) vaddr; }
/* * kmap_atomic/kunmap_atomic is significantly faster than kmap/kunmap because * no global lock is needed and because the kmap code must perform a global TLB * invalidation when the kmap pool wraps. * * However when holding an atomic kmap is is not legal to sleep, so atomic * kmaps are appropriate for short, tight code paths only. */ void *kmap_atomic(struct page *page, enum km_type type) { enum fixed_addresses idx; unsigned long vaddr; /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */ inc_preempt_count(); if (!PageHighMem(page)) return page_address(page); idx = type + KM_TYPE_NR*smp_processor_id(); vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); #ifdef CONFIG_DEBUG_HIGHMEM if (!pte_none(*(kmap_pte-idx))) BUG(); #endif set_pte(kmap_pte-idx, mk_pte(page, kmap_prot)); __flush_tlb_one(vaddr); return (void*) vaddr; }
static inline void preempt_conditional_sti(struct pt_regs *regs) { inc_preempt_count(); if (regs->flags & X86_EFLAGS_IF) local_irq_enable(); }