void * module_map (unsigned long size)
{
void * addr;
struct vm_struct **p, *tmp, *area;
size = PAGE_ALIGN(size);
if (!size || size > MODULES_LEN) return NULL;
addr = (void *) MODULES_VADDR;
for (p = &modvmlist; (tmp = *p) ; p = &tmp->next) {
if (size + (unsigned long) addr < (unsigned long) tmp->addr)
break;
addr = (void *) (tmp->size + (unsigned long) tmp->addr);
}
if ((unsigned long) addr + size >= MODULES_END) return NULL;
area = (struct vm_struct *) kmalloc(sizeof(*area), GFP_KERNEL);
if (!area) return NULL;
area->size = size + PAGE_SIZE;
area->addr = addr;
area->next = *p;
*p = area;
if (vmalloc_area_pages(VMALLOC_VMADDR(addr), size)) {
vfree(addr);
return NULL;
}
return addr;
}
/* we parse the page tables in order to find the direct mapping of
the page. This works only without holding any locks for pages we
are sure that they do not move in memory.
*/
volatile void *virt_to_kseg(volatile void *address)
{
pgd_t *pgd; pmd_t *pmd; pte_t *ptep, pte;
unsigned long va, ret = 0UL;
va=VMALLOC_VMADDR((unsigned long)address);
/* get the page directory. Use the kernel memory map. */
pgd = pgd_offset_k(va);
/* check whether we found an entry */
if (!pgd_none(*pgd))
{
/* get the page middle directory */
pmd = pmd_offset(pgd, va);
/* check whether we found an entry */
if (!pmd_none(*pmd))
{
/* get a pointer to the page table entry */
ptep = pte_offset(pmd, va);
pte = *ptep;
/* check for a valid page */
if (pte_present(pte))
{
/* get the address the page is refering to */
ret = (unsigned long)page_address(pte_page(pte));
/* add the offset within the page to the page address */
ret |= (va & (PAGE_SIZE -1));
}
}
}
return((volatile void *)ret);
}
/*From: http://www.scs.ch/~frey/linux/memorymap.html*/
volatile void *virt_to_kseg(volatile void *address) {
pgd_t *pgd; pmd_t *pmd; pte_t *ptep, pte;
unsigned long va, ret = 0UL;
va=VMALLOC_VMADDR((unsigned long)address);
/* get the page directory. Use the kernel memory map. */
pgd = pgd_offset_k(va);
/* check whether we found an entry */
if (!pgd_none(*pgd)) {
/*I'm not sure if we need this, or the line for 2.4*/
/*above will work reliably too*/
/*If you know, please email me :-)*/
pud_t *pud = pud_offset(pgd, va);
pmd = pmd_offset(pud, va);
/* check whether we found an entry */
if (!pmd_none(*pmd)) {
/* get a pointer to the page table entry */
ptep = pte_offset_map(pmd, va);
pte = *ptep;
/* check for a valid page */
if (pte_present(pte)) {
/* get the address the page is refering to */
ret = (unsigned long)page_address(pte_page(pte));
/* add the offset within the page to the page address */
ret |= (va & (PAGE_SIZE -1));
}
}
}
return((volatile void *)ret);
}
/*
* Remap an arbitrary physical address space into the kernel virtual
* address space. Needed when the kernel wants to access high addresses
* directly.
*
* NOTE! We need to allow non-page-aligned mappings too: we will obviously
* have to convert them into an offset in a page-aligned mapping, but the
* caller shouldn't need to know that small detail.
*/
void * __ioremap(unsigned long phys_addr, unsigned long size, unsigned long flags)
{
void * addr;
struct vm_struct * area;
unsigned long offset, last_addr;
/* Don't allow wraparound or zero size */
last_addr = phys_addr + size - 1;
if (!size || last_addr < phys_addr)
return NULL;
/*
* Mappings have to be page-aligned
*/
offset = phys_addr & ~PAGE_MASK;
phys_addr &= PAGE_MASK;
size = PAGE_ALIGN(last_addr) - phys_addr;
/*
* Ok, go for it..
*/
area = get_vm_area(size, VM_IOREMAP);
DEBUG_IOREMAP(("Get vm_area returns 0x%08x addr 0x%08x \n", \
area, area->addr));
if (!area)
return NULL;
addr = area->addr;
if (remap_area_pages(VMALLOC_VMADDR(addr), phys_addr, size, flags)) {
vfree(addr);
return NULL;
}
return (void *) (offset + (char *)addr);
}
/*
* This function will allocate the requested contiguous pages and
* map them into the kernel's vmalloc() space. This is done so we
* get unique mapping for these pages, outside of the kernel's 1:1
* virtual:physical mapping. This is necessary so we can cover large
* portions of the kernel with single large page TLB entries, and
* still get unique uncached pages for consistent DMA.
*/
void *consistent_alloc(int gfp, size_t size, dma_addr_t *dma_handle)
{
struct vm_struct *area;
unsigned long page, va, pa;
void *ret;
int order, err, i;
if (in_interrupt())
BUG();
/* only allocate page size areas */
size = PAGE_ALIGN(size);
order = get_order(size);
page = __get_free_pages(gfp, order);
if (!page) {
BUG();
return NULL;
}
/* allocate some common virtual space to map the new pages */
area = get_vm_area(size, VM_ALLOC);
if (area == 0) {
free_pages(page, order);
return NULL;
}
va = VMALLOC_VMADDR(area->addr);
ret = (void *) va;
/* this gives us the real physical address of the first page */
*dma_handle = pa = virt_to_bus((void *) page);
/* set refcount=1 on all pages in an order>0 allocation so that vfree() will actually free
* all pages that were allocated.
*/
if (order > 0) {
struct page *rpage = virt_to_page(page);
for (i = 1; i < (1 << order); i++)
set_page_count(rpage + i, 1);
}
err = 0;
for (i = 0; i < size && err == 0; i += PAGE_SIZE)
err = map_page(va + i, pa + i, PAGE_KERNEL_NOCACHE);
if (err) {
vfree((void *) va);
return NULL;
}
/* we need to ensure that there are no cachelines in use, or worse dirty in this area
* - can't do until after virtual address mappings are created
*/
frv_cache_invalidate(va, va + size);
return ret;
}
/* Here we want the physical address of the memory.
* This is used when initializing the contents of the
* area and marking the pages as reserved.
*/
unsigned long kvirt_to_pa(unsigned long adr)
{
unsigned long va, kva, ret;
va = VMALLOC_VMADDR(adr);
kva = uvirt_to_kva(pgd_offset_k(va), va);
ret = __pa(kva);
return ret;
}
/*
* Remap an arbitrary physical address space into the kernel virtual
* address space. Needed when the kernel wants to access high addresses
* directly.
*
* NOTE! We need to allow non-page-aligned mappings too: we will obviously
* have to convert them into an offset in a page-aligned mapping, but the
* caller shouldn't need to know that small detail.
*/
void * __ioremap(unsigned long phys_addr, unsigned long size, unsigned long flags)
{
void * addr;
struct vm_struct * area;
unsigned long offset, last_addr;
/* Don't allow wraparound or zero size */
last_addr = phys_addr + size - 1;
if (!size || last_addr < phys_addr)
return NULL;
#if 0
/* TODO: Here we can put checks for driver-writer abuse... */
/*
* Don't remap the low PCI/ISA area, it's always mapped..
*/
if (phys_addr >= 0xA0000 && last_addr < 0x100000)
return phys_to_virt(phys_addr);
/*
* Don't allow anybody to remap normal RAM that we're using..
*/
if (phys_addr < virt_to_phys(high_memory)) {
char *t_addr, *t_end;
struct page *page;
t_addr = __va(phys_addr);
t_end = t_addr + (size - 1);
for(page = virt_to_page(t_addr); page <= virt_to_page(t_end); page++)
if(!PageReserved(page))
return NULL;
}
#endif
/*
* Mappings have to be page-aligned
*/
offset = phys_addr & ~PAGE_MASK;
phys_addr &= PAGE_MASK;
size = PAGE_ALIGN(last_addr+1) - phys_addr;
/*
* Ok, go for it..
*/
area = get_vm_area(size, VM_IOREMAP);
if (!area)
return NULL;
addr = area->addr;
if (remap_area_pages(VMALLOC_VMADDR(addr), phys_addr, size, flags)) {
vfree(addr);
return NULL;
}
return (void *) (offset + (char *)addr);
}
static inline unsigned long kvirt_to_pa(unsigned long adr)
{
unsigned long va, kva, ret;
va = VMALLOC_VMADDR(adr);
kva = uvirt_to_kva(pgd_offset_k(va), va);
ret = __pa(kva);
// printk(KERN_INFO "kv2pa(%lx-->%lx) \n", adr, ret);
return ret;
}
void * __ioremap(unsigned long phys_addr, unsigned long size, unsigned long flags)
{
void * addr;
struct vm_struct * area;
unsigned long offset, last_addr;
/* Don't allow wraparound or zero size */
last_addr = phys_addr + size - 1;
if (!size || last_addr < phys_addr)
return NULL;
/*
* Map objects in the low 512mb of address space using KSEG1, otherwise
* map using page tables.
*/
if (IS_LOW512(phys_addr) && IS_LOW512(phys_addr + size - 1))
return (void *) KSEG1ADDR(phys_addr);
/*
* Don't allow anybody to remap normal RAM that we're using..
*/
if (phys_addr < virt_to_phys(high_memory)) {
char *t_addr, *t_end;
struct page *page;
t_addr = __va(phys_addr);
t_end = t_addr + (size - 1);
for(page = virt_to_page(t_addr); page <= virt_to_page(t_end); page++)
if(!PageReserved(page))
return NULL;
}
/*
* Mappings have to be page-aligned
*/
offset = phys_addr & ~PAGE_MASK;
phys_addr &= PAGE_MASK;
size = PAGE_ALIGN(last_addr) - phys_addr;
/*
* Ok, go for it..
*/
area = get_vm_area(size, VM_IOREMAP);
if (!area)
return NULL;
addr = area->addr;
if (remap_area_pages(VMALLOC_VMADDR(addr), phys_addr, size, flags)) {
vfree(addr);
return NULL;
}
return (void *) (offset + (char *)addr);
}
struct page *scullv_vma_nopage(struct vm_area_struct *vma,
unsigned long address, int write)
{
unsigned long offset;
ScullV_Dev *ptr, *dev = scullv_vma_to_dev(vma);
struct page *page = NOPAGE_SIGBUS;
void *pageptr = NULL; /* default to "missing" */
pgd_t *pgd; pmd_t *pmd; pte_t *pte;
unsigned long lpage;
down(&dev->sem);
offset = (address - vma->vm_start) + VMA_OFFSET(vma);
if (offset >= dev->size) goto out; /* out of range */
/*
* Now retrieve the scullv device from the list,then the page.
* If the device has holes, the process receives a SIGBUS when
* accessing the hole.
*/
offset >>= PAGE_SHIFT; /* offset is a number of pages */
for (ptr = dev; ptr && offset >= dev->qset;) {
ptr = ptr->next;
offset -= dev->qset;
}
if (ptr && ptr->data) pageptr = ptr->data[offset];
if (!pageptr) goto out; /* hole or end-of-file */
/*
* After scullv lookup, "page" is now the address of the page
* needed by the current process. Since it's a vmalloc address,
* first retrieve the unsigned long value to be looked up
* in page tables.
*/
lpage = VMALLOC_VMADDR(pageptr);
#ifdef LINUX_24
spin_lock(&init_mm.page_table_lock);
#endif
pgd = pgd_offset(&init_mm, lpage);
pmd = pmd_offset(pgd, lpage);
pte = pte_offset(pmd, lpage);
#ifdef LINUX_24
page = pte_page(*pte);
spin_unlock(&init_mm.page_table_lock);
#else
page = mem_map + MAP_NR(pte_page(*pte));
#endif
/* got it, now increment the count */
get_page(page);
out:
up(&dev->sem);
return page;
}
void * vmalloc_prot(unsigned long size, pgprot_t prot)
{
void * addr;
struct vm_struct *area;
size = PAGE_ALIGN(size);
if (!size || size > (max_mapnr << PAGE_SHIFT))
return NULL;
area = get_vm_area(size);
if (!area)
return NULL;
addr = area->addr;
if (vmalloc_area_pages(VMALLOC_VMADDR(addr), size, prot)) {
vfree(addr);
return NULL;
}
return addr;
}
struct page *kmem_vm_nopage(struct vm_area_struct *vma, unsigned long address, int write)
{
unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
unsigned long kaddr;
pgd_t *pgd;
pmd_t *pmd;
pte_t *ptep, pte;
struct page *page = NULL;
/* address is user VA; convert to kernel VA of desired page */
kaddr = (address - vma->vm_start) + offset;
kaddr = VMALLOC_VMADDR(kaddr);
spin_lock(&init_mm.page_table_lock);
/* Lookup page structure for kernel VA */
pgd = pgd_offset(&init_mm, kaddr);
if (pgd_none(*pgd) || pgd_bad(*pgd))
goto out;
pmd = pmd_offset(pgd, kaddr);
if (pmd_none(*pmd) || pmd_bad(*pmd))
goto out;
ptep = pte_offset(pmd, kaddr);
if (!ptep)
goto out;
pte = *ptep;
if (!pte_present(pte))
goto out;
if (write && !pte_write(pte))
goto out;
page = pte_page(pte);
if (!VALID_PAGE(page)) {
page = NULL;
goto out;
}
/* Increment reference count on page */
get_page(page);
out:
spin_unlock(&init_mm.page_table_lock);
return page;
}
void module_unmap (void * addr)
{
struct vm_struct **p, *tmp;
if (!addr)
return;
if ((PAGE_SIZE-1) & (unsigned long) addr) {
printk("Trying to unmap module with bad address (%p)\n", addr);
return;
}
for (p = &modvmlist ; (tmp = *p) ; p = &tmp->next) {
if (tmp->addr == addr) {
*p = tmp->next;
vmfree_area_pages(VMALLOC_VMADDR(tmp->addr), tmp->size);
kfree(tmp);
return;
}
}
printk("Trying to unmap nonexistent module vm area (%p)\n", addr);
}
/*
* Remap an arbitrary physical address space into the kernel virtual
* address space. Needed when the kernel wants to access high addresses
* directly.
*/
void * __ioremap(unsigned long phys_addr, unsigned long size, unsigned long flags)
{
void * addr;
struct vm_struct * area;
if (phys_addr < virt_to_phys(high_memory))
return phys_to_virt(phys_addr);
if (phys_addr & ~PAGE_MASK)
return NULL;
size = PAGE_ALIGN(size);
if (!size || size > phys_addr + size)
return NULL;
area = get_vm_area(size, VM_IOREMAP);
if (!area)
return NULL;
addr = area->addr;
if (remap_area_pages(VMALLOC_VMADDR(addr), phys_addr, size, flags)) {
vfree(addr);
return NULL;
}
return addr;
}
static void __iomem *__ioremap(phys_addr_t addr, unsigned long size,
unsigned long flags)
{
unsigned long v, i;
phys_addr_t p;
int err;
/*
* Choose an address to map it to.
* Once the vmalloc system is running, we use it.
* Before then, we use space going down from ioremap_base
* (ioremap_bot records where we're up to).
*/
p = addr & PAGE_MASK;
size = PAGE_ALIGN(addr + size) - p;
/*
* Don't allow anybody to remap normal RAM that we're using.
* mem_init() sets high_memory so only do the check after that.
*
* However, allow remap of rootfs: TBD
*/
if (mem_init_done &&
p >= memory_start && p < virt_to_phys(high_memory) &&
!(p >= virt_to_phys((unsigned long)&__bss_stop) &&
p < virt_to_phys((unsigned long)__bss_stop))) {
printk(KERN_WARNING "__ioremap(): phys addr "PTE_FMT
" is RAM lr %p\n", (unsigned long)p,
__builtin_return_address(0));
return NULL;
}
if (size == 0)
return NULL;
/*
* Is it already mapped? If the whole area is mapped then we're
* done, otherwise remap it since we want to keep the virt addrs for
* each request contiguous.
*
* We make the assumption here that if the bottom and top
* of the range we want are mapped then it's mapped to the
* same virt address (and this is contiguous).
* -- Cort
*/
if (mem_init_done) {
struct vm_struct *area;
area = get_vm_area(size, VM_IOREMAP);
if (area == NULL)
return NULL;
v = VMALLOC_VMADDR(area->addr);
} else {
v = (ioremap_bot -= size);
}
if ((flags & _PAGE_PRESENT) == 0)
flags |= _PAGE_KERNEL;
if (flags & _PAGE_NO_CACHE)
flags |= _PAGE_GUARDED;
err = 0;
for (i = 0; i < size && err == 0; i += PAGE_SIZE)
err = map_page(v + i, p + i, flags);
if (err) {
if (mem_init_done)
vfree((void *)v);
return NULL;
}
return (void __iomem *) (v + ((unsigned long)addr & ~PAGE_MASK));
}
void *
__ioremap(unsigned long addr, unsigned long size, unsigned long flags)
{
unsigned long p, v, i;
int err;
/*
* Choose an address to map it to.
* Once the vmalloc system is running, we use it.
* Before then, we use space going down from ioremap_base
* (ioremap_bot records where we're up to).
*/
p = addr & PAGE_MASK;
size = PAGE_ALIGN(addr + size) - p;
/*
* If the address lies within the first 16 MB, assume it's in ISA
* memory space
*/
if (p < 16*1024*1024)
p += _ISA_MEM_BASE;
/*
* Don't allow anybody to remap normal RAM that we're using.
* mem_init() sets high_memory so only do the check after that.
*/
if ( mem_init_done && (p < virt_to_phys(high_memory)) )
{
printk("__ioremap(): phys addr %0lx is RAM lr %p\n", p,
__builtin_return_address(0));
return NULL;
}
if (size == 0)
return NULL;
/*
* Is it already mapped? Perhaps overlapped by a previous
* BAT mapping. If the whole area is mapped then we're done,
* otherwise remap it since we want to keep the virt addrs for
* each request contiguous.
*
* We make the assumption here that if the bottom and top
* of the range we want are mapped then it's mapped to the
* same virt address (and this is contiguous).
* -- Cort
*/
if ((v = p_mapped_by_bats(p)) /*&& p_mapped_by_bats(p+size-1)*/ )
goto out;
if (mem_init_done) {
struct vm_struct *area;
area = get_vm_area(size, VM_IOREMAP);
if (area == 0)
return NULL;
v = VMALLOC_VMADDR(area->addr);
} else {
v = (ioremap_bot -= size);
}
if ((flags & _PAGE_PRESENT) == 0)
flags |= _PAGE_KERNEL;
if (flags & _PAGE_NO_CACHE)
flags |= _PAGE_GUARDED;
/*
* Should check if it is a candidate for a BAT mapping
*/
err = 0;
for (i = 0; i < size && err == 0; i += PAGE_SIZE)
err = map_page(v+i, p+i, flags);
if (err) {
if (mem_init_done)
vfree((void *)v);
return NULL;
}
out:
return (void *) (v + (addr & ~PAGE_MASK));
}
