/*! Unmaps a range of pages of an area.
The default implementation just iterates over all virtual pages of the
range and calls UnmapPage(). This is obviously not particularly efficient.
*/
void
VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size,
bool updatePageQueue)
{
ASSERT(base % B_PAGE_SIZE == 0);
ASSERT(size % B_PAGE_SIZE == 0);
addr_t address = base;
addr_t end = address + size;
#if DEBUG_PAGE_ACCESS
for (; address != end; address += B_PAGE_SIZE) {
phys_addr_t physicalAddress;
uint32 flags;
if (Query(address, &physicalAddress, &flags) == B_OK
&& (flags & PAGE_PRESENT) != 0) {
vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE);
if (page != NULL) {
DEBUG_PAGE_ACCESS_START(page);
UnmapPage(area, address, updatePageQueue);
DEBUG_PAGE_ACCESS_END(page);
} else
UnmapPage(area, address, updatePageQueue);
}
}
#else
for (; address != end; address += B_PAGE_SIZE)
UnmapPage(area, address, updatePageQueue);
#endif
}
/*! Unmaps all of an area's pages.
If \a deletingAddressSpace is \c true, the address space the area belongs to
is in the process of being destroyed and isn't used by anyone anymore. For
some architectures this can be used for optimizations (e.g. not unmapping
pages or at least not needing to invalidate TLB entries).
If \a ignoreTopCachePageFlags is \c true, the area is in the process of
being destroyed and its top cache is otherwise unreferenced. I.e. all mapped
pages that live in the top cache area going to be freed and the page
accessed and modified flags don't need to be propagated.
The default implementation just iterates over all virtual pages of the
area and calls UnmapPage(). This is obviously not particularly efficient.
*/
void
VMTranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace,
bool ignoreTopCachePageFlags)
{
addr_t address = area->Base();
addr_t end = address + area->Size();
#if DEBUG_PAGE_ACCESS
for (; address != end; address += B_PAGE_SIZE) {
phys_addr_t physicalAddress;
uint32 flags;
if (Query(address, &physicalAddress, &flags) == B_OK
&& (flags & PAGE_PRESENT) != 0) {
vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE);
if (page != NULL) {
DEBUG_PAGE_ACCESS_START(page);
UnmapPage(area, address, true);
DEBUG_PAGE_ACCESS_END(page);
} else
UnmapPage(area, address, true);
}
}
#else
for (; address != end; address += B_PAGE_SIZE)
UnmapPage(area, address, true);
#endif
}
void
PrecacheIO::IOFinished(status_t status, bool partialTransfer,
generic_size_t bytesTransferred)
{
AutoLocker<VMCache> locker(fCache);
// Make successfully loaded pages accessible again (partially
// transferred pages are considered failed)
phys_size_t pagesTransferred
= (bytesTransferred + B_PAGE_SIZE - 1) / B_PAGE_SIZE;
if (fOffset + (off_t)bytesTransferred > fCache->virtual_end)
bytesTransferred = fCache->virtual_end - fOffset;
for (uint32 i = 0; i < pagesTransferred; i++) {
if (i == pagesTransferred - 1
&& (bytesTransferred % B_PAGE_SIZE) != 0) {
// clear partial page
size_t bytesTouched = bytesTransferred % B_PAGE_SIZE;
vm_memset_physical(
((phys_addr_t)fPages[i]->physical_page_number << PAGE_SHIFT)
+ bytesTouched,
0, B_PAGE_SIZE - bytesTouched);
}
DEBUG_PAGE_ACCESS_TRANSFER(fPages[i], fAllocatingThread);
fCache->MarkPageUnbusy(fPages[i]);
DEBUG_PAGE_ACCESS_END(fPages[i]);
}
// Free pages after failed I/O
for (uint32 i = pagesTransferred; i < fPageCount; i++) {
DEBUG_PAGE_ACCESS_TRANSFER(fPages[i], fAllocatingThread);
fCache->NotifyPageEvents(fPages[i], PAGE_EVENT_NOT_BUSY);
fCache->RemovePage(fPages[i]);
vm_page_set_state(fPages[i], PAGE_STATE_FREE);
}
delete this;
}
void
ARMVMTranslationMap32Bit::UnmapArea(VMArea* area, bool deletingAddressSpace,
bool ignoreTopCachePageFlags)
{
if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) {
ARMVMTranslationMap32Bit::UnmapPages(area, area->Base(), area->Size(),
true);
return;
}
bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags;
page_directory_entry* pd = fPagingStructures->pgdir_virt;
RecursiveLocker locker(fLock);
VMAreaMappings mappings;
mappings.MoveFrom(&area->mappings);
for (VMAreaMappings::Iterator it = mappings.GetIterator();
vm_page_mapping* mapping = it.Next();) {
vm_page* page = mapping->page;
page->mappings.Remove(mapping);
VMCache* cache = page->Cache();
bool pageFullyUnmapped = false;
if (!page->IsMapped()) {
atomic_add(&gMappedPagesCount, -1);
pageFullyUnmapped = true;
}
if (unmapPages || cache != area->cache) {
addr_t address = area->Base()
+ ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset);
int index = VADDR_TO_PDENT(address);
if ((pd[index] & ARM_PDE_TYPE_MASK) == 0) {
panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
"has no page dir entry", page, area, address);
continue;
}
ThreadCPUPinner pinner(thread_get_current_thread());
page_table_entry* pt
= (page_table_entry*)fPageMapper->GetPageTableAt(
pd[index] & ARM_PDE_ADDRESS_MASK);
page_table_entry oldEntry
= ARMPagingMethod32Bit::ClearPageTableEntry(
&pt[VADDR_TO_PTENT(address)]);
pinner.Unlock();
if ((oldEntry & ARM_PTE_TYPE_MASK) == 0) {
panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
"has no page table entry", page, area, address);
continue;
}
// transfer the accessed/dirty flags to the page and invalidate
// the mapping, if necessary
if (true /*(oldEntry & ARM_PTE_ACCESSED) != 0*/) { // XXX IRA
page->accessed = true;
if (!deletingAddressSpace)
InvalidatePage(address);
}
if (true /*(oldEntry & ARM_PTE_DIRTY) != 0*/)
page->modified = true;
if (pageFullyUnmapped) {
DEBUG_PAGE_ACCESS_START(page);
if (cache->temporary)
vm_page_set_state(page, PAGE_STATE_INACTIVE);
else if (page->modified)
vm_page_set_state(page, PAGE_STATE_MODIFIED);
else
vm_page_set_state(page, PAGE_STATE_CACHED);
DEBUG_PAGE_ACCESS_END(page);
}
}
fMapCount--;
}
Flush();
// flush explicitely, since we directly use the lock
locker.Unlock();
bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = mappings.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
}
void
ARMVMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size,
bool updatePageQueue)
{
if (size == 0)
return;
addr_t start = base;
addr_t end = base + size - 1;
TRACE("ARMVMTranslationMap32Bit::UnmapPages(%p, %#" B_PRIxADDR ", %#"
B_PRIxADDR ")\n", area, start, end);
page_directory_entry* pd = fPagingStructures->pgdir_virt;
VMAreaMappings queue;
RecursiveLocker locker(fLock);
do {
int index = VADDR_TO_PDENT(start);
if ((pd[index] & ARM_PDE_TYPE_MASK) == 0) {
// no page table here, move the start up to access the next page
// table
start = ROUNDUP(start + 1, kPageTableAlignment);
continue;
}
Thread* thread = thread_get_current_thread();
ThreadCPUPinner pinner(thread);
page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt(
pd[index] & ARM_PDE_ADDRESS_MASK);
for (index = VADDR_TO_PTENT(start); (index < 1024) && (start < end);
index++, start += B_PAGE_SIZE) {
page_table_entry oldEntry
= ARMPagingMethod32Bit::ClearPageTableEntry(&pt[index]);
if ((oldEntry & ARM_PTE_TYPE_MASK) == 0)
continue;
fMapCount--;
if (true /*(oldEntry & ARM_PTE_ACCESSED) != 0*/) { // XXX IRA
// Note, that we only need to invalidate the address, if the
// accessed flags was set, since only then the entry could have
// been in any TLB.
InvalidatePage(start);
}
if (area->cache_type != CACHE_TYPE_DEVICE) {
// get the page
vm_page* page = vm_lookup_page(
(oldEntry & ARM_PTE_ADDRESS_MASK) / B_PAGE_SIZE);
ASSERT(page != NULL);
DEBUG_PAGE_ACCESS_START(page);
// transfer the accessed/dirty flags to the page
if (/*(oldEntry & ARM_PTE_ACCESSED) != 0*/ true) // XXX IRA
page->accessed = true;
if (/*(oldEntry & ARM_PTE_DIRTY) != 0 */ true)
page->modified = true;
// remove the mapping object/decrement the wired_count of the
// page
if (area->wiring == B_NO_LOCK) {
vm_page_mapping* mapping = NULL;
vm_page_mappings::Iterator iterator
= page->mappings.GetIterator();
while ((mapping = iterator.Next()) != NULL) {
if (mapping->area == area)
break;
}
ASSERT(mapping != NULL);
area->mappings.Remove(mapping);
page->mappings.Remove(mapping);
queue.Add(mapping);
} else
page->DecrementWiredCount();
if (!page->IsMapped()) {
atomic_add(&gMappedPagesCount, -1);
if (updatePageQueue) {
if (page->Cache()->temporary)
vm_page_set_state(page, PAGE_STATE_INACTIVE);
else if (page->modified)
vm_page_set_state(page, PAGE_STATE_MODIFIED);
else
vm_page_set_state(page, PAGE_STATE_CACHED);
}
}
DEBUG_PAGE_ACCESS_END(page);
}
}
Flush();
// flush explicitly, since we directly use the lock
} while (start != 0 && start < end);
// TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not
// really critical here, as in all cases this method is used, the unmapped
// area range is unmapped for good (resized/cut) and the pages will likely
// be freed.
locker.Unlock();
// free removed mappings
bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = queue.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
}
status_t
ARMVMTranslationMap32Bit::Map(addr_t va, phys_addr_t pa, uint32 attributes,
uint32 memoryType, vm_page_reservation* reservation)
{
TRACE("map_tmap: entry pa 0x%lx va 0x%lx\n", pa, va);
/*
dprintf("pgdir at 0x%x\n", pgdir);
dprintf("index is %d\n", va / B_PAGE_SIZE / 1024);
dprintf("final at 0x%x\n", &pgdir[va / B_PAGE_SIZE / 1024]);
dprintf("value is 0x%x\n", *(int *)&pgdir[va / B_PAGE_SIZE / 1024]);
dprintf("present bit is %d\n", pgdir[va / B_PAGE_SIZE / 1024].present);
dprintf("addr is %d\n", pgdir[va / B_PAGE_SIZE / 1024].addr);
*/
page_directory_entry* pd = fPagingStructures->pgdir_virt;
// check to see if a page table exists for this range
uint32 index = VADDR_TO_PDENT(va);
if ((pd[index] & ARM_PDE_TYPE_MASK) == 0) {
phys_addr_t pgtable;
vm_page *page;
// we need to allocate a pgtable
page = vm_page_allocate_page(reservation,
PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);
DEBUG_PAGE_ACCESS_END(page);
pgtable = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE;
TRACE("map_tmap: asked for free page for pgtable. 0x%lx\n", pgtable);
// put it in the pgdir
ARMPagingMethod32Bit::PutPageTableInPageDir(&pd[index], pgtable,
attributes
| ((attributes & B_USER_PROTECTION) != 0
? B_WRITE_AREA : B_KERNEL_WRITE_AREA));
// update any other page directories, if it maps kernel space
if (index >= FIRST_KERNEL_PGDIR_ENT
&& index < (FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS)) {
ARMPagingStructures32Bit::UpdateAllPageDirs(index, pd[index]);
}
fMapCount++;
}
// now, fill in the pentry
Thread* thread = thread_get_current_thread();
ThreadCPUPinner pinner(thread);
page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt(
pd[index] & ARM_PDE_ADDRESS_MASK);
index = VADDR_TO_PTENT(va);
ASSERT_PRINT((pt[index] & ARM_PTE_TYPE_MASK) == 0,
"virtual address: %#" B_PRIxADDR ", existing pte: %#" B_PRIx32, va,
pt[index]);
ARMPagingMethod32Bit::PutPageTableEntryInTable(&pt[index], pa, attributes,
memoryType, fIsKernelMap);
pinner.Unlock();
// Note: We don't need to invalidate the TLB for this address, as previously
// the entry was not present and the TLB doesn't cache those entries.
fMapCount++;
return 0;
}
void
X86VMTranslationMap64Bit::UnmapArea(VMArea* area, bool deletingAddressSpace,
bool ignoreTopCachePageFlags)
{
TRACE("X86VMTranslationMap64Bit::UnmapArea(%p)\n", area);
if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) {
X86VMTranslationMap64Bit::UnmapPages(area, area->Base(), area->Size(),
true);
return;
}
bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags;
RecursiveLocker locker(fLock);
ThreadCPUPinner pinner(thread_get_current_thread());
VMAreaMappings mappings;
mappings.MoveFrom(&area->mappings);
for (VMAreaMappings::Iterator it = mappings.GetIterator();
vm_page_mapping* mapping = it.Next();) {
vm_page* page = mapping->page;
page->mappings.Remove(mapping);
VMCache* cache = page->Cache();
bool pageFullyUnmapped = false;
if (!page->IsMapped()) {
atomic_add(&gMappedPagesCount, -1);
pageFullyUnmapped = true;
}
if (unmapPages || cache != area->cache) {
addr_t address = area->Base()
+ ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset);
uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress(
fPagingStructures->VirtualPML4(), address, fIsKernelMap,
false, NULL, fPageMapper, fMapCount);
if (entry == NULL) {
panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
"has no page table", page, area, address);
continue;
}
uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(entry);
if ((oldEntry & X86_64_PTE_PRESENT) == 0) {
panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
"has no page table entry", page, area, address);
continue;
}
// transfer the accessed/dirty flags to the page and invalidate
// the mapping, if necessary
if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
page->accessed = true;
if (!deletingAddressSpace)
InvalidatePage(address);
}
if ((oldEntry & X86_64_PTE_DIRTY) != 0)
page->modified = true;
if (pageFullyUnmapped) {
DEBUG_PAGE_ACCESS_START(page);
if (cache->temporary)
vm_page_set_state(page, PAGE_STATE_INACTIVE);
else if (page->modified)
vm_page_set_state(page, PAGE_STATE_MODIFIED);
else
vm_page_set_state(page, PAGE_STATE_CACHED);
DEBUG_PAGE_ACCESS_END(page);
}
}
fMapCount--;
}
Flush();
// flush explicitely, since we directly use the lock
locker.Unlock();
bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = mappings.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
}
void
X86VMTranslationMap64Bit::UnmapPages(VMArea* area, addr_t base, size_t size,
bool updatePageQueue)
{
if (size == 0)
return;
addr_t start = base;
addr_t end = base + size - 1;
TRACE("X86VMTranslationMap64Bit::UnmapPages(%p, %#" B_PRIxADDR ", %#"
B_PRIxADDR ")\n", area, start, end);
VMAreaMappings queue;
RecursiveLocker locker(fLock);
ThreadCPUPinner pinner(thread_get_current_thread());
do {
uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress(
fPagingStructures->VirtualPML4(), start, fIsKernelMap, false,
NULL, fPageMapper, fMapCount);
if (pageTable == NULL) {
// Move on to the next page table.
start = ROUNDUP(start + 1, k64BitPageTableRange);
continue;
}
for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount;
index < k64BitTableEntryCount && start < end;
index++, start += B_PAGE_SIZE) {
uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(
&pageTable[index]);
if ((oldEntry & X86_64_PTE_PRESENT) == 0)
continue;
fMapCount--;
if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
// Note, that we only need to invalidate the address, if the
// accessed flags was set, since only then the entry could have
// been in any TLB.
InvalidatePage(start);
}
if (area->cache_type != CACHE_TYPE_DEVICE) {
// get the page
vm_page* page = vm_lookup_page(
(oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE);
ASSERT(page != NULL);
DEBUG_PAGE_ACCESS_START(page);
// transfer the accessed/dirty flags to the page
if ((oldEntry & X86_64_PTE_ACCESSED) != 0)
page->accessed = true;
if ((oldEntry & X86_64_PTE_DIRTY) != 0)
page->modified = true;
// remove the mapping object/decrement the wired_count of the
// page
if (area->wiring == B_NO_LOCK) {
vm_page_mapping* mapping = NULL;
vm_page_mappings::Iterator iterator
= page->mappings.GetIterator();
while ((mapping = iterator.Next()) != NULL) {
if (mapping->area == area)
break;
}
ASSERT(mapping != NULL);
area->mappings.Remove(mapping);
page->mappings.Remove(mapping);
queue.Add(mapping);
} else
page->DecrementWiredCount();
if (!page->IsMapped()) {
atomic_add(&gMappedPagesCount, -1);
if (updatePageQueue) {
if (page->Cache()->temporary)
vm_page_set_state(page, PAGE_STATE_INACTIVE);
else if (page->modified)
vm_page_set_state(page, PAGE_STATE_MODIFIED);
else
vm_page_set_state(page, PAGE_STATE_CACHED);
}
}
DEBUG_PAGE_ACCESS_END(page);
}
}
Flush();
// flush explicitly, since we directly use the lock
} while (start != 0 && start < end);
// TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not
// really critical here, as in all cases this method is used, the unmapped
// area range is unmapped for good (resized/cut) and the pages will likely
// be freed.
locker.Unlock();
// free removed mappings
bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = queue.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
}
M68KVMTranslationMap040::~M68KVMTranslationMap040()
{
if (fPagingStructures == NULL)
return;
if (fPageMapper != NULL)
fPageMapper->Delete();
if (fPagingStructures->pgroot_virt != NULL) {
page_root_entry *pgroot_virt = fPagingStructures->pgroot_virt;
// cycle through and free all of the user space pgdirs & pgtables
// since the size of tables don't match B_PAGE_SIZE,
// we alloc several at once, based on modulos,
// we make sure they are either all in the tree or none.
for (uint32 i = VADDR_TO_PRENT(USER_BASE);
i <= VADDR_TO_PRENT(USER_BASE + (USER_SIZE - 1)); i++) {
addr_t pgdir_pn;
page_directory_entry *pgdir;
vm_page *dirpage;
if (PRE_TYPE(pgroot_virt[i]) == DT_INVALID)
continue;
if (PRE_TYPE(pgroot_virt[i]) != DT_ROOT) {
panic("rtdir[%ld]: buggy descriptor type", i);
return;
}
// XXX:suboptimal (done 8 times)
pgdir_pn = PRE_TO_PN(pgroot_virt[i]);
dirpage = vm_lookup_page(pgdir_pn);
pgdir = &(((page_directory_entry *)dirpage)[i%NUM_DIRTBL_PER_PAGE]);
for (uint32 j = 0; j <= NUM_DIRENT_PER_TBL;
j+=NUM_PAGETBL_PER_PAGE) {
addr_t pgtbl_pn;
page_table_entry *pgtbl;
vm_page *page;
if (PDE_TYPE(pgdir[j]) == DT_INVALID)
continue;
if (PDE_TYPE(pgdir[j]) != DT_DIR) {
panic("pgroot[%ld][%ld]: buggy descriptor type", i, j);
return;
}
pgtbl_pn = PDE_TO_PN(pgdir[j]);
page = vm_lookup_page(pgtbl_pn);
pgtbl = (page_table_entry *)page;
if (!page) {
panic("destroy_tmap: didn't find pgtable page\n");
return;
}
DEBUG_PAGE_ACCESS_START(page);
vm_page_set_state(page, PAGE_STATE_FREE);
}
if (((i + 1) % NUM_DIRTBL_PER_PAGE) == 0) {
DEBUG_PAGE_ACCESS_END(dirpage);
vm_page_set_state(dirpage, PAGE_STATE_FREE);
}
}
#if 0
//X86
for (uint32 i = VADDR_TO_PDENT(USER_BASE);
i <= VADDR_TO_PDENT(USER_BASE + (USER_SIZE - 1)); i++) {
if ((fPagingStructures->pgdir_virt[i] & M68K_PDE_PRESENT) != 0) {
addr_t address = fPagingStructures->pgdir_virt[i]
& M68K_PDE_ADDRESS_MASK;
vm_page* page = vm_lookup_page(address / B_PAGE_SIZE);
if (!page)
panic("destroy_tmap: didn't find pgtable page\n");
DEBUG_PAGE_ACCESS_START(page);
vm_page_set_state(page, PAGE_STATE_FREE);
}
}
#endif
}
fPagingStructures->RemoveReference();
}
status_t
M68KVMTranslationMap040::Map(addr_t va, phys_addr_t pa, uint32 attributes,
uint32 memoryType, vm_page_reservation* reservation)
{
TRACE("M68KVMTranslationMap040::Map: entry pa 0x%lx va 0x%lx\n", pa, va);
/*
dprintf("pgdir at 0x%x\n", pgdir);
dprintf("index is %d\n", va / B_PAGE_SIZE / 1024);
dprintf("final at 0x%x\n", &pgdir[va / B_PAGE_SIZE / 1024]);
dprintf("value is 0x%x\n", *(int *)&pgdir[va / B_PAGE_SIZE / 1024]);
dprintf("present bit is %d\n", pgdir[va / B_PAGE_SIZE / 1024].present);
dprintf("addr is %d\n", pgdir[va / B_PAGE_SIZE / 1024].addr);
*/
page_root_entry *pr = fPagingStructures->pgroot_virt;
page_directory_entry *pd;
page_table_entry *pt;
addr_t pd_pg, pt_pg;
uint32 rindex, dindex, pindex;
// check to see if a page directory exists for this range
rindex = VADDR_TO_PRENT(va);
if (PRE_TYPE(pr[rindex]) != DT_ROOT) {
phys_addr_t pgdir;
vm_page *page;
uint32 i;
// we need to allocate a pgdir group
page = vm_page_allocate_page(reservation,
PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);
DEBUG_PAGE_ACCESS_END(page);
pgdir = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE;
TRACE("::Map: asked for free page for pgdir. 0x%lx\n", pgdir);
// for each pgdir on the allocated page:
for (i = 0; i < NUM_DIRTBL_PER_PAGE; i++) {
uint32 aindex = rindex & ~(NUM_DIRTBL_PER_PAGE-1); /* aligned */
page_root_entry *apr = &pr[aindex + i];
// put in the pgroot
M68KPagingMethod040::PutPageDirInPageRoot(apr, pgdir, attributes
| ((attributes & B_USER_PROTECTION) != 0
? B_WRITE_AREA : B_KERNEL_WRITE_AREA));
// update any other page roots, if it maps kernel space
//XXX: suboptimal, should batch them
if ((aindex+i) >= FIRST_KERNEL_PGDIR_ENT && (aindex+i)
< (FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS))
M68KPagingStructures040::UpdateAllPageDirs((aindex+i),
pr[aindex+i]);
pgdir += SIZ_DIRTBL;
}
fMapCount++;
}
// now, fill in the pentry
//XXX: is this required?
Thread* thread = thread_get_current_thread();
ThreadCPUPinner pinner(thread);
pd = (page_directory_entry*)MapperGetPageTableAt(
PRE_TO_PA(pr[rindex]));
//pinner.Unlock();
// we want the table at rindex, not at rindex%(tbl/page)
//pd += (rindex % NUM_DIRTBL_PER_PAGE) * NUM_DIRENT_PER_TBL;
// check to see if a page table exists for this range
dindex = VADDR_TO_PDENT(va);
if (PDE_TYPE(pd[dindex]) != DT_DIR) {
phys_addr_t pgtable;
vm_page *page;
uint32 i;
// we need to allocate a pgtable group
page = vm_page_allocate_page(reservation,
PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);
DEBUG_PAGE_ACCESS_END(page);
pgtable = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE;
TRACE("::Map: asked for free page for pgtable. 0x%lx\n", pgtable);
// for each pgtable on the allocated page:
for (i = 0; i < NUM_PAGETBL_PER_PAGE; i++) {
uint32 aindex = dindex & ~(NUM_PAGETBL_PER_PAGE-1); /* aligned */
page_directory_entry *apd = &pd[aindex + i];
// put in the pgdir
M68KPagingMethod040::PutPageTableInPageDir(apd, pgtable, attributes
| ((attributes & B_USER_PROTECTION) != 0
? B_WRITE_AREA : B_KERNEL_WRITE_AREA));
// no need to update other page directories for kernel space;
// the root-level already point to us.
pgtable += SIZ_PAGETBL;
}
#warning M68K: really mean map_count++ ??
fMapCount++;
}
// now, fill in the pentry
//ThreadCPUPinner pinner(thread);
pt = (page_table_entry*)MapperGetPageTableAt(PDE_TO_PA(pd[dindex]));
// we want the table at rindex, not at rindex%(tbl/page)
//pt += (dindex % NUM_PAGETBL_PER_PAGE) * NUM_PAGEENT_PER_TBL;
pindex = VADDR_TO_PTENT(va);
ASSERT_PRINT((PTE_TYPE(pt[pindex]) != DT_INVALID) == 0,
"virtual address: %#" B_PRIxADDR ", existing pte: %#" B_PRIx32, va,
pt[pindex]);
M68KPagingMethod040::PutPageTableEntryInTable(&pt[pindex], pa, attributes,
memoryType, fIsKernelMap);
pinner.Unlock();
// Note: We don't need to invalidate the TLB for this address, as previously
// the entry was not present and the TLB doesn't cache those entries.
fMapCount++;
return B_OK;
}
static status_t
cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer,
size_t* _size, bool doWrite)
{
if (_cacheRef == NULL)
panic("cache_io() called with NULL ref!\n");
file_cache_ref* ref = (file_cache_ref*)_cacheRef;
VMCache* cache = ref->cache;
off_t fileSize = cache->virtual_end;
bool useBuffer = buffer != 0;
TRACE(("cache_io(ref = %p, offset = %Ld, buffer = %p, size = %lu, %s)\n",
ref, offset, (void*)buffer, *_size, doWrite ? "write" : "read"));
// out of bounds access?
if (offset >= fileSize || offset < 0) {
*_size = 0;
return B_OK;
}
int32 pageOffset = offset & (B_PAGE_SIZE - 1);
size_t size = *_size;
offset -= pageOffset;
if ((off_t)(offset + pageOffset + size) > fileSize) {
// adapt size to be within the file's offsets
size = fileSize - pageOffset - offset;
*_size = size;
}
if (size == 0)
return B_OK;
// "offset" and "lastOffset" are always aligned to B_PAGE_SIZE,
// the "last*" variables always point to the end of the last
// satisfied request part
const uint32 kMaxChunkSize = MAX_IO_VECS * B_PAGE_SIZE;
size_t bytesLeft = size, lastLeft = size;
int32 lastPageOffset = pageOffset;
addr_t lastBuffer = buffer;
off_t lastOffset = offset;
size_t lastReservedPages = min_c(MAX_IO_VECS, (pageOffset + bytesLeft
+ B_PAGE_SIZE - 1) >> PAGE_SHIFT);
size_t reservePages = 0;
size_t pagesProcessed = 0;
cache_func function = NULL;
vm_page_reservation reservation;
reserve_pages(ref, &reservation, lastReservedPages, doWrite);
AutoLocker<VMCache> locker(cache);
while (bytesLeft > 0) {
// Periodically reevaluate the low memory situation and select the
// read/write hook accordingly
if (pagesProcessed % 32 == 0) {
if (size >= BYPASS_IO_SIZE
&& low_resource_state(B_KERNEL_RESOURCE_PAGES)
!= B_NO_LOW_RESOURCE) {
// In low memory situations we bypass the cache beyond a
// certain I/O size.
function = doWrite ? write_to_file : read_from_file;
} else
function = doWrite ? write_to_cache : read_into_cache;
}
// check if this page is already in memory
vm_page* page = cache->LookupPage(offset);
if (page != NULL) {
// The page may be busy - since we need to unlock the cache sometime
// in the near future, we need to satisfy the request of the pages
// we didn't get yet (to make sure no one else interferes in the
// meantime).
status_t status = satisfy_cache_io(ref, cookie, function, offset,
buffer, useBuffer, pageOffset, bytesLeft, reservePages,
lastOffset, lastBuffer, lastPageOffset, lastLeft,
lastReservedPages, &reservation);
if (status != B_OK)
return status;
// Since satisfy_cache_io() unlocks the cache, we need to look up
// the page again.
page = cache->LookupPage(offset);
if (page != NULL && page->busy) {
cache->WaitForPageEvents(page, PAGE_EVENT_NOT_BUSY, true);
continue;
}
}
size_t bytesInPage = min_c(size_t(B_PAGE_SIZE - pageOffset), bytesLeft);
TRACE(("lookup page from offset %Ld: %p, size = %lu, pageOffset "
"= %lu\n", offset, page, bytesLeft, pageOffset));
if (page != NULL) {
if (doWrite || useBuffer) {
// Since the following user_mem{cpy,set}() might cause a page
// fault, which in turn might cause pages to be reserved, we
// need to unlock the cache temporarily to avoid a potential
// deadlock. To make sure that our page doesn't go away, we mark
// it busy for the time.
page->busy = true;
locker.Unlock();
// copy the contents of the page already in memory
phys_addr_t pageAddress
= (phys_addr_t)page->physical_page_number * B_PAGE_SIZE
+ pageOffset;
bool userBuffer = IS_USER_ADDRESS(buffer);
if (doWrite) {
if (useBuffer) {
vm_memcpy_to_physical(pageAddress, (void*)buffer,
bytesInPage, userBuffer);
} else {
vm_memset_physical(pageAddress, 0, bytesInPage);
}
} else if (useBuffer) {
vm_memcpy_from_physical((void*)buffer, pageAddress,
bytesInPage, userBuffer);
}
locker.Lock();
if (doWrite) {
DEBUG_PAGE_ACCESS_START(page);
page->modified = true;
if (page->State() != PAGE_STATE_MODIFIED)
vm_page_set_state(page, PAGE_STATE_MODIFIED);
DEBUG_PAGE_ACCESS_END(page);
}
cache->MarkPageUnbusy(page);
}
// If it is cached only, requeue the page, so the respective queue
// roughly remains LRU first sorted.
if (page->State() == PAGE_STATE_CACHED
|| page->State() == PAGE_STATE_MODIFIED) {
DEBUG_PAGE_ACCESS_START(page);
vm_page_requeue(page, true);
DEBUG_PAGE_ACCESS_END(page);
}
if (bytesLeft <= bytesInPage) {
// we've read the last page, so we're done!
locker.Unlock();
vm_page_unreserve_pages(&reservation);
return B_OK;
}
// prepare a potential gap request
lastBuffer = buffer + bytesInPage;
lastLeft = bytesLeft - bytesInPage;
lastOffset = offset + B_PAGE_SIZE;
lastPageOffset = 0;
}
if (bytesLeft <= bytesInPage)
break;
buffer += bytesInPage;
bytesLeft -= bytesInPage;
pageOffset = 0;
offset += B_PAGE_SIZE;
pagesProcessed++;
if (buffer - lastBuffer + lastPageOffset >= kMaxChunkSize) {
status_t status = satisfy_cache_io(ref, cookie, function, offset,
buffer, useBuffer, pageOffset, bytesLeft, reservePages,
lastOffset, lastBuffer, lastPageOffset, lastLeft,
lastReservedPages, &reservation);
if (status != B_OK)
return status;
}
}
// fill the last remaining bytes of the request (either write or read)
return function(ref, cookie, lastOffset, lastPageOffset, lastBuffer,
lastLeft, useBuffer, &reservation, 0);
}
/*! Like read_into_cache() but writes data into the cache.
To preserve data consistency, it might also read pages into the cache,
though, if only a partial page gets written.
The same restrictions apply.
*/
static status_t
write_to_cache(file_cache_ref* ref, void* cookie, off_t offset,
int32 pageOffset, addr_t buffer, size_t bufferSize, bool useBuffer,
vm_page_reservation* reservation, size_t reservePages)
{
// TODO: We're using way too much stack! Rather allocate a sufficiently
// large chunk on the heap.
generic_io_vec vecs[MAX_IO_VECS];
uint32 vecCount = 0;
generic_size_t numBytes = PAGE_ALIGN(pageOffset + bufferSize);
vm_page* pages[MAX_IO_VECS];
int32 pageIndex = 0;
status_t status = B_OK;
// ToDo: this should be settable somewhere
bool writeThrough = false;
// allocate pages for the cache and mark them busy
for (generic_size_t pos = 0; pos < numBytes; pos += B_PAGE_SIZE) {
// TODO: if space is becoming tight, and this cache is already grown
// big - shouldn't we better steal the pages directly in that case?
// (a working set like approach for the file cache)
// TODO: the pages we allocate here should have been reserved upfront
// in cache_io()
vm_page* page = pages[pageIndex++] = vm_page_allocate_page(
reservation,
(writeThrough ? PAGE_STATE_CACHED : PAGE_STATE_MODIFIED)
| VM_PAGE_ALLOC_BUSY);
page->modified = !writeThrough;
ref->cache->InsertPage(page, offset + pos);
add_to_iovec(vecs, vecCount, MAX_IO_VECS,
page->physical_page_number * B_PAGE_SIZE, B_PAGE_SIZE);
}
push_access(ref, offset, bufferSize, true);
ref->cache->Unlock();
vm_page_unreserve_pages(reservation);
// copy contents (and read in partially written pages first)
if (pageOffset != 0) {
// This is only a partial write, so we have to read the rest of the page
// from the file to have consistent data in the cache
generic_io_vec readVec = { vecs[0].base, B_PAGE_SIZE };
generic_size_t bytesRead = B_PAGE_SIZE;
status = vfs_read_pages(ref->vnode, cookie, offset, &readVec, 1,
B_PHYSICAL_IO_REQUEST, &bytesRead);
// ToDo: handle errors for real!
if (status < B_OK)
panic("1. vfs_read_pages() failed: %s!\n", strerror(status));
}
size_t lastPageOffset = (pageOffset + bufferSize) % B_PAGE_SIZE;
if (lastPageOffset != 0) {
// get the last page in the I/O vectors
generic_addr_t last = vecs[vecCount - 1].base
+ vecs[vecCount - 1].length - B_PAGE_SIZE;
if ((off_t)(offset + pageOffset + bufferSize) == ref->cache->virtual_end) {
// the space in the page after this write action needs to be cleaned
vm_memset_physical(last + lastPageOffset, 0,
B_PAGE_SIZE - lastPageOffset);
} else {
// the end of this write does not happen on a page boundary, so we
// need to fetch the last page before we can update it
generic_io_vec readVec = { last, B_PAGE_SIZE };
generic_size_t bytesRead = B_PAGE_SIZE;
status = vfs_read_pages(ref->vnode, cookie,
PAGE_ALIGN(offset + pageOffset + bufferSize) - B_PAGE_SIZE,
&readVec, 1, B_PHYSICAL_IO_REQUEST, &bytesRead);
// ToDo: handle errors for real!
if (status < B_OK)
panic("vfs_read_pages() failed: %s!\n", strerror(status));
if (bytesRead < B_PAGE_SIZE) {
// the space beyond the file size needs to be cleaned
vm_memset_physical(last + bytesRead, 0,
B_PAGE_SIZE - bytesRead);
}
}
}
for (uint32 i = 0; i < vecCount; i++) {
generic_addr_t base = vecs[i].base;
generic_size_t bytes = min_c((generic_size_t)bufferSize,
generic_size_t(vecs[i].length - pageOffset));
if (useBuffer) {
// copy data from user buffer
vm_memcpy_to_physical(base + pageOffset, (void*)buffer, bytes,
IS_USER_ADDRESS(buffer));
} else {
// clear buffer instead
vm_memset_physical(base + pageOffset, 0, bytes);
}
bufferSize -= bytes;
if (bufferSize == 0)
break;
buffer += bytes;
pageOffset = 0;
}
if (writeThrough) {
// write cached pages back to the file if we were asked to do that
status_t status = vfs_write_pages(ref->vnode, cookie, offset, vecs,
vecCount, B_PHYSICAL_IO_REQUEST, &numBytes);
if (status < B_OK) {
// ToDo: remove allocated pages, ...?
panic("file_cache: remove allocated pages! write pages failed: %s\n",
strerror(status));
}
}
if (status == B_OK)
reserve_pages(ref, reservation, reservePages, true);
ref->cache->Lock();
// make the pages accessible in the cache
for (int32 i = pageIndex; i-- > 0;) {
ref->cache->MarkPageUnbusy(pages[i]);
DEBUG_PAGE_ACCESS_END(pages[i]);
}
return status;
}
/*! Reads the requested amount of data into the cache, and allocates
pages needed to fulfill that request. This function is called by cache_io().
It can only handle a certain amount of bytes, and the caller must make
sure that it matches that criterion.
The cache_ref lock must be held when calling this function; during
operation it will unlock the cache, though.
*/
static status_t
read_into_cache(file_cache_ref* ref, void* cookie, off_t offset,
int32 pageOffset, addr_t buffer, size_t bufferSize, bool useBuffer,
vm_page_reservation* reservation, size_t reservePages)
{
TRACE(("read_into_cache(offset = %Ld, pageOffset = %ld, buffer = %#lx, "
"bufferSize = %lu\n", offset, pageOffset, buffer, bufferSize));
VMCache* cache = ref->cache;
// TODO: We're using way too much stack! Rather allocate a sufficiently
// large chunk on the heap.
generic_io_vec vecs[MAX_IO_VECS];
uint32 vecCount = 0;
generic_size_t numBytes = PAGE_ALIGN(pageOffset + bufferSize);
vm_page* pages[MAX_IO_VECS];
int32 pageIndex = 0;
// allocate pages for the cache and mark them busy
for (generic_size_t pos = 0; pos < numBytes; pos += B_PAGE_SIZE) {
vm_page* page = pages[pageIndex++] = vm_page_allocate_page(
reservation, PAGE_STATE_CACHED | VM_PAGE_ALLOC_BUSY);
cache->InsertPage(page, offset + pos);
add_to_iovec(vecs, vecCount, MAX_IO_VECS,
page->physical_page_number * B_PAGE_SIZE, B_PAGE_SIZE);
// TODO: check if the array is large enough (currently panics)!
}
push_access(ref, offset, bufferSize, false);
cache->Unlock();
vm_page_unreserve_pages(reservation);
// read file into reserved pages
status_t status = read_pages_and_clear_partial(ref, cookie, offset, vecs,
vecCount, B_PHYSICAL_IO_REQUEST, &numBytes);
if (status != B_OK) {
// reading failed, free allocated pages
dprintf("file_cache: read pages failed: %s\n", strerror(status));
cache->Lock();
for (int32 i = 0; i < pageIndex; i++) {
cache->NotifyPageEvents(pages[i], PAGE_EVENT_NOT_BUSY);
cache->RemovePage(pages[i]);
vm_page_set_state(pages[i], PAGE_STATE_FREE);
}
return status;
}
// copy the pages if needed and unmap them again
for (int32 i = 0; i < pageIndex; i++) {
if (useBuffer && bufferSize != 0) {
size_t bytes = min_c(bufferSize, (size_t)B_PAGE_SIZE - pageOffset);
vm_memcpy_from_physical((void*)buffer,
pages[i]->physical_page_number * B_PAGE_SIZE + pageOffset,
bytes, IS_USER_ADDRESS(buffer));
buffer += bytes;
bufferSize -= bytes;
pageOffset = 0;
}
}
reserve_pages(ref, reservation, reservePages, false);
cache->Lock();
// make the pages accessible in the cache
for (int32 i = pageIndex; i-- > 0;) {
DEBUG_PAGE_ACCESS_END(pages[i]);
cache->MarkPageUnbusy(pages[i]);
}
return B_OK;
}
