/*! Unmaps a range of pages of an area.
The default implementation just iterates over all virtual pages of the
range and calls UnmapPage(). This is obviously not particularly efficient.
*/
void
VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size,
bool updatePageQueue)
{
ASSERT(base % B_PAGE_SIZE == 0);
ASSERT(size % B_PAGE_SIZE == 0);
addr_t address = base;
addr_t end = address + size;
#if DEBUG_PAGE_ACCESS
for (; address != end; address += B_PAGE_SIZE) {
phys_addr_t physicalAddress;
uint32 flags;
if (Query(address, &physicalAddress, &flags) == B_OK
&& (flags & PAGE_PRESENT) != 0) {
vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE);
if (page != NULL) {
DEBUG_PAGE_ACCESS_START(page);
UnmapPage(area, address, updatePageQueue);
DEBUG_PAGE_ACCESS_END(page);
} else
UnmapPage(area, address, updatePageQueue);
}
}
#else
for (; address != end; address += B_PAGE_SIZE)
UnmapPage(area, address, updatePageQueue);
#endif
}
/*! Unmaps all of an area's pages.
If \a deletingAddressSpace is \c true, the address space the area belongs to
is in the process of being destroyed and isn't used by anyone anymore. For
some architectures this can be used for optimizations (e.g. not unmapping
pages or at least not needing to invalidate TLB entries).
If \a ignoreTopCachePageFlags is \c true, the area is in the process of
being destroyed and its top cache is otherwise unreferenced. I.e. all mapped
pages that live in the top cache area going to be freed and the page
accessed and modified flags don't need to be propagated.
The default implementation just iterates over all virtual pages of the
area and calls UnmapPage(). This is obviously not particularly efficient.
*/
void
VMTranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace,
bool ignoreTopCachePageFlags)
{
addr_t address = area->Base();
addr_t end = address + area->Size();
#if DEBUG_PAGE_ACCESS
for (; address != end; address += B_PAGE_SIZE) {
phys_addr_t physicalAddress;
uint32 flags;
if (Query(address, &physicalAddress, &flags) == B_OK
&& (flags & PAGE_PRESENT) != 0) {
vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE);
if (page != NULL) {
DEBUG_PAGE_ACCESS_START(page);
UnmapPage(area, address, true);
DEBUG_PAGE_ACCESS_END(page);
} else
UnmapPage(area, address, true);
}
}
#else
for (; address != end; address += B_PAGE_SIZE)
UnmapPage(area, address, true);
#endif
}
ARMVMTranslationMap32Bit::~ARMVMTranslationMap32Bit()
{
if (fPagingStructures == NULL)
return;
if (fPageMapper != NULL)
fPageMapper->Delete();
if (fPagingStructures->pgdir_virt != NULL) {
// cycle through and free all of the user space pgtables
for (uint32 i = VADDR_TO_PDENT(USER_BASE);
i <= VADDR_TO_PDENT(USER_BASE + (USER_SIZE - 1)); i++) {
if ((fPagingStructures->pgdir_virt[i] & ARM_PDE_TYPE_MASK) != 0) {
addr_t address = fPagingStructures->pgdir_virt[i]
& ARM_PDE_ADDRESS_MASK;
vm_page* page = vm_lookup_page(address / B_PAGE_SIZE);
if (!page)
panic("destroy_tmap: didn't find pgtable page\n");
DEBUG_PAGE_ACCESS_START(page);
vm_page_set_state(page, PAGE_STATE_FREE);
}
}
}
fPagingStructures->RemoveReference();
}
static void
reserve_pages(file_cache_ref* ref, vm_page_reservation* reservation,
size_t reservePages, bool isWrite)
{
if (low_resource_state(B_KERNEL_RESOURCE_PAGES) != B_NO_LOW_RESOURCE) {
VMCache* cache = ref->cache;
cache->Lock();
if (cache->consumers.IsEmpty() && cache->areas == NULL
&& access_is_sequential(ref)) {
// we are not mapped, and we're accessed sequentially
if (isWrite) {
// Just write some pages back, and actually wait until they
// have been written back in order to relieve the page pressure
// a bit.
int32 index = ref->last_access_index;
int32 previous = index - 1;
if (previous < 0)
previous = LAST_ACCESSES - 1;
vm_page_write_modified_page_range(cache,
ref->LastAccessPageOffset(previous, true),
ref->LastAccessPageOffset(index, true));
} else {
// free some pages from our cache
// TODO: start with oldest
uint32 left = reservePages;
vm_page* page;
for (VMCachePagesTree::Iterator it = cache->pages.GetIterator();
(page = it.Next()) != NULL && left > 0;) {
if (page->State() == PAGE_STATE_CACHED && !page->busy) {
DEBUG_PAGE_ACCESS_START(page);
ASSERT(!page->IsMapped());
ASSERT(!page->modified);
cache->RemovePage(page);
vm_page_set_state(page, PAGE_STATE_FREE);
left--;
}
}
}
}
cache->Unlock();
}
vm_page_reserve_pages(reservation, reservePages, VM_PRIORITY_USER);
}
void
ARMVMTranslationMap32Bit::UnmapArea(VMArea* area, bool deletingAddressSpace,
bool ignoreTopCachePageFlags)
{
if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) {
ARMVMTranslationMap32Bit::UnmapPages(area, area->Base(), area->Size(),
true);
return;
}
bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags;
page_directory_entry* pd = fPagingStructures->pgdir_virt;
RecursiveLocker locker(fLock);
VMAreaMappings mappings;
mappings.MoveFrom(&area->mappings);
for (VMAreaMappings::Iterator it = mappings.GetIterator();
vm_page_mapping* mapping = it.Next();) {
vm_page* page = mapping->page;
page->mappings.Remove(mapping);
VMCache* cache = page->Cache();
bool pageFullyUnmapped = false;
if (!page->IsMapped()) {
atomic_add(&gMappedPagesCount, -1);
pageFullyUnmapped = true;
}
if (unmapPages || cache != area->cache) {
addr_t address = area->Base()
+ ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset);
int index = VADDR_TO_PDENT(address);
if ((pd[index] & ARM_PDE_TYPE_MASK) == 0) {
panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
"has no page dir entry", page, area, address);
continue;
}
ThreadCPUPinner pinner(thread_get_current_thread());
page_table_entry* pt
= (page_table_entry*)fPageMapper->GetPageTableAt(
pd[index] & ARM_PDE_ADDRESS_MASK);
page_table_entry oldEntry
= ARMPagingMethod32Bit::ClearPageTableEntry(
&pt[VADDR_TO_PTENT(address)]);
pinner.Unlock();
if ((oldEntry & ARM_PTE_TYPE_MASK) == 0) {
panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
"has no page table entry", page, area, address);
continue;
}
// transfer the accessed/dirty flags to the page and invalidate
// the mapping, if necessary
if (true /*(oldEntry & ARM_PTE_ACCESSED) != 0*/) { // XXX IRA
page->accessed = true;
if (!deletingAddressSpace)
InvalidatePage(address);
}
if (true /*(oldEntry & ARM_PTE_DIRTY) != 0*/)
page->modified = true;
if (pageFullyUnmapped) {
DEBUG_PAGE_ACCESS_START(page);
if (cache->temporary)
vm_page_set_state(page, PAGE_STATE_INACTIVE);
else if (page->modified)
vm_page_set_state(page, PAGE_STATE_MODIFIED);
else
vm_page_set_state(page, PAGE_STATE_CACHED);
DEBUG_PAGE_ACCESS_END(page);
}
}
fMapCount--;
}
Flush();
// flush explicitely, since we directly use the lock
locker.Unlock();
bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = mappings.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
}
void
ARMVMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size,
bool updatePageQueue)
{
if (size == 0)
return;
addr_t start = base;
addr_t end = base + size - 1;
TRACE("ARMVMTranslationMap32Bit::UnmapPages(%p, %#" B_PRIxADDR ", %#"
B_PRIxADDR ")\n", area, start, end);
page_directory_entry* pd = fPagingStructures->pgdir_virt;
VMAreaMappings queue;
RecursiveLocker locker(fLock);
do {
int index = VADDR_TO_PDENT(start);
if ((pd[index] & ARM_PDE_TYPE_MASK) == 0) {
// no page table here, move the start up to access the next page
// table
start = ROUNDUP(start + 1, kPageTableAlignment);
continue;
}
Thread* thread = thread_get_current_thread();
ThreadCPUPinner pinner(thread);
page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt(
pd[index] & ARM_PDE_ADDRESS_MASK);
for (index = VADDR_TO_PTENT(start); (index < 1024) && (start < end);
index++, start += B_PAGE_SIZE) {
page_table_entry oldEntry
= ARMPagingMethod32Bit::ClearPageTableEntry(&pt[index]);
if ((oldEntry & ARM_PTE_TYPE_MASK) == 0)
continue;
fMapCount--;
if (true /*(oldEntry & ARM_PTE_ACCESSED) != 0*/) { // XXX IRA
// Note, that we only need to invalidate the address, if the
// accessed flags was set, since only then the entry could have
// been in any TLB.
InvalidatePage(start);
}
if (area->cache_type != CACHE_TYPE_DEVICE) {
// get the page
vm_page* page = vm_lookup_page(
(oldEntry & ARM_PTE_ADDRESS_MASK) / B_PAGE_SIZE);
ASSERT(page != NULL);
DEBUG_PAGE_ACCESS_START(page);
// transfer the accessed/dirty flags to the page
if (/*(oldEntry & ARM_PTE_ACCESSED) != 0*/ true) // XXX IRA
page->accessed = true;
if (/*(oldEntry & ARM_PTE_DIRTY) != 0 */ true)
page->modified = true;
// remove the mapping object/decrement the wired_count of the
// page
if (area->wiring == B_NO_LOCK) {
vm_page_mapping* mapping = NULL;
vm_page_mappings::Iterator iterator
= page->mappings.GetIterator();
while ((mapping = iterator.Next()) != NULL) {
if (mapping->area == area)
break;
}
ASSERT(mapping != NULL);
area->mappings.Remove(mapping);
page->mappings.Remove(mapping);
queue.Add(mapping);
} else
page->DecrementWiredCount();
if (!page->IsMapped()) {
atomic_add(&gMappedPagesCount, -1);
if (updatePageQueue) {
if (page->Cache()->temporary)
vm_page_set_state(page, PAGE_STATE_INACTIVE);
else if (page->modified)
vm_page_set_state(page, PAGE_STATE_MODIFIED);
else
vm_page_set_state(page, PAGE_STATE_CACHED);
}
}
DEBUG_PAGE_ACCESS_END(page);
}
}
Flush();
// flush explicitly, since we directly use the lock
} while (start != 0 && start < end);
// TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not
// really critical here, as in all cases this method is used, the unmapped
// area range is unmapped for good (resized/cut) and the pages will likely
// be freed.
locker.Unlock();
// free removed mappings
bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = queue.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
}
void
X86VMTranslationMap64Bit::UnmapArea(VMArea* area, bool deletingAddressSpace,
bool ignoreTopCachePageFlags)
{
TRACE("X86VMTranslationMap64Bit::UnmapArea(%p)\n", area);
if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) {
X86VMTranslationMap64Bit::UnmapPages(area, area->Base(), area->Size(),
true);
return;
}
bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags;
RecursiveLocker locker(fLock);
ThreadCPUPinner pinner(thread_get_current_thread());
VMAreaMappings mappings;
mappings.MoveFrom(&area->mappings);
for (VMAreaMappings::Iterator it = mappings.GetIterator();
vm_page_mapping* mapping = it.Next();) {
vm_page* page = mapping->page;
page->mappings.Remove(mapping);
VMCache* cache = page->Cache();
bool pageFullyUnmapped = false;
if (!page->IsMapped()) {
atomic_add(&gMappedPagesCount, -1);
pageFullyUnmapped = true;
}
if (unmapPages || cache != area->cache) {
addr_t address = area->Base()
+ ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset);
uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress(
fPagingStructures->VirtualPML4(), address, fIsKernelMap,
false, NULL, fPageMapper, fMapCount);
if (entry == NULL) {
panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
"has no page table", page, area, address);
continue;
}
uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(entry);
if ((oldEntry & X86_64_PTE_PRESENT) == 0) {
panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
"has no page table entry", page, area, address);
continue;
}
// transfer the accessed/dirty flags to the page and invalidate
// the mapping, if necessary
if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
page->accessed = true;
if (!deletingAddressSpace)
InvalidatePage(address);
}
if ((oldEntry & X86_64_PTE_DIRTY) != 0)
page->modified = true;
if (pageFullyUnmapped) {
DEBUG_PAGE_ACCESS_START(page);
if (cache->temporary)
vm_page_set_state(page, PAGE_STATE_INACTIVE);
else if (page->modified)
vm_page_set_state(page, PAGE_STATE_MODIFIED);
else
vm_page_set_state(page, PAGE_STATE_CACHED);
DEBUG_PAGE_ACCESS_END(page);
}
}
fMapCount--;
}
Flush();
// flush explicitely, since we directly use the lock
locker.Unlock();
bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = mappings.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
}
X86VMTranslationMap64Bit::~X86VMTranslationMap64Bit()
{
TRACE("X86VMTranslationMap64Bit::~X86VMTranslationMap64Bit()\n");
if (fPagingStructures == NULL)
return;
if (fPageMapper != NULL) {
phys_addr_t address;
vm_page* page;
// Free all structures in the bottom half of the PML4 (user memory).
uint64* virtualPML4 = fPagingStructures->VirtualPML4();
for (uint32 i = 0; i < 256; i++) {
if ((virtualPML4[i] & X86_64_PML4E_PRESENT) == 0)
continue;
uint64* virtualPDPT = (uint64*)fPageMapper->GetPageTableAt(
virtualPML4[i] & X86_64_PML4E_ADDRESS_MASK);
for (uint32 j = 0; j < 512; j++) {
if ((virtualPDPT[j] & X86_64_PDPTE_PRESENT) == 0)
continue;
uint64* virtualPageDir = (uint64*)fPageMapper->GetPageTableAt(
virtualPDPT[j] & X86_64_PDPTE_ADDRESS_MASK);
for (uint32 k = 0; k < 512; k++) {
if ((virtualPageDir[k] & X86_64_PDE_PRESENT) == 0)
continue;
address = virtualPageDir[k] & X86_64_PDE_ADDRESS_MASK;
page = vm_lookup_page(address / B_PAGE_SIZE);
if (page == NULL) {
panic("page table %u %u %u on invalid page %#"
B_PRIxPHYSADDR "\n", i, j, k, address);
}
DEBUG_PAGE_ACCESS_START(page);
vm_page_set_state(page, PAGE_STATE_FREE);
}
address = virtualPDPT[j] & X86_64_PDPTE_ADDRESS_MASK;
page = vm_lookup_page(address / B_PAGE_SIZE);
if (page == NULL) {
panic("page directory %u %u on invalid page %#"
B_PRIxPHYSADDR "\n", i, j, address);
}
DEBUG_PAGE_ACCESS_START(page);
vm_page_set_state(page, PAGE_STATE_FREE);
}
address = virtualPML4[i] & X86_64_PML4E_ADDRESS_MASK;
page = vm_lookup_page(address / B_PAGE_SIZE);
if (page == NULL) {
panic("PDPT %u on invalid page %#" B_PRIxPHYSADDR "\n", i,
address);
}
DEBUG_PAGE_ACCESS_START(page);
vm_page_set_state(page, PAGE_STATE_FREE);
}
fPageMapper->Delete();
}
fPagingStructures->RemoveReference();
}
void
X86VMTranslationMap64Bit::UnmapPages(VMArea* area, addr_t base, size_t size,
bool updatePageQueue)
{
if (size == 0)
return;
addr_t start = base;
addr_t end = base + size - 1;
TRACE("X86VMTranslationMap64Bit::UnmapPages(%p, %#" B_PRIxADDR ", %#"
B_PRIxADDR ")\n", area, start, end);
VMAreaMappings queue;
RecursiveLocker locker(fLock);
ThreadCPUPinner pinner(thread_get_current_thread());
do {
uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress(
fPagingStructures->VirtualPML4(), start, fIsKernelMap, false,
NULL, fPageMapper, fMapCount);
if (pageTable == NULL) {
// Move on to the next page table.
start = ROUNDUP(start + 1, k64BitPageTableRange);
continue;
}
for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount;
index < k64BitTableEntryCount && start < end;
index++, start += B_PAGE_SIZE) {
uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(
&pageTable[index]);
if ((oldEntry & X86_64_PTE_PRESENT) == 0)
continue;
fMapCount--;
if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
// Note, that we only need to invalidate the address, if the
// accessed flags was set, since only then the entry could have
// been in any TLB.
InvalidatePage(start);
}
if (area->cache_type != CACHE_TYPE_DEVICE) {
// get the page
vm_page* page = vm_lookup_page(
(oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE);
ASSERT(page != NULL);
DEBUG_PAGE_ACCESS_START(page);
// transfer the accessed/dirty flags to the page
if ((oldEntry & X86_64_PTE_ACCESSED) != 0)
page->accessed = true;
if ((oldEntry & X86_64_PTE_DIRTY) != 0)
page->modified = true;
// remove the mapping object/decrement the wired_count of the
// page
if (area->wiring == B_NO_LOCK) {
vm_page_mapping* mapping = NULL;
vm_page_mappings::Iterator iterator
= page->mappings.GetIterator();
while ((mapping = iterator.Next()) != NULL) {
if (mapping->area == area)
break;
}
ASSERT(mapping != NULL);
area->mappings.Remove(mapping);
page->mappings.Remove(mapping);
queue.Add(mapping);
} else
page->DecrementWiredCount();
if (!page->IsMapped()) {
atomic_add(&gMappedPagesCount, -1);
if (updatePageQueue) {
if (page->Cache()->temporary)
vm_page_set_state(page, PAGE_STATE_INACTIVE);
else if (page->modified)
vm_page_set_state(page, PAGE_STATE_MODIFIED);
else
vm_page_set_state(page, PAGE_STATE_CACHED);
}
}
DEBUG_PAGE_ACCESS_END(page);
}
}
Flush();
// flush explicitly, since we directly use the lock
} while (start != 0 && start < end);
// TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not
// really critical here, as in all cases this method is used, the unmapped
// area range is unmapped for good (resized/cut) and the pages will likely
// be freed.
locker.Unlock();
// free removed mappings
bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = queue.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
}
M68KVMTranslationMap040::~M68KVMTranslationMap040()
{
if (fPagingStructures == NULL)
return;
if (fPageMapper != NULL)
fPageMapper->Delete();
if (fPagingStructures->pgroot_virt != NULL) {
page_root_entry *pgroot_virt = fPagingStructures->pgroot_virt;
// cycle through and free all of the user space pgdirs & pgtables
// since the size of tables don't match B_PAGE_SIZE,
// we alloc several at once, based on modulos,
// we make sure they are either all in the tree or none.
for (uint32 i = VADDR_TO_PRENT(USER_BASE);
i <= VADDR_TO_PRENT(USER_BASE + (USER_SIZE - 1)); i++) {
addr_t pgdir_pn;
page_directory_entry *pgdir;
vm_page *dirpage;
if (PRE_TYPE(pgroot_virt[i]) == DT_INVALID)
continue;
if (PRE_TYPE(pgroot_virt[i]) != DT_ROOT) {
panic("rtdir[%ld]: buggy descriptor type", i);
return;
}
// XXX:suboptimal (done 8 times)
pgdir_pn = PRE_TO_PN(pgroot_virt[i]);
dirpage = vm_lookup_page(pgdir_pn);
pgdir = &(((page_directory_entry *)dirpage)[i%NUM_DIRTBL_PER_PAGE]);
for (uint32 j = 0; j <= NUM_DIRENT_PER_TBL;
j+=NUM_PAGETBL_PER_PAGE) {
addr_t pgtbl_pn;
page_table_entry *pgtbl;
vm_page *page;
if (PDE_TYPE(pgdir[j]) == DT_INVALID)
continue;
if (PDE_TYPE(pgdir[j]) != DT_DIR) {
panic("pgroot[%ld][%ld]: buggy descriptor type", i, j);
return;
}
pgtbl_pn = PDE_TO_PN(pgdir[j]);
page = vm_lookup_page(pgtbl_pn);
pgtbl = (page_table_entry *)page;
if (!page) {
panic("destroy_tmap: didn't find pgtable page\n");
return;
}
DEBUG_PAGE_ACCESS_START(page);
vm_page_set_state(page, PAGE_STATE_FREE);
}
if (((i + 1) % NUM_DIRTBL_PER_PAGE) == 0) {
DEBUG_PAGE_ACCESS_END(dirpage);
vm_page_set_state(dirpage, PAGE_STATE_FREE);
}
}
#if 0
//X86
for (uint32 i = VADDR_TO_PDENT(USER_BASE);
i <= VADDR_TO_PDENT(USER_BASE + (USER_SIZE - 1)); i++) {
if ((fPagingStructures->pgdir_virt[i] & M68K_PDE_PRESENT) != 0) {
addr_t address = fPagingStructures->pgdir_virt[i]
& M68K_PDE_ADDRESS_MASK;
vm_page* page = vm_lookup_page(address / B_PAGE_SIZE);
if (!page)
panic("destroy_tmap: didn't find pgtable page\n");
DEBUG_PAGE_ACCESS_START(page);
vm_page_set_state(page, PAGE_STATE_FREE);
}
}
#endif
}
fPagingStructures->RemoveReference();
}
static status_t
cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer,
size_t* _size, bool doWrite)
{
if (_cacheRef == NULL)
panic("cache_io() called with NULL ref!\n");
file_cache_ref* ref = (file_cache_ref*)_cacheRef;
VMCache* cache = ref->cache;
off_t fileSize = cache->virtual_end;
bool useBuffer = buffer != 0;
TRACE(("cache_io(ref = %p, offset = %Ld, buffer = %p, size = %lu, %s)\n",
ref, offset, (void*)buffer, *_size, doWrite ? "write" : "read"));
// out of bounds access?
if (offset >= fileSize || offset < 0) {
*_size = 0;
return B_OK;
}
int32 pageOffset = offset & (B_PAGE_SIZE - 1);
size_t size = *_size;
offset -= pageOffset;
if ((off_t)(offset + pageOffset + size) > fileSize) {
// adapt size to be within the file's offsets
size = fileSize - pageOffset - offset;
*_size = size;
}
if (size == 0)
return B_OK;
// "offset" and "lastOffset" are always aligned to B_PAGE_SIZE,
// the "last*" variables always point to the end of the last
// satisfied request part
const uint32 kMaxChunkSize = MAX_IO_VECS * B_PAGE_SIZE;
size_t bytesLeft = size, lastLeft = size;
int32 lastPageOffset = pageOffset;
addr_t lastBuffer = buffer;
off_t lastOffset = offset;
size_t lastReservedPages = min_c(MAX_IO_VECS, (pageOffset + bytesLeft
+ B_PAGE_SIZE - 1) >> PAGE_SHIFT);
size_t reservePages = 0;
size_t pagesProcessed = 0;
cache_func function = NULL;
vm_page_reservation reservation;
reserve_pages(ref, &reservation, lastReservedPages, doWrite);
AutoLocker<VMCache> locker(cache);
while (bytesLeft > 0) {
// Periodically reevaluate the low memory situation and select the
// read/write hook accordingly
if (pagesProcessed % 32 == 0) {
if (size >= BYPASS_IO_SIZE
&& low_resource_state(B_KERNEL_RESOURCE_PAGES)
!= B_NO_LOW_RESOURCE) {
// In low memory situations we bypass the cache beyond a
// certain I/O size.
function = doWrite ? write_to_file : read_from_file;
} else
function = doWrite ? write_to_cache : read_into_cache;
}
// check if this page is already in memory
vm_page* page = cache->LookupPage(offset);
if (page != NULL) {
// The page may be busy - since we need to unlock the cache sometime
// in the near future, we need to satisfy the request of the pages
// we didn't get yet (to make sure no one else interferes in the
// meantime).
status_t status = satisfy_cache_io(ref, cookie, function, offset,
buffer, useBuffer, pageOffset, bytesLeft, reservePages,
lastOffset, lastBuffer, lastPageOffset, lastLeft,
lastReservedPages, &reservation);
if (status != B_OK)
return status;
// Since satisfy_cache_io() unlocks the cache, we need to look up
// the page again.
page = cache->LookupPage(offset);
if (page != NULL && page->busy) {
cache->WaitForPageEvents(page, PAGE_EVENT_NOT_BUSY, true);
continue;
}
}
size_t bytesInPage = min_c(size_t(B_PAGE_SIZE - pageOffset), bytesLeft);
TRACE(("lookup page from offset %Ld: %p, size = %lu, pageOffset "
"= %lu\n", offset, page, bytesLeft, pageOffset));
if (page != NULL) {
if (doWrite || useBuffer) {
// Since the following user_mem{cpy,set}() might cause a page
// fault, which in turn might cause pages to be reserved, we
// need to unlock the cache temporarily to avoid a potential
// deadlock. To make sure that our page doesn't go away, we mark
// it busy for the time.
page->busy = true;
locker.Unlock();
// copy the contents of the page already in memory
phys_addr_t pageAddress
= (phys_addr_t)page->physical_page_number * B_PAGE_SIZE
+ pageOffset;
bool userBuffer = IS_USER_ADDRESS(buffer);
if (doWrite) {
if (useBuffer) {
vm_memcpy_to_physical(pageAddress, (void*)buffer,
bytesInPage, userBuffer);
} else {
vm_memset_physical(pageAddress, 0, bytesInPage);
}
} else if (useBuffer) {
vm_memcpy_from_physical((void*)buffer, pageAddress,
bytesInPage, userBuffer);
}
locker.Lock();
if (doWrite) {
DEBUG_PAGE_ACCESS_START(page);
page->modified = true;
if (page->State() != PAGE_STATE_MODIFIED)
vm_page_set_state(page, PAGE_STATE_MODIFIED);
DEBUG_PAGE_ACCESS_END(page);
}
cache->MarkPageUnbusy(page);
}
// If it is cached only, requeue the page, so the respective queue
// roughly remains LRU first sorted.
if (page->State() == PAGE_STATE_CACHED
|| page->State() == PAGE_STATE_MODIFIED) {
DEBUG_PAGE_ACCESS_START(page);
vm_page_requeue(page, true);
DEBUG_PAGE_ACCESS_END(page);
}
if (bytesLeft <= bytesInPage) {
// we've read the last page, so we're done!
locker.Unlock();
vm_page_unreserve_pages(&reservation);
return B_OK;
}
// prepare a potential gap request
lastBuffer = buffer + bytesInPage;
lastLeft = bytesLeft - bytesInPage;
lastOffset = offset + B_PAGE_SIZE;
lastPageOffset = 0;
}
if (bytesLeft <= bytesInPage)
break;
buffer += bytesInPage;
bytesLeft -= bytesInPage;
pageOffset = 0;
offset += B_PAGE_SIZE;
pagesProcessed++;
if (buffer - lastBuffer + lastPageOffset >= kMaxChunkSize) {
status_t status = satisfy_cache_io(ref, cookie, function, offset,
buffer, useBuffer, pageOffset, bytesLeft, reservePages,
lastOffset, lastBuffer, lastPageOffset, lastLeft,
lastReservedPages, &reservation);
if (status != B_OK)
return status;
}
}
// fill the last remaining bytes of the request (either write or read)
return function(ref, cookie, lastOffset, lastPageOffset, lastBuffer,
lastLeft, useBuffer, &reservation, 0);
}
