/*! Called by UnmapPage() after performing the architecture specific part. Looks up the page, updates its flags, removes the page-area mapping, and requeues the page, if necessary. */ void VMTranslationMap::PageUnmapped(VMArea* area, page_num_t pageNumber, bool accessed, bool modified, bool updatePageQueue) { if (area->cache_type == CACHE_TYPE_DEVICE) { recursive_lock_unlock(&fLock); return; } // get the page vm_page* page = vm_lookup_page(pageNumber); ASSERT_PRINT(page != NULL, "page number: %#" B_PRIxPHYSADDR ", accessed: %d, modified: %d", pageNumber, accessed, modified); // transfer the accessed/dirty flags to the page page->accessed |= accessed; page->modified |= modified; // remove the mapping object/decrement the wired_count of the page vm_page_mapping* mapping = NULL; if (area->wiring == B_NO_LOCK) { vm_page_mappings::Iterator iterator = page->mappings.GetIterator(); while ((mapping = iterator.Next()) != NULL) { if (mapping->area == area) { area->mappings.Remove(mapping); page->mappings.Remove(mapping); break; } } ASSERT_PRINT(mapping != NULL, "page: %p, page number: %#" B_PRIxPHYSADDR ", accessed: %d, modified: %d", page, pageNumber, accessed, modified); } else page->DecrementWiredCount(); recursive_lock_unlock(&fLock); if (!page->IsMapped()) { atomic_add(&gMappedPagesCount, -1); if (updatePageQueue) { if (page->Cache()->temporary) vm_page_set_state(page, PAGE_STATE_INACTIVE); else if (page->modified) vm_page_set_state(page, PAGE_STATE_MODIFIED); else vm_page_set_state(page, PAGE_STATE_CACHED); } } if (mapping != NULL) { bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); object_cache_free(gPageMappingsObjectCache, mapping, CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0)); } }
ARMVMTranslationMap32Bit::~ARMVMTranslationMap32Bit() { if (fPagingStructures == NULL) return; if (fPageMapper != NULL) fPageMapper->Delete(); if (fPagingStructures->pgdir_virt != NULL) { // cycle through and free all of the user space pgtables for (uint32 i = VADDR_TO_PDENT(USER_BASE); i <= VADDR_TO_PDENT(USER_BASE + (USER_SIZE - 1)); i++) { if ((fPagingStructures->pgdir_virt[i] & ARM_PDE_TYPE_MASK) != 0) { addr_t address = fPagingStructures->pgdir_virt[i] & ARM_PDE_ADDRESS_MASK; vm_page* page = vm_lookup_page(address / B_PAGE_SIZE); if (!page) panic("destroy_tmap: didn't find pgtable page\n"); DEBUG_PAGE_ACCESS_START(page); vm_page_set_state(page, PAGE_STATE_FREE); } } } fPagingStructures->RemoveReference(); }
static void reserve_pages(file_cache_ref* ref, vm_page_reservation* reservation, size_t reservePages, bool isWrite) { if (low_resource_state(B_KERNEL_RESOURCE_PAGES) != B_NO_LOW_RESOURCE) { VMCache* cache = ref->cache; cache->Lock(); if (cache->consumers.IsEmpty() && cache->areas == NULL && access_is_sequential(ref)) { // we are not mapped, and we're accessed sequentially if (isWrite) { // Just write some pages back, and actually wait until they // have been written back in order to relieve the page pressure // a bit. int32 index = ref->last_access_index; int32 previous = index - 1; if (previous < 0) previous = LAST_ACCESSES - 1; vm_page_write_modified_page_range(cache, ref->LastAccessPageOffset(previous, true), ref->LastAccessPageOffset(index, true)); } else { // free some pages from our cache // TODO: start with oldest uint32 left = reservePages; vm_page* page; for (VMCachePagesTree::Iterator it = cache->pages.GetIterator(); (page = it.Next()) != NULL && left > 0;) { if (page->State() == PAGE_STATE_CACHED && !page->busy) { DEBUG_PAGE_ACCESS_START(page); ASSERT(!page->IsMapped()); ASSERT(!page->modified); cache->RemovePage(page); vm_page_set_state(page, PAGE_STATE_FREE); left--; } } } } cache->Unlock(); } vm_page_reserve_pages(reservation, reservePages, VM_PRIORITY_USER); }
void PrecacheIO::IOFinished(status_t status, bool partialTransfer, generic_size_t bytesTransferred) { AutoLocker<VMCache> locker(fCache); // Make successfully loaded pages accessible again (partially // transferred pages are considered failed) phys_size_t pagesTransferred = (bytesTransferred + B_PAGE_SIZE - 1) / B_PAGE_SIZE; if (fOffset + (off_t)bytesTransferred > fCache->virtual_end) bytesTransferred = fCache->virtual_end - fOffset; for (uint32 i = 0; i < pagesTransferred; i++) { if (i == pagesTransferred - 1 && (bytesTransferred % B_PAGE_SIZE) != 0) { // clear partial page size_t bytesTouched = bytesTransferred % B_PAGE_SIZE; vm_memset_physical( ((phys_addr_t)fPages[i]->physical_page_number << PAGE_SHIFT) + bytesTouched, 0, B_PAGE_SIZE - bytesTouched); } DEBUG_PAGE_ACCESS_TRANSFER(fPages[i], fAllocatingThread); fCache->MarkPageUnbusy(fPages[i]); DEBUG_PAGE_ACCESS_END(fPages[i]); } // Free pages after failed I/O for (uint32 i = pagesTransferred; i < fPageCount; i++) { DEBUG_PAGE_ACCESS_TRANSFER(fPages[i], fAllocatingThread); fCache->NotifyPageEvents(fPages[i], PAGE_EVENT_NOT_BUSY); fCache->RemovePage(fPages[i]); vm_page_set_state(fPages[i], PAGE_STATE_FREE); } delete this; }
/*! Iteratively correct the reported capacity by trying to read from the device close to its end. */ static uint64 test_capacity(cd_driver_info *info) { static const size_t kMaxEntries = 4; const uint32 blockSize = info->block_size; const size_t kBufferSize = blockSize * 4; TRACE("test_capacity: read with buffer size %" B_PRIuSIZE ", block size %" B_PRIu32", capacity %llu\n", kBufferSize, blockSize, info->original_capacity); info->capacity = info->original_capacity; size_t numBlocks = B_PAGE_SIZE / blockSize; uint64 offset = info->original_capacity; if (offset <= numBlocks) return B_OK; offset -= numBlocks; scsi_ccb *request = info->scsi->alloc_ccb(info->scsi_device); if (request == NULL) return B_NO_MEMORY; // Allocate buffer physical_entry entries[4]; size_t numEntries = 0; vm_page_reservation reservation; vm_page_reserve_pages(&reservation, (kBufferSize - 1 + B_PAGE_SIZE) / B_PAGE_SIZE, VM_PRIORITY_SYSTEM); for (size_t left = kBufferSize; numEntries < kMaxEntries && left > 0; numEntries++) { size_t bytes = std::min(left, (size_t)B_PAGE_SIZE); vm_page* page = vm_page_allocate_page(&reservation, PAGE_STATE_WIRED | VM_PAGE_ALLOC_BUSY); entries[numEntries].address = page->physical_page_number * B_PAGE_SIZE; entries[numEntries].size = bytes;; left -= bytes; } vm_page_unreserve_pages(&reservation); // Read close to the end of the device to find out its real end // Only try 1 second before the end (= 75 blocks) while (offset > info->original_capacity - 75) { size_t bytesTransferred; status_t status = sSCSIPeripheral->read_write(info->scsi_periph_device, request, offset, numBlocks, entries, numEntries, false, &bytesTransferred); TRACE("test_capacity: read from offset %llu: %s\n", offset, strerror(status)); if (status == B_OK || (request->sense[0] & 0x7f) != 0x70) break; switch (request->sense[2]) { case SCSIS_KEY_MEDIUM_ERROR: case SCSIS_KEY_ILLEGAL_REQUEST: case SCSIS_KEY_VOLUME_OVERFLOW: { // find out the problematic sector uint32 errorBlock = (request->sense[3] << 24U) | (request->sense[4] << 16U) | (request->sense[5] << 8U) | request->sense[6]; if (errorBlock >= offset) info->capacity = errorBlock; break; } default: break; } if (numBlocks > offset) break; offset -= numBlocks; } info->scsi->free_ccb(request); for (size_t i = 0; i < numEntries; i++) { vm_page_set_state(vm_lookup_page(entries[i].address / B_PAGE_SIZE), PAGE_STATE_FREE); } if (info->capacity != info->original_capacity) { dprintf("scsi_cd: adjusted capacity from %llu to %llu blocks.\n", info->original_capacity, info->capacity); } return B_OK; }
void ARMVMTranslationMap32Bit::UnmapArea(VMArea* area, bool deletingAddressSpace, bool ignoreTopCachePageFlags) { if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { ARMVMTranslationMap32Bit::UnmapPages(area, area->Base(), area->Size(), true); return; } bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; page_directory_entry* pd = fPagingStructures->pgdir_virt; RecursiveLocker locker(fLock); VMAreaMappings mappings; mappings.MoveFrom(&area->mappings); for (VMAreaMappings::Iterator it = mappings.GetIterator(); vm_page_mapping* mapping = it.Next();) { vm_page* page = mapping->page; page->mappings.Remove(mapping); VMCache* cache = page->Cache(); bool pageFullyUnmapped = false; if (!page->IsMapped()) { atomic_add(&gMappedPagesCount, -1); pageFullyUnmapped = true; } if (unmapPages || cache != area->cache) { addr_t address = area->Base() + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); int index = VADDR_TO_PDENT(address); if ((pd[index] & ARM_PDE_TYPE_MASK) == 0) { panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " "has no page dir entry", page, area, address); continue; } ThreadCPUPinner pinner(thread_get_current_thread()); page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( pd[index] & ARM_PDE_ADDRESS_MASK); page_table_entry oldEntry = ARMPagingMethod32Bit::ClearPageTableEntry( &pt[VADDR_TO_PTENT(address)]); pinner.Unlock(); if ((oldEntry & ARM_PTE_TYPE_MASK) == 0) { panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " "has no page table entry", page, area, address); continue; } // transfer the accessed/dirty flags to the page and invalidate // the mapping, if necessary if (true /*(oldEntry & ARM_PTE_ACCESSED) != 0*/) { // XXX IRA page->accessed = true; if (!deletingAddressSpace) InvalidatePage(address); } if (true /*(oldEntry & ARM_PTE_DIRTY) != 0*/) page->modified = true; if (pageFullyUnmapped) { DEBUG_PAGE_ACCESS_START(page); if (cache->temporary) vm_page_set_state(page, PAGE_STATE_INACTIVE); else if (page->modified) vm_page_set_state(page, PAGE_STATE_MODIFIED); else vm_page_set_state(page, PAGE_STATE_CACHED); DEBUG_PAGE_ACCESS_END(page); } } fMapCount--; } Flush(); // flush explicitely, since we directly use the lock locker.Unlock(); bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = mappings.RemoveHead()) object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); }
void ARMVMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size, bool updatePageQueue) { if (size == 0) return; addr_t start = base; addr_t end = base + size - 1; TRACE("ARMVMTranslationMap32Bit::UnmapPages(%p, %#" B_PRIxADDR ", %#" B_PRIxADDR ")\n", area, start, end); page_directory_entry* pd = fPagingStructures->pgdir_virt; VMAreaMappings queue; RecursiveLocker locker(fLock); do { int index = VADDR_TO_PDENT(start); if ((pd[index] & ARM_PDE_TYPE_MASK) == 0) { // no page table here, move the start up to access the next page // table start = ROUNDUP(start + 1, kPageTableAlignment); continue; } Thread* thread = thread_get_current_thread(); ThreadCPUPinner pinner(thread); page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( pd[index] & ARM_PDE_ADDRESS_MASK); for (index = VADDR_TO_PTENT(start); (index < 1024) && (start < end); index++, start += B_PAGE_SIZE) { page_table_entry oldEntry = ARMPagingMethod32Bit::ClearPageTableEntry(&pt[index]); if ((oldEntry & ARM_PTE_TYPE_MASK) == 0) continue; fMapCount--; if (true /*(oldEntry & ARM_PTE_ACCESSED) != 0*/) { // XXX IRA // Note, that we only need to invalidate the address, if the // accessed flags was set, since only then the entry could have // been in any TLB. InvalidatePage(start); } if (area->cache_type != CACHE_TYPE_DEVICE) { // get the page vm_page* page = vm_lookup_page( (oldEntry & ARM_PTE_ADDRESS_MASK) / B_PAGE_SIZE); ASSERT(page != NULL); DEBUG_PAGE_ACCESS_START(page); // transfer the accessed/dirty flags to the page if (/*(oldEntry & ARM_PTE_ACCESSED) != 0*/ true) // XXX IRA page->accessed = true; if (/*(oldEntry & ARM_PTE_DIRTY) != 0 */ true) page->modified = true; // remove the mapping object/decrement the wired_count of the // page if (area->wiring == B_NO_LOCK) { vm_page_mapping* mapping = NULL; vm_page_mappings::Iterator iterator = page->mappings.GetIterator(); while ((mapping = iterator.Next()) != NULL) { if (mapping->area == area) break; } ASSERT(mapping != NULL); area->mappings.Remove(mapping); page->mappings.Remove(mapping); queue.Add(mapping); } else page->DecrementWiredCount(); if (!page->IsMapped()) { atomic_add(&gMappedPagesCount, -1); if (updatePageQueue) { if (page->Cache()->temporary) vm_page_set_state(page, PAGE_STATE_INACTIVE); else if (page->modified) vm_page_set_state(page, PAGE_STATE_MODIFIED); else vm_page_set_state(page, PAGE_STATE_CACHED); } } DEBUG_PAGE_ACCESS_END(page); } } Flush(); // flush explicitly, since we directly use the lock } while (start != 0 && start < end); // TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not // really critical here, as in all cases this method is used, the unmapped // area range is unmapped for good (resized/cut) and the pages will likely // be freed. locker.Unlock(); // free removed mappings bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = queue.RemoveHead()) object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); }
void X86VMTranslationMap64Bit::UnmapArea(VMArea* area, bool deletingAddressSpace, bool ignoreTopCachePageFlags) { TRACE("X86VMTranslationMap64Bit::UnmapArea(%p)\n", area); if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { X86VMTranslationMap64Bit::UnmapPages(area, area->Base(), area->Size(), true); return; } bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; RecursiveLocker locker(fLock); ThreadCPUPinner pinner(thread_get_current_thread()); VMAreaMappings mappings; mappings.MoveFrom(&area->mappings); for (VMAreaMappings::Iterator it = mappings.GetIterator(); vm_page_mapping* mapping = it.Next();) { vm_page* page = mapping->page; page->mappings.Remove(mapping); VMCache* cache = page->Cache(); bool pageFullyUnmapped = false; if (!page->IsMapped()) { atomic_add(&gMappedPagesCount, -1); pageFullyUnmapped = true; } if (unmapPages || cache != area->cache) { addr_t address = area->Base() + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress( fPagingStructures->VirtualPML4(), address, fIsKernelMap, false, NULL, fPageMapper, fMapCount); if (entry == NULL) { panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " "has no page table", page, area, address); continue; } uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(entry); if ((oldEntry & X86_64_PTE_PRESENT) == 0) { panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " "has no page table entry", page, area, address); continue; } // transfer the accessed/dirty flags to the page and invalidate // the mapping, if necessary if ((oldEntry & X86_64_PTE_ACCESSED) != 0) { page->accessed = true; if (!deletingAddressSpace) InvalidatePage(address); } if ((oldEntry & X86_64_PTE_DIRTY) != 0) page->modified = true; if (pageFullyUnmapped) { DEBUG_PAGE_ACCESS_START(page); if (cache->temporary) vm_page_set_state(page, PAGE_STATE_INACTIVE); else if (page->modified) vm_page_set_state(page, PAGE_STATE_MODIFIED); else vm_page_set_state(page, PAGE_STATE_CACHED); DEBUG_PAGE_ACCESS_END(page); } } fMapCount--; } Flush(); // flush explicitely, since we directly use the lock locker.Unlock(); bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = mappings.RemoveHead()) object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); }
X86VMTranslationMap64Bit::~X86VMTranslationMap64Bit() { TRACE("X86VMTranslationMap64Bit::~X86VMTranslationMap64Bit()\n"); if (fPagingStructures == NULL) return; if (fPageMapper != NULL) { phys_addr_t address; vm_page* page; // Free all structures in the bottom half of the PML4 (user memory). uint64* virtualPML4 = fPagingStructures->VirtualPML4(); for (uint32 i = 0; i < 256; i++) { if ((virtualPML4[i] & X86_64_PML4E_PRESENT) == 0) continue; uint64* virtualPDPT = (uint64*)fPageMapper->GetPageTableAt( virtualPML4[i] & X86_64_PML4E_ADDRESS_MASK); for (uint32 j = 0; j < 512; j++) { if ((virtualPDPT[j] & X86_64_PDPTE_PRESENT) == 0) continue; uint64* virtualPageDir = (uint64*)fPageMapper->GetPageTableAt( virtualPDPT[j] & X86_64_PDPTE_ADDRESS_MASK); for (uint32 k = 0; k < 512; k++) { if ((virtualPageDir[k] & X86_64_PDE_PRESENT) == 0) continue; address = virtualPageDir[k] & X86_64_PDE_ADDRESS_MASK; page = vm_lookup_page(address / B_PAGE_SIZE); if (page == NULL) { panic("page table %u %u %u on invalid page %#" B_PRIxPHYSADDR "\n", i, j, k, address); } DEBUG_PAGE_ACCESS_START(page); vm_page_set_state(page, PAGE_STATE_FREE); } address = virtualPDPT[j] & X86_64_PDPTE_ADDRESS_MASK; page = vm_lookup_page(address / B_PAGE_SIZE); if (page == NULL) { panic("page directory %u %u on invalid page %#" B_PRIxPHYSADDR "\n", i, j, address); } DEBUG_PAGE_ACCESS_START(page); vm_page_set_state(page, PAGE_STATE_FREE); } address = virtualPML4[i] & X86_64_PML4E_ADDRESS_MASK; page = vm_lookup_page(address / B_PAGE_SIZE); if (page == NULL) { panic("PDPT %u on invalid page %#" B_PRIxPHYSADDR "\n", i, address); } DEBUG_PAGE_ACCESS_START(page); vm_page_set_state(page, PAGE_STATE_FREE); } fPageMapper->Delete(); } fPagingStructures->RemoveReference(); }
void X86VMTranslationMap64Bit::UnmapPages(VMArea* area, addr_t base, size_t size, bool updatePageQueue) { if (size == 0) return; addr_t start = base; addr_t end = base + size - 1; TRACE("X86VMTranslationMap64Bit::UnmapPages(%p, %#" B_PRIxADDR ", %#" B_PRIxADDR ")\n", area, start, end); VMAreaMappings queue; RecursiveLocker locker(fLock); ThreadCPUPinner pinner(thread_get_current_thread()); do { uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress( fPagingStructures->VirtualPML4(), start, fIsKernelMap, false, NULL, fPageMapper, fMapCount); if (pageTable == NULL) { // Move on to the next page table. start = ROUNDUP(start + 1, k64BitPageTableRange); continue; } for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount; index < k64BitTableEntryCount && start < end; index++, start += B_PAGE_SIZE) { uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry( &pageTable[index]); if ((oldEntry & X86_64_PTE_PRESENT) == 0) continue; fMapCount--; if ((oldEntry & X86_64_PTE_ACCESSED) != 0) { // Note, that we only need to invalidate the address, if the // accessed flags was set, since only then the entry could have // been in any TLB. InvalidatePage(start); } if (area->cache_type != CACHE_TYPE_DEVICE) { // get the page vm_page* page = vm_lookup_page( (oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE); ASSERT(page != NULL); DEBUG_PAGE_ACCESS_START(page); // transfer the accessed/dirty flags to the page if ((oldEntry & X86_64_PTE_ACCESSED) != 0) page->accessed = true; if ((oldEntry & X86_64_PTE_DIRTY) != 0) page->modified = true; // remove the mapping object/decrement the wired_count of the // page if (area->wiring == B_NO_LOCK) { vm_page_mapping* mapping = NULL; vm_page_mappings::Iterator iterator = page->mappings.GetIterator(); while ((mapping = iterator.Next()) != NULL) { if (mapping->area == area) break; } ASSERT(mapping != NULL); area->mappings.Remove(mapping); page->mappings.Remove(mapping); queue.Add(mapping); } else page->DecrementWiredCount(); if (!page->IsMapped()) { atomic_add(&gMappedPagesCount, -1); if (updatePageQueue) { if (page->Cache()->temporary) vm_page_set_state(page, PAGE_STATE_INACTIVE); else if (page->modified) vm_page_set_state(page, PAGE_STATE_MODIFIED); else vm_page_set_state(page, PAGE_STATE_CACHED); } } DEBUG_PAGE_ACCESS_END(page); } } Flush(); // flush explicitly, since we directly use the lock } while (start != 0 && start < end); // TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not // really critical here, as in all cases this method is used, the unmapped // area range is unmapped for good (resized/cut) and the pages will likely // be freed. locker.Unlock(); // free removed mappings bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = queue.RemoveHead()) object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); }
M68KVMTranslationMap040::~M68KVMTranslationMap040() { if (fPagingStructures == NULL) return; if (fPageMapper != NULL) fPageMapper->Delete(); if (fPagingStructures->pgroot_virt != NULL) { page_root_entry *pgroot_virt = fPagingStructures->pgroot_virt; // cycle through and free all of the user space pgdirs & pgtables // since the size of tables don't match B_PAGE_SIZE, // we alloc several at once, based on modulos, // we make sure they are either all in the tree or none. for (uint32 i = VADDR_TO_PRENT(USER_BASE); i <= VADDR_TO_PRENT(USER_BASE + (USER_SIZE - 1)); i++) { addr_t pgdir_pn; page_directory_entry *pgdir; vm_page *dirpage; if (PRE_TYPE(pgroot_virt[i]) == DT_INVALID) continue; if (PRE_TYPE(pgroot_virt[i]) != DT_ROOT) { panic("rtdir[%ld]: buggy descriptor type", i); return; } // XXX:suboptimal (done 8 times) pgdir_pn = PRE_TO_PN(pgroot_virt[i]); dirpage = vm_lookup_page(pgdir_pn); pgdir = &(((page_directory_entry *)dirpage)[i%NUM_DIRTBL_PER_PAGE]); for (uint32 j = 0; j <= NUM_DIRENT_PER_TBL; j+=NUM_PAGETBL_PER_PAGE) { addr_t pgtbl_pn; page_table_entry *pgtbl; vm_page *page; if (PDE_TYPE(pgdir[j]) == DT_INVALID) continue; if (PDE_TYPE(pgdir[j]) != DT_DIR) { panic("pgroot[%ld][%ld]: buggy descriptor type", i, j); return; } pgtbl_pn = PDE_TO_PN(pgdir[j]); page = vm_lookup_page(pgtbl_pn); pgtbl = (page_table_entry *)page; if (!page) { panic("destroy_tmap: didn't find pgtable page\n"); return; } DEBUG_PAGE_ACCESS_START(page); vm_page_set_state(page, PAGE_STATE_FREE); } if (((i + 1) % NUM_DIRTBL_PER_PAGE) == 0) { DEBUG_PAGE_ACCESS_END(dirpage); vm_page_set_state(dirpage, PAGE_STATE_FREE); } } #if 0 //X86 for (uint32 i = VADDR_TO_PDENT(USER_BASE); i <= VADDR_TO_PDENT(USER_BASE + (USER_SIZE - 1)); i++) { if ((fPagingStructures->pgdir_virt[i] & M68K_PDE_PRESENT) != 0) { addr_t address = fPagingStructures->pgdir_virt[i] & M68K_PDE_ADDRESS_MASK; vm_page* page = vm_lookup_page(address / B_PAGE_SIZE); if (!page) panic("destroy_tmap: didn't find pgtable page\n"); DEBUG_PAGE_ACCESS_START(page); vm_page_set_state(page, PAGE_STATE_FREE); } } #endif } fPagingStructures->RemoveReference(); }
static status_t cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, size_t* _size, bool doWrite) { if (_cacheRef == NULL) panic("cache_io() called with NULL ref!\n"); file_cache_ref* ref = (file_cache_ref*)_cacheRef; VMCache* cache = ref->cache; off_t fileSize = cache->virtual_end; bool useBuffer = buffer != 0; TRACE(("cache_io(ref = %p, offset = %Ld, buffer = %p, size = %lu, %s)\n", ref, offset, (void*)buffer, *_size, doWrite ? "write" : "read")); // out of bounds access? if (offset >= fileSize || offset < 0) { *_size = 0; return B_OK; } int32 pageOffset = offset & (B_PAGE_SIZE - 1); size_t size = *_size; offset -= pageOffset; if ((off_t)(offset + pageOffset + size) > fileSize) { // adapt size to be within the file's offsets size = fileSize - pageOffset - offset; *_size = size; } if (size == 0) return B_OK; // "offset" and "lastOffset" are always aligned to B_PAGE_SIZE, // the "last*" variables always point to the end of the last // satisfied request part const uint32 kMaxChunkSize = MAX_IO_VECS * B_PAGE_SIZE; size_t bytesLeft = size, lastLeft = size; int32 lastPageOffset = pageOffset; addr_t lastBuffer = buffer; off_t lastOffset = offset; size_t lastReservedPages = min_c(MAX_IO_VECS, (pageOffset + bytesLeft + B_PAGE_SIZE - 1) >> PAGE_SHIFT); size_t reservePages = 0; size_t pagesProcessed = 0; cache_func function = NULL; vm_page_reservation reservation; reserve_pages(ref, &reservation, lastReservedPages, doWrite); AutoLocker<VMCache> locker(cache); while (bytesLeft > 0) { // Periodically reevaluate the low memory situation and select the // read/write hook accordingly if (pagesProcessed % 32 == 0) { if (size >= BYPASS_IO_SIZE && low_resource_state(B_KERNEL_RESOURCE_PAGES) != B_NO_LOW_RESOURCE) { // In low memory situations we bypass the cache beyond a // certain I/O size. function = doWrite ? write_to_file : read_from_file; } else function = doWrite ? write_to_cache : read_into_cache; } // check if this page is already in memory vm_page* page = cache->LookupPage(offset); if (page != NULL) { // The page may be busy - since we need to unlock the cache sometime // in the near future, we need to satisfy the request of the pages // we didn't get yet (to make sure no one else interferes in the // meantime). status_t status = satisfy_cache_io(ref, cookie, function, offset, buffer, useBuffer, pageOffset, bytesLeft, reservePages, lastOffset, lastBuffer, lastPageOffset, lastLeft, lastReservedPages, &reservation); if (status != B_OK) return status; // Since satisfy_cache_io() unlocks the cache, we need to look up // the page again. page = cache->LookupPage(offset); if (page != NULL && page->busy) { cache->WaitForPageEvents(page, PAGE_EVENT_NOT_BUSY, true); continue; } } size_t bytesInPage = min_c(size_t(B_PAGE_SIZE - pageOffset), bytesLeft); TRACE(("lookup page from offset %Ld: %p, size = %lu, pageOffset " "= %lu\n", offset, page, bytesLeft, pageOffset)); if (page != NULL) { if (doWrite || useBuffer) { // Since the following user_mem{cpy,set}() might cause a page // fault, which in turn might cause pages to be reserved, we // need to unlock the cache temporarily to avoid a potential // deadlock. To make sure that our page doesn't go away, we mark // it busy for the time. page->busy = true; locker.Unlock(); // copy the contents of the page already in memory phys_addr_t pageAddress = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE + pageOffset; bool userBuffer = IS_USER_ADDRESS(buffer); if (doWrite) { if (useBuffer) { vm_memcpy_to_physical(pageAddress, (void*)buffer, bytesInPage, userBuffer); } else { vm_memset_physical(pageAddress, 0, bytesInPage); } } else if (useBuffer) { vm_memcpy_from_physical((void*)buffer, pageAddress, bytesInPage, userBuffer); } locker.Lock(); if (doWrite) { DEBUG_PAGE_ACCESS_START(page); page->modified = true; if (page->State() != PAGE_STATE_MODIFIED) vm_page_set_state(page, PAGE_STATE_MODIFIED); DEBUG_PAGE_ACCESS_END(page); } cache->MarkPageUnbusy(page); } // If it is cached only, requeue the page, so the respective queue // roughly remains LRU first sorted. if (page->State() == PAGE_STATE_CACHED || page->State() == PAGE_STATE_MODIFIED) { DEBUG_PAGE_ACCESS_START(page); vm_page_requeue(page, true); DEBUG_PAGE_ACCESS_END(page); } if (bytesLeft <= bytesInPage) { // we've read the last page, so we're done! locker.Unlock(); vm_page_unreserve_pages(&reservation); return B_OK; } // prepare a potential gap request lastBuffer = buffer + bytesInPage; lastLeft = bytesLeft - bytesInPage; lastOffset = offset + B_PAGE_SIZE; lastPageOffset = 0; } if (bytesLeft <= bytesInPage) break; buffer += bytesInPage; bytesLeft -= bytesInPage; pageOffset = 0; offset += B_PAGE_SIZE; pagesProcessed++; if (buffer - lastBuffer + lastPageOffset >= kMaxChunkSize) { status_t status = satisfy_cache_io(ref, cookie, function, offset, buffer, useBuffer, pageOffset, bytesLeft, reservePages, lastOffset, lastBuffer, lastPageOffset, lastLeft, lastReservedPages, &reservation); if (status != B_OK) return status; } } // fill the last remaining bytes of the request (either write or read) return function(ref, cookie, lastOffset, lastPageOffset, lastBuffer, lastLeft, useBuffer, &reservation, 0); }
/*! Reads the requested amount of data into the cache, and allocates pages needed to fulfill that request. This function is called by cache_io(). It can only handle a certain amount of bytes, and the caller must make sure that it matches that criterion. The cache_ref lock must be held when calling this function; during operation it will unlock the cache, though. */ static status_t read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, int32 pageOffset, addr_t buffer, size_t bufferSize, bool useBuffer, vm_page_reservation* reservation, size_t reservePages) { TRACE(("read_into_cache(offset = %Ld, pageOffset = %ld, buffer = %#lx, " "bufferSize = %lu\n", offset, pageOffset, buffer, bufferSize)); VMCache* cache = ref->cache; // TODO: We're using way too much stack! Rather allocate a sufficiently // large chunk on the heap. generic_io_vec vecs[MAX_IO_VECS]; uint32 vecCount = 0; generic_size_t numBytes = PAGE_ALIGN(pageOffset + bufferSize); vm_page* pages[MAX_IO_VECS]; int32 pageIndex = 0; // allocate pages for the cache and mark them busy for (generic_size_t pos = 0; pos < numBytes; pos += B_PAGE_SIZE) { vm_page* page = pages[pageIndex++] = vm_page_allocate_page( reservation, PAGE_STATE_CACHED | VM_PAGE_ALLOC_BUSY); cache->InsertPage(page, offset + pos); add_to_iovec(vecs, vecCount, MAX_IO_VECS, page->physical_page_number * B_PAGE_SIZE, B_PAGE_SIZE); // TODO: check if the array is large enough (currently panics)! } push_access(ref, offset, bufferSize, false); cache->Unlock(); vm_page_unreserve_pages(reservation); // read file into reserved pages status_t status = read_pages_and_clear_partial(ref, cookie, offset, vecs, vecCount, B_PHYSICAL_IO_REQUEST, &numBytes); if (status != B_OK) { // reading failed, free allocated pages dprintf("file_cache: read pages failed: %s\n", strerror(status)); cache->Lock(); for (int32 i = 0; i < pageIndex; i++) { cache->NotifyPageEvents(pages[i], PAGE_EVENT_NOT_BUSY); cache->RemovePage(pages[i]); vm_page_set_state(pages[i], PAGE_STATE_FREE); } return status; } // copy the pages if needed and unmap them again for (int32 i = 0; i < pageIndex; i++) { if (useBuffer && bufferSize != 0) { size_t bytes = min_c(bufferSize, (size_t)B_PAGE_SIZE - pageOffset); vm_memcpy_from_physical((void*)buffer, pages[i]->physical_page_number * B_PAGE_SIZE + pageOffset, bytes, IS_USER_ADDRESS(buffer)); buffer += bytes; bufferSize -= bytes; pageOffset = 0; } } reserve_pages(ref, reservation, reservePages, false); cache->Lock(); // make the pages accessible in the cache for (int32 i = pageIndex; i-- > 0;) { DEBUG_PAGE_ACCESS_END(pages[i]); cache->MarkPageUnbusy(pages[i]); } return B_OK; }