/*! Called by UnmapPage() after performing the architecture specific part.
Looks up the page, updates its flags, removes the page-area mapping, and
requeues the page, if necessary.
*/
void
VMTranslationMap::PageUnmapped(VMArea* area, page_num_t pageNumber,
bool accessed, bool modified, bool updatePageQueue)
{
if (area->cache_type == CACHE_TYPE_DEVICE) {
recursive_lock_unlock(&fLock);
return;
}
// get the page
vm_page* page = vm_lookup_page(pageNumber);
ASSERT_PRINT(page != NULL, "page number: %#" B_PRIxPHYSADDR
", accessed: %d, modified: %d", pageNumber, accessed, modified);
// transfer the accessed/dirty flags to the page
page->accessed |= accessed;
page->modified |= modified;
// remove the mapping object/decrement the wired_count of the page
vm_page_mapping* mapping = NULL;
if (area->wiring == B_NO_LOCK) {
vm_page_mappings::Iterator iterator = page->mappings.GetIterator();
while ((mapping = iterator.Next()) != NULL) {
if (mapping->area == area) {
area->mappings.Remove(mapping);
page->mappings.Remove(mapping);
break;
}
}
ASSERT_PRINT(mapping != NULL, "page: %p, page number: %#"
B_PRIxPHYSADDR ", accessed: %d, modified: %d", page,
pageNumber, accessed, modified);
} else
page->DecrementWiredCount();
recursive_lock_unlock(&fLock);
if (!page->IsMapped()) {
atomic_add(&gMappedPagesCount, -1);
if (updatePageQueue) {
if (page->Cache()->temporary)
vm_page_set_state(page, PAGE_STATE_INACTIVE);
else if (page->modified)
vm_page_set_state(page, PAGE_STATE_MODIFIED);
else
vm_page_set_state(page, PAGE_STATE_CACHED);
}
}
if (mapping != NULL) {
bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
object_cache_free(gPageMappingsObjectCache, mapping,
CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0));
}
}
ARMVMTranslationMap32Bit::~ARMVMTranslationMap32Bit()
{
if (fPagingStructures == NULL)
return;
if (fPageMapper != NULL)
fPageMapper->Delete();
if (fPagingStructures->pgdir_virt != NULL) {
// cycle through and free all of the user space pgtables
for (uint32 i = VADDR_TO_PDENT(USER_BASE);
i <= VADDR_TO_PDENT(USER_BASE + (USER_SIZE - 1)); i++) {
if ((fPagingStructures->pgdir_virt[i] & ARM_PDE_TYPE_MASK) != 0) {
addr_t address = fPagingStructures->pgdir_virt[i]
& ARM_PDE_ADDRESS_MASK;
vm_page* page = vm_lookup_page(address / B_PAGE_SIZE);
if (!page)
panic("destroy_tmap: didn't find pgtable page\n");
DEBUG_PAGE_ACCESS_START(page);
vm_page_set_state(page, PAGE_STATE_FREE);
}
}
}
fPagingStructures->RemoveReference();
}
static void
reserve_pages(file_cache_ref* ref, vm_page_reservation* reservation,
size_t reservePages, bool isWrite)
{
if (low_resource_state(B_KERNEL_RESOURCE_PAGES) != B_NO_LOW_RESOURCE) {
VMCache* cache = ref->cache;
cache->Lock();
if (cache->consumers.IsEmpty() && cache->areas == NULL
&& access_is_sequential(ref)) {
// we are not mapped, and we're accessed sequentially
if (isWrite) {
// Just write some pages back, and actually wait until they
// have been written back in order to relieve the page pressure
// a bit.
int32 index = ref->last_access_index;
int32 previous = index - 1;
if (previous < 0)
previous = LAST_ACCESSES - 1;
vm_page_write_modified_page_range(cache,
ref->LastAccessPageOffset(previous, true),
ref->LastAccessPageOffset(index, true));
} else {
// free some pages from our cache
// TODO: start with oldest
uint32 left = reservePages;
vm_page* page;
for (VMCachePagesTree::Iterator it = cache->pages.GetIterator();
(page = it.Next()) != NULL && left > 0;) {
if (page->State() == PAGE_STATE_CACHED && !page->busy) {
DEBUG_PAGE_ACCESS_START(page);
ASSERT(!page->IsMapped());
ASSERT(!page->modified);
cache->RemovePage(page);
vm_page_set_state(page, PAGE_STATE_FREE);
left--;
}
}
}
}
cache->Unlock();
}
vm_page_reserve_pages(reservation, reservePages, VM_PRIORITY_USER);
}
void
PrecacheIO::IOFinished(status_t status, bool partialTransfer,
generic_size_t bytesTransferred)
{
AutoLocker<VMCache> locker(fCache);
// Make successfully loaded pages accessible again (partially
// transferred pages are considered failed)
phys_size_t pagesTransferred
= (bytesTransferred + B_PAGE_SIZE - 1) / B_PAGE_SIZE;
if (fOffset + (off_t)bytesTransferred > fCache->virtual_end)
bytesTransferred = fCache->virtual_end - fOffset;
for (uint32 i = 0; i < pagesTransferred; i++) {
if (i == pagesTransferred - 1
&& (bytesTransferred % B_PAGE_SIZE) != 0) {
// clear partial page
size_t bytesTouched = bytesTransferred % B_PAGE_SIZE;
vm_memset_physical(
((phys_addr_t)fPages[i]->physical_page_number << PAGE_SHIFT)
+ bytesTouched,
0, B_PAGE_SIZE - bytesTouched);
}
DEBUG_PAGE_ACCESS_TRANSFER(fPages[i], fAllocatingThread);
fCache->MarkPageUnbusy(fPages[i]);
DEBUG_PAGE_ACCESS_END(fPages[i]);
}
// Free pages after failed I/O
for (uint32 i = pagesTransferred; i < fPageCount; i++) {
DEBUG_PAGE_ACCESS_TRANSFER(fPages[i], fAllocatingThread);
fCache->NotifyPageEvents(fPages[i], PAGE_EVENT_NOT_BUSY);
fCache->RemovePage(fPages[i]);
vm_page_set_state(fPages[i], PAGE_STATE_FREE);
}
delete this;
}
/*! Iteratively correct the reported capacity by trying to read from the device
close to its end.
*/
static uint64
test_capacity(cd_driver_info *info)
{
static const size_t kMaxEntries = 4;
const uint32 blockSize = info->block_size;
const size_t kBufferSize = blockSize * 4;
TRACE("test_capacity: read with buffer size %" B_PRIuSIZE ", block size %"
B_PRIu32", capacity %llu\n", kBufferSize, blockSize,
info->original_capacity);
info->capacity = info->original_capacity;
size_t numBlocks = B_PAGE_SIZE / blockSize;
uint64 offset = info->original_capacity;
if (offset <= numBlocks)
return B_OK;
offset -= numBlocks;
scsi_ccb *request = info->scsi->alloc_ccb(info->scsi_device);
if (request == NULL)
return B_NO_MEMORY;
// Allocate buffer
physical_entry entries[4];
size_t numEntries = 0;
vm_page_reservation reservation;
vm_page_reserve_pages(&reservation,
(kBufferSize - 1 + B_PAGE_SIZE) / B_PAGE_SIZE, VM_PRIORITY_SYSTEM);
for (size_t left = kBufferSize; numEntries < kMaxEntries && left > 0;
numEntries++) {
size_t bytes = std::min(left, (size_t)B_PAGE_SIZE);
vm_page* page = vm_page_allocate_page(&reservation,
PAGE_STATE_WIRED | VM_PAGE_ALLOC_BUSY);
entries[numEntries].address = page->physical_page_number * B_PAGE_SIZE;
entries[numEntries].size = bytes;;
left -= bytes;
}
vm_page_unreserve_pages(&reservation);
// Read close to the end of the device to find out its real end
// Only try 1 second before the end (= 75 blocks)
while (offset > info->original_capacity - 75) {
size_t bytesTransferred;
status_t status = sSCSIPeripheral->read_write(info->scsi_periph_device,
request, offset, numBlocks, entries, numEntries, false,
&bytesTransferred);
TRACE("test_capacity: read from offset %llu: %s\n", offset,
strerror(status));
if (status == B_OK || (request->sense[0] & 0x7f) != 0x70)
break;
switch (request->sense[2]) {
case SCSIS_KEY_MEDIUM_ERROR:
case SCSIS_KEY_ILLEGAL_REQUEST:
case SCSIS_KEY_VOLUME_OVERFLOW:
{
// find out the problematic sector
uint32 errorBlock = (request->sense[3] << 24U)
| (request->sense[4] << 16U) | (request->sense[5] << 8U)
| request->sense[6];
if (errorBlock >= offset)
info->capacity = errorBlock;
break;
}
default:
break;
}
if (numBlocks > offset)
break;
offset -= numBlocks;
}
info->scsi->free_ccb(request);
for (size_t i = 0; i < numEntries; i++) {
vm_page_set_state(vm_lookup_page(entries[i].address / B_PAGE_SIZE),
PAGE_STATE_FREE);
}
if (info->capacity != info->original_capacity) {
dprintf("scsi_cd: adjusted capacity from %llu to %llu blocks.\n",
info->original_capacity, info->capacity);
}
return B_OK;
}
void
ARMVMTranslationMap32Bit::UnmapArea(VMArea* area, bool deletingAddressSpace,
bool ignoreTopCachePageFlags)
{
if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) {
ARMVMTranslationMap32Bit::UnmapPages(area, area->Base(), area->Size(),
true);
return;
}
bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags;
page_directory_entry* pd = fPagingStructures->pgdir_virt;
RecursiveLocker locker(fLock);
VMAreaMappings mappings;
mappings.MoveFrom(&area->mappings);
for (VMAreaMappings::Iterator it = mappings.GetIterator();
vm_page_mapping* mapping = it.Next();) {
vm_page* page = mapping->page;
page->mappings.Remove(mapping);
VMCache* cache = page->Cache();
bool pageFullyUnmapped = false;
if (!page->IsMapped()) {
atomic_add(&gMappedPagesCount, -1);
pageFullyUnmapped = true;
}
if (unmapPages || cache != area->cache) {
addr_t address = area->Base()
+ ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset);
int index = VADDR_TO_PDENT(address);
if ((pd[index] & ARM_PDE_TYPE_MASK) == 0) {
panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
"has no page dir entry", page, area, address);
continue;
}
ThreadCPUPinner pinner(thread_get_current_thread());
page_table_entry* pt
= (page_table_entry*)fPageMapper->GetPageTableAt(
pd[index] & ARM_PDE_ADDRESS_MASK);
page_table_entry oldEntry
= ARMPagingMethod32Bit::ClearPageTableEntry(
&pt[VADDR_TO_PTENT(address)]);
pinner.Unlock();
if ((oldEntry & ARM_PTE_TYPE_MASK) == 0) {
panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
"has no page table entry", page, area, address);
continue;
}
// transfer the accessed/dirty flags to the page and invalidate
// the mapping, if necessary
if (true /*(oldEntry & ARM_PTE_ACCESSED) != 0*/) { // XXX IRA
page->accessed = true;
if (!deletingAddressSpace)
InvalidatePage(address);
}
if (true /*(oldEntry & ARM_PTE_DIRTY) != 0*/)
page->modified = true;
if (pageFullyUnmapped) {
DEBUG_PAGE_ACCESS_START(page);
if (cache->temporary)
vm_page_set_state(page, PAGE_STATE_INACTIVE);
else if (page->modified)
vm_page_set_state(page, PAGE_STATE_MODIFIED);
else
vm_page_set_state(page, PAGE_STATE_CACHED);
DEBUG_PAGE_ACCESS_END(page);
}
}
fMapCount--;
}
Flush();
// flush explicitely, since we directly use the lock
locker.Unlock();
bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = mappings.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
}
void
ARMVMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size,
bool updatePageQueue)
{
if (size == 0)
return;
addr_t start = base;
addr_t end = base + size - 1;
TRACE("ARMVMTranslationMap32Bit::UnmapPages(%p, %#" B_PRIxADDR ", %#"
B_PRIxADDR ")\n", area, start, end);
page_directory_entry* pd = fPagingStructures->pgdir_virt;
VMAreaMappings queue;
RecursiveLocker locker(fLock);
do {
int index = VADDR_TO_PDENT(start);
if ((pd[index] & ARM_PDE_TYPE_MASK) == 0) {
// no page table here, move the start up to access the next page
// table
start = ROUNDUP(start + 1, kPageTableAlignment);
continue;
}
Thread* thread = thread_get_current_thread();
ThreadCPUPinner pinner(thread);
page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt(
pd[index] & ARM_PDE_ADDRESS_MASK);
for (index = VADDR_TO_PTENT(start); (index < 1024) && (start < end);
index++, start += B_PAGE_SIZE) {
page_table_entry oldEntry
= ARMPagingMethod32Bit::ClearPageTableEntry(&pt[index]);
if ((oldEntry & ARM_PTE_TYPE_MASK) == 0)
continue;
fMapCount--;
if (true /*(oldEntry & ARM_PTE_ACCESSED) != 0*/) { // XXX IRA
// Note, that we only need to invalidate the address, if the
// accessed flags was set, since only then the entry could have
// been in any TLB.
InvalidatePage(start);
}
if (area->cache_type != CACHE_TYPE_DEVICE) {
// get the page
vm_page* page = vm_lookup_page(
(oldEntry & ARM_PTE_ADDRESS_MASK) / B_PAGE_SIZE);
ASSERT(page != NULL);
DEBUG_PAGE_ACCESS_START(page);
// transfer the accessed/dirty flags to the page
if (/*(oldEntry & ARM_PTE_ACCESSED) != 0*/ true) // XXX IRA
page->accessed = true;
if (/*(oldEntry & ARM_PTE_DIRTY) != 0 */ true)
page->modified = true;
// remove the mapping object/decrement the wired_count of the
// page
if (area->wiring == B_NO_LOCK) {
vm_page_mapping* mapping = NULL;
vm_page_mappings::Iterator iterator
= page->mappings.GetIterator();
while ((mapping = iterator.Next()) != NULL) {
if (mapping->area == area)
break;
}
ASSERT(mapping != NULL);
area->mappings.Remove(mapping);
page->mappings.Remove(mapping);
queue.Add(mapping);
} else
page->DecrementWiredCount();
if (!page->IsMapped()) {
atomic_add(&gMappedPagesCount, -1);
if (updatePageQueue) {
if (page->Cache()->temporary)
vm_page_set_state(page, PAGE_STATE_INACTIVE);
else if (page->modified)
vm_page_set_state(page, PAGE_STATE_MODIFIED);
else
vm_page_set_state(page, PAGE_STATE_CACHED);
}
}
DEBUG_PAGE_ACCESS_END(page);
}
}
Flush();
// flush explicitly, since we directly use the lock
} while (start != 0 && start < end);
// TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not
// really critical here, as in all cases this method is used, the unmapped
// area range is unmapped for good (resized/cut) and the pages will likely
// be freed.
locker.Unlock();
// free removed mappings
bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = queue.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
}
void
X86VMTranslationMap64Bit::UnmapArea(VMArea* area, bool deletingAddressSpace,
bool ignoreTopCachePageFlags)
{
TRACE("X86VMTranslationMap64Bit::UnmapArea(%p)\n", area);
if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) {
X86VMTranslationMap64Bit::UnmapPages(area, area->Base(), area->Size(),
true);
return;
}
bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags;
RecursiveLocker locker(fLock);
ThreadCPUPinner pinner(thread_get_current_thread());
VMAreaMappings mappings;
mappings.MoveFrom(&area->mappings);
for (VMAreaMappings::Iterator it = mappings.GetIterator();
vm_page_mapping* mapping = it.Next();) {
vm_page* page = mapping->page;
page->mappings.Remove(mapping);
VMCache* cache = page->Cache();
bool pageFullyUnmapped = false;
if (!page->IsMapped()) {
atomic_add(&gMappedPagesCount, -1);
pageFullyUnmapped = true;
}
if (unmapPages || cache != area->cache) {
addr_t address = area->Base()
+ ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset);
uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress(
fPagingStructures->VirtualPML4(), address, fIsKernelMap,
false, NULL, fPageMapper, fMapCount);
if (entry == NULL) {
panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
"has no page table", page, area, address);
continue;
}
uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(entry);
if ((oldEntry & X86_64_PTE_PRESENT) == 0) {
panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
"has no page table entry", page, area, address);
continue;
}
// transfer the accessed/dirty flags to the page and invalidate
// the mapping, if necessary
if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
page->accessed = true;
if (!deletingAddressSpace)
InvalidatePage(address);
}
if ((oldEntry & X86_64_PTE_DIRTY) != 0)
page->modified = true;
if (pageFullyUnmapped) {
DEBUG_PAGE_ACCESS_START(page);
if (cache->temporary)
vm_page_set_state(page, PAGE_STATE_INACTIVE);
else if (page->modified)
vm_page_set_state(page, PAGE_STATE_MODIFIED);
else
vm_page_set_state(page, PAGE_STATE_CACHED);
DEBUG_PAGE_ACCESS_END(page);
}
}
fMapCount--;
}
Flush();
// flush explicitely, since we directly use the lock
locker.Unlock();
bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = mappings.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
}
X86VMTranslationMap64Bit::~X86VMTranslationMap64Bit()
{
TRACE("X86VMTranslationMap64Bit::~X86VMTranslationMap64Bit()\n");
if (fPagingStructures == NULL)
return;
if (fPageMapper != NULL) {
phys_addr_t address;
vm_page* page;
// Free all structures in the bottom half of the PML4 (user memory).
uint64* virtualPML4 = fPagingStructures->VirtualPML4();
for (uint32 i = 0; i < 256; i++) {
if ((virtualPML4[i] & X86_64_PML4E_PRESENT) == 0)
continue;
uint64* virtualPDPT = (uint64*)fPageMapper->GetPageTableAt(
virtualPML4[i] & X86_64_PML4E_ADDRESS_MASK);
for (uint32 j = 0; j < 512; j++) {
if ((virtualPDPT[j] & X86_64_PDPTE_PRESENT) == 0)
continue;
uint64* virtualPageDir = (uint64*)fPageMapper->GetPageTableAt(
virtualPDPT[j] & X86_64_PDPTE_ADDRESS_MASK);
for (uint32 k = 0; k < 512; k++) {
if ((virtualPageDir[k] & X86_64_PDE_PRESENT) == 0)
continue;
address = virtualPageDir[k] & X86_64_PDE_ADDRESS_MASK;
page = vm_lookup_page(address / B_PAGE_SIZE);
if (page == NULL) {
panic("page table %u %u %u on invalid page %#"
B_PRIxPHYSADDR "\n", i, j, k, address);
}
DEBUG_PAGE_ACCESS_START(page);
vm_page_set_state(page, PAGE_STATE_FREE);
}
address = virtualPDPT[j] & X86_64_PDPTE_ADDRESS_MASK;
page = vm_lookup_page(address / B_PAGE_SIZE);
if (page == NULL) {
panic("page directory %u %u on invalid page %#"
B_PRIxPHYSADDR "\n", i, j, address);
}
DEBUG_PAGE_ACCESS_START(page);
vm_page_set_state(page, PAGE_STATE_FREE);
}
address = virtualPML4[i] & X86_64_PML4E_ADDRESS_MASK;
page = vm_lookup_page(address / B_PAGE_SIZE);
if (page == NULL) {
panic("PDPT %u on invalid page %#" B_PRIxPHYSADDR "\n", i,
address);
}
DEBUG_PAGE_ACCESS_START(page);
vm_page_set_state(page, PAGE_STATE_FREE);
}
fPageMapper->Delete();
}
fPagingStructures->RemoveReference();
}
void
X86VMTranslationMap64Bit::UnmapPages(VMArea* area, addr_t base, size_t size,
bool updatePageQueue)
{
if (size == 0)
return;
addr_t start = base;
addr_t end = base + size - 1;
TRACE("X86VMTranslationMap64Bit::UnmapPages(%p, %#" B_PRIxADDR ", %#"
B_PRIxADDR ")\n", area, start, end);
VMAreaMappings queue;
RecursiveLocker locker(fLock);
ThreadCPUPinner pinner(thread_get_current_thread());
do {
uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress(
fPagingStructures->VirtualPML4(), start, fIsKernelMap, false,
NULL, fPageMapper, fMapCount);
if (pageTable == NULL) {
// Move on to the next page table.
start = ROUNDUP(start + 1, k64BitPageTableRange);
continue;
}
for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount;
index < k64BitTableEntryCount && start < end;
index++, start += B_PAGE_SIZE) {
uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(
&pageTable[index]);
if ((oldEntry & X86_64_PTE_PRESENT) == 0)
continue;
fMapCount--;
if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
// Note, that we only need to invalidate the address, if the
// accessed flags was set, since only then the entry could have
// been in any TLB.
InvalidatePage(start);
}
if (area->cache_type != CACHE_TYPE_DEVICE) {
// get the page
vm_page* page = vm_lookup_page(
(oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE);
ASSERT(page != NULL);
DEBUG_PAGE_ACCESS_START(page);
// transfer the accessed/dirty flags to the page
if ((oldEntry & X86_64_PTE_ACCESSED) != 0)
page->accessed = true;
if ((oldEntry & X86_64_PTE_DIRTY) != 0)
page->modified = true;
// remove the mapping object/decrement the wired_count of the
// page
if (area->wiring == B_NO_LOCK) {
vm_page_mapping* mapping = NULL;
vm_page_mappings::Iterator iterator
= page->mappings.GetIterator();
while ((mapping = iterator.Next()) != NULL) {
if (mapping->area == area)
break;
}
ASSERT(mapping != NULL);
area->mappings.Remove(mapping);
page->mappings.Remove(mapping);
queue.Add(mapping);
} else
page->DecrementWiredCount();
if (!page->IsMapped()) {
atomic_add(&gMappedPagesCount, -1);
if (updatePageQueue) {
if (page->Cache()->temporary)
vm_page_set_state(page, PAGE_STATE_INACTIVE);
else if (page->modified)
vm_page_set_state(page, PAGE_STATE_MODIFIED);
else
vm_page_set_state(page, PAGE_STATE_CACHED);
}
}
DEBUG_PAGE_ACCESS_END(page);
}
}
Flush();
// flush explicitly, since we directly use the lock
} while (start != 0 && start < end);
// TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not
// really critical here, as in all cases this method is used, the unmapped
// area range is unmapped for good (resized/cut) and the pages will likely
// be freed.
locker.Unlock();
// free removed mappings
bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = queue.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
}
M68KVMTranslationMap040::~M68KVMTranslationMap040()
{
if (fPagingStructures == NULL)
return;
if (fPageMapper != NULL)
fPageMapper->Delete();
if (fPagingStructures->pgroot_virt != NULL) {
page_root_entry *pgroot_virt = fPagingStructures->pgroot_virt;
// cycle through and free all of the user space pgdirs & pgtables
// since the size of tables don't match B_PAGE_SIZE,
// we alloc several at once, based on modulos,
// we make sure they are either all in the tree or none.
for (uint32 i = VADDR_TO_PRENT(USER_BASE);
i <= VADDR_TO_PRENT(USER_BASE + (USER_SIZE - 1)); i++) {
addr_t pgdir_pn;
page_directory_entry *pgdir;
vm_page *dirpage;
if (PRE_TYPE(pgroot_virt[i]) == DT_INVALID)
continue;
if (PRE_TYPE(pgroot_virt[i]) != DT_ROOT) {
panic("rtdir[%ld]: buggy descriptor type", i);
return;
}
// XXX:suboptimal (done 8 times)
pgdir_pn = PRE_TO_PN(pgroot_virt[i]);
dirpage = vm_lookup_page(pgdir_pn);
pgdir = &(((page_directory_entry *)dirpage)[i%NUM_DIRTBL_PER_PAGE]);
for (uint32 j = 0; j <= NUM_DIRENT_PER_TBL;
j+=NUM_PAGETBL_PER_PAGE) {
addr_t pgtbl_pn;
page_table_entry *pgtbl;
vm_page *page;
if (PDE_TYPE(pgdir[j]) == DT_INVALID)
continue;
if (PDE_TYPE(pgdir[j]) != DT_DIR) {
panic("pgroot[%ld][%ld]: buggy descriptor type", i, j);
return;
}
pgtbl_pn = PDE_TO_PN(pgdir[j]);
page = vm_lookup_page(pgtbl_pn);
pgtbl = (page_table_entry *)page;
if (!page) {
panic("destroy_tmap: didn't find pgtable page\n");
return;
}
DEBUG_PAGE_ACCESS_START(page);
vm_page_set_state(page, PAGE_STATE_FREE);
}
if (((i + 1) % NUM_DIRTBL_PER_PAGE) == 0) {
DEBUG_PAGE_ACCESS_END(dirpage);
vm_page_set_state(dirpage, PAGE_STATE_FREE);
}
}
#if 0
//X86
for (uint32 i = VADDR_TO_PDENT(USER_BASE);
i <= VADDR_TO_PDENT(USER_BASE + (USER_SIZE - 1)); i++) {
if ((fPagingStructures->pgdir_virt[i] & M68K_PDE_PRESENT) != 0) {
addr_t address = fPagingStructures->pgdir_virt[i]
& M68K_PDE_ADDRESS_MASK;
vm_page* page = vm_lookup_page(address / B_PAGE_SIZE);
if (!page)
panic("destroy_tmap: didn't find pgtable page\n");
DEBUG_PAGE_ACCESS_START(page);
vm_page_set_state(page, PAGE_STATE_FREE);
}
}
#endif
}
fPagingStructures->RemoveReference();
}
static status_t
cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer,
size_t* _size, bool doWrite)
{
if (_cacheRef == NULL)
panic("cache_io() called with NULL ref!\n");
file_cache_ref* ref = (file_cache_ref*)_cacheRef;
VMCache* cache = ref->cache;
off_t fileSize = cache->virtual_end;
bool useBuffer = buffer != 0;
TRACE(("cache_io(ref = %p, offset = %Ld, buffer = %p, size = %lu, %s)\n",
ref, offset, (void*)buffer, *_size, doWrite ? "write" : "read"));
// out of bounds access?
if (offset >= fileSize || offset < 0) {
*_size = 0;
return B_OK;
}
int32 pageOffset = offset & (B_PAGE_SIZE - 1);
size_t size = *_size;
offset -= pageOffset;
if ((off_t)(offset + pageOffset + size) > fileSize) {
// adapt size to be within the file's offsets
size = fileSize - pageOffset - offset;
*_size = size;
}
if (size == 0)
return B_OK;
// "offset" and "lastOffset" are always aligned to B_PAGE_SIZE,
// the "last*" variables always point to the end of the last
// satisfied request part
const uint32 kMaxChunkSize = MAX_IO_VECS * B_PAGE_SIZE;
size_t bytesLeft = size, lastLeft = size;
int32 lastPageOffset = pageOffset;
addr_t lastBuffer = buffer;
off_t lastOffset = offset;
size_t lastReservedPages = min_c(MAX_IO_VECS, (pageOffset + bytesLeft
+ B_PAGE_SIZE - 1) >> PAGE_SHIFT);
size_t reservePages = 0;
size_t pagesProcessed = 0;
cache_func function = NULL;
vm_page_reservation reservation;
reserve_pages(ref, &reservation, lastReservedPages, doWrite);
AutoLocker<VMCache> locker(cache);
while (bytesLeft > 0) {
// Periodically reevaluate the low memory situation and select the
// read/write hook accordingly
if (pagesProcessed % 32 == 0) {
if (size >= BYPASS_IO_SIZE
&& low_resource_state(B_KERNEL_RESOURCE_PAGES)
!= B_NO_LOW_RESOURCE) {
// In low memory situations we bypass the cache beyond a
// certain I/O size.
function = doWrite ? write_to_file : read_from_file;
} else
function = doWrite ? write_to_cache : read_into_cache;
}
// check if this page is already in memory
vm_page* page = cache->LookupPage(offset);
if (page != NULL) {
// The page may be busy - since we need to unlock the cache sometime
// in the near future, we need to satisfy the request of the pages
// we didn't get yet (to make sure no one else interferes in the
// meantime).
status_t status = satisfy_cache_io(ref, cookie, function, offset,
buffer, useBuffer, pageOffset, bytesLeft, reservePages,
lastOffset, lastBuffer, lastPageOffset, lastLeft,
lastReservedPages, &reservation);
if (status != B_OK)
return status;
// Since satisfy_cache_io() unlocks the cache, we need to look up
// the page again.
page = cache->LookupPage(offset);
if (page != NULL && page->busy) {
cache->WaitForPageEvents(page, PAGE_EVENT_NOT_BUSY, true);
continue;
}
}
size_t bytesInPage = min_c(size_t(B_PAGE_SIZE - pageOffset), bytesLeft);
TRACE(("lookup page from offset %Ld: %p, size = %lu, pageOffset "
"= %lu\n", offset, page, bytesLeft, pageOffset));
if (page != NULL) {
if (doWrite || useBuffer) {
// Since the following user_mem{cpy,set}() might cause a page
// fault, which in turn might cause pages to be reserved, we
// need to unlock the cache temporarily to avoid a potential
// deadlock. To make sure that our page doesn't go away, we mark
// it busy for the time.
page->busy = true;
locker.Unlock();
// copy the contents of the page already in memory
phys_addr_t pageAddress
= (phys_addr_t)page->physical_page_number * B_PAGE_SIZE
+ pageOffset;
bool userBuffer = IS_USER_ADDRESS(buffer);
if (doWrite) {
if (useBuffer) {
vm_memcpy_to_physical(pageAddress, (void*)buffer,
bytesInPage, userBuffer);
} else {
vm_memset_physical(pageAddress, 0, bytesInPage);
}
} else if (useBuffer) {
vm_memcpy_from_physical((void*)buffer, pageAddress,
bytesInPage, userBuffer);
}
locker.Lock();
if (doWrite) {
DEBUG_PAGE_ACCESS_START(page);
page->modified = true;
if (page->State() != PAGE_STATE_MODIFIED)
vm_page_set_state(page, PAGE_STATE_MODIFIED);
DEBUG_PAGE_ACCESS_END(page);
}
cache->MarkPageUnbusy(page);
}
// If it is cached only, requeue the page, so the respective queue
// roughly remains LRU first sorted.
if (page->State() == PAGE_STATE_CACHED
|| page->State() == PAGE_STATE_MODIFIED) {
DEBUG_PAGE_ACCESS_START(page);
vm_page_requeue(page, true);
DEBUG_PAGE_ACCESS_END(page);
}
if (bytesLeft <= bytesInPage) {
// we've read the last page, so we're done!
locker.Unlock();
vm_page_unreserve_pages(&reservation);
return B_OK;
}
// prepare a potential gap request
lastBuffer = buffer + bytesInPage;
lastLeft = bytesLeft - bytesInPage;
lastOffset = offset + B_PAGE_SIZE;
lastPageOffset = 0;
}
if (bytesLeft <= bytesInPage)
break;
buffer += bytesInPage;
bytesLeft -= bytesInPage;
pageOffset = 0;
offset += B_PAGE_SIZE;
pagesProcessed++;
if (buffer - lastBuffer + lastPageOffset >= kMaxChunkSize) {
status_t status = satisfy_cache_io(ref, cookie, function, offset,
buffer, useBuffer, pageOffset, bytesLeft, reservePages,
lastOffset, lastBuffer, lastPageOffset, lastLeft,
lastReservedPages, &reservation);
if (status != B_OK)
return status;
}
}
// fill the last remaining bytes of the request (either write or read)
return function(ref, cookie, lastOffset, lastPageOffset, lastBuffer,
lastLeft, useBuffer, &reservation, 0);
}
/*! Reads the requested amount of data into the cache, and allocates
pages needed to fulfill that request. This function is called by cache_io().
It can only handle a certain amount of bytes, and the caller must make
sure that it matches that criterion.
The cache_ref lock must be held when calling this function; during
operation it will unlock the cache, though.
*/
static status_t
read_into_cache(file_cache_ref* ref, void* cookie, off_t offset,
int32 pageOffset, addr_t buffer, size_t bufferSize, bool useBuffer,
vm_page_reservation* reservation, size_t reservePages)
{
TRACE(("read_into_cache(offset = %Ld, pageOffset = %ld, buffer = %#lx, "
"bufferSize = %lu\n", offset, pageOffset, buffer, bufferSize));
VMCache* cache = ref->cache;
// TODO: We're using way too much stack! Rather allocate a sufficiently
// large chunk on the heap.
generic_io_vec vecs[MAX_IO_VECS];
uint32 vecCount = 0;
generic_size_t numBytes = PAGE_ALIGN(pageOffset + bufferSize);
vm_page* pages[MAX_IO_VECS];
int32 pageIndex = 0;
// allocate pages for the cache and mark them busy
for (generic_size_t pos = 0; pos < numBytes; pos += B_PAGE_SIZE) {
vm_page* page = pages[pageIndex++] = vm_page_allocate_page(
reservation, PAGE_STATE_CACHED | VM_PAGE_ALLOC_BUSY);
cache->InsertPage(page, offset + pos);
add_to_iovec(vecs, vecCount, MAX_IO_VECS,
page->physical_page_number * B_PAGE_SIZE, B_PAGE_SIZE);
// TODO: check if the array is large enough (currently panics)!
}
push_access(ref, offset, bufferSize, false);
cache->Unlock();
vm_page_unreserve_pages(reservation);
// read file into reserved pages
status_t status = read_pages_and_clear_partial(ref, cookie, offset, vecs,
vecCount, B_PHYSICAL_IO_REQUEST, &numBytes);
if (status != B_OK) {
// reading failed, free allocated pages
dprintf("file_cache: read pages failed: %s\n", strerror(status));
cache->Lock();
for (int32 i = 0; i < pageIndex; i++) {
cache->NotifyPageEvents(pages[i], PAGE_EVENT_NOT_BUSY);
cache->RemovePage(pages[i]);
vm_page_set_state(pages[i], PAGE_STATE_FREE);
}
return status;
}
// copy the pages if needed and unmap them again
for (int32 i = 0; i < pageIndex; i++) {
if (useBuffer && bufferSize != 0) {
size_t bytes = min_c(bufferSize, (size_t)B_PAGE_SIZE - pageOffset);
vm_memcpy_from_physical((void*)buffer,
pages[i]->physical_page_number * B_PAGE_SIZE + pageOffset,
bytes, IS_USER_ADDRESS(buffer));
buffer += bytes;
bufferSize -= bytes;
pageOffset = 0;
}
}
reserve_pages(ref, reservation, reservePages, false);
cache->Lock();
// make the pages accessible in the cache
for (int32 i = pageIndex; i-- > 0;) {
DEBUG_PAGE_ACCESS_END(pages[i]);
cache->MarkPageUnbusy(pages[i]);
}
return B_OK;
}
