bool AUMCircularBufferInit(AUMCircularBuffer *buffer, int length) { buffer->length = round_page(length); // We need whole page sizes // Temporarily allocate twice the length, so we have the contiguous address space to // support a second instance of the buffer directly after vm_address_t bufferAddress; if ( !checkResult(vm_allocate(mach_task_self(), &bufferAddress, buffer->length * 2, TRUE /* (don't use the current bufferAddress value) */), "Buffer allocation") ) return false; // Now replace the second half of the allocation with a virtual copy of the first half. Deallocate the second half... if ( !checkResult(vm_deallocate(mach_task_self(), bufferAddress + buffer->length, buffer->length), "Buffer deallocation") ) return false; // Then create a memory entry that refers to the buffer vm_size_t entry_length = buffer->length; mach_port_t memoryEntry; if ( !checkResult(mach_make_memory_entry(mach_task_self(), &entry_length, bufferAddress, VM_PROT_READ|VM_PROT_WRITE, &memoryEntry, 0), "Create memory entry") ) { vm_deallocate(mach_task_self(), bufferAddress, buffer->length); return false; } // And map the memory entry to the address space immediately after the buffer vm_address_t virtualAddress = bufferAddress + buffer->length; if ( !checkResult(vm_map(mach_task_self(), &virtualAddress, buffer->length, 0, FALSE, memoryEntry, 0, FALSE, VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ | VM_PROT_WRITE, VM_INHERIT_DEFAULT), "Map buffer memory") ) { vm_deallocate(mach_task_self(), bufferAddress, buffer->length); return false; } if ( virtualAddress != bufferAddress+buffer->length ) { printf("Couldn't map buffer memory to end of buffer\n"); vm_deallocate(mach_task_self(), virtualAddress, buffer->length); vm_deallocate(mach_task_self(), bufferAddress, buffer->length); return false; } buffer->buffer = (void*)bufferAddress; buffer->fillCount = 0; buffer->head = buffer->tail = 0; return true; }
static void* commpage_allocate( vm_map_t submap, // commpage32_map or commpage_map64 size_t area_used, // _COMM_PAGE32_AREA_USED or _COMM_PAGE64_AREA_USED vm_prot_t uperm) { vm_offset_t kernel_addr = 0; // address of commpage in kernel map vm_offset_t zero = 0; vm_size_t size = area_used; // size actually populated vm_map_entry_t entry; ipc_port_t handle; kern_return_t kr; if (submap == NULL) panic("commpage submap is null"); if ((kr = vm_map(kernel_map, &kernel_addr, area_used, 0, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_OSFMK), NULL, 0, FALSE, VM_PROT_ALL, VM_PROT_ALL, VM_INHERIT_NONE))) panic("cannot allocate commpage %d", kr); if ((kr = vm_map_wire(kernel_map, kernel_addr, kernel_addr+area_used, VM_PROT_DEFAULT|VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_OSFMK), FALSE))) panic("cannot wire commpage: %d", kr); /* * Now that the object is created and wired into the kernel map, mark it so that no delay * copy-on-write will ever be performed on it as a result of mapping it into user-space. * If such a delayed copy ever occurred, we could remove the kernel's wired mapping - and * that would be a real disaster. * * JMM - What we really need is a way to create it like this in the first place. */ if (!(kr = vm_map_lookup_entry( kernel_map, vm_map_trunc_page(kernel_addr, VM_MAP_PAGE_MASK(kernel_map)), &entry) || entry->is_sub_map)) panic("cannot find commpage entry %d", kr); VME_OBJECT(entry)->copy_strategy = MEMORY_OBJECT_COPY_NONE; if ((kr = mach_make_memory_entry( kernel_map, // target map &size, // size kernel_addr, // offset (address in kernel map) uperm, // protections as specified &handle, // this is the object handle we get NULL ))) // parent_entry (what is this?) panic("cannot make entry for commpage %d", kr); if ((kr = vm_map_64( submap, // target map (shared submap) &zero, // address (map into 1st page in submap) area_used, // size 0, // mask VM_FLAGS_FIXED, // flags (it must be 1st page in submap) handle, // port is the memory entry we just made 0, // offset (map 1st page in memory entry) FALSE, // copy uperm, // cur_protection (R-only in user map) uperm, // max_protection VM_INHERIT_SHARE ))) // inheritance panic("cannot map commpage %d", kr); ipc_port_release(handle); /* Make the kernel mapping non-executable. This cannot be done * at the time of map entry creation as mach_make_memory_entry * cannot handle disjoint permissions at this time. */ kr = vm_protect(kernel_map, kernel_addr, area_used, FALSE, VM_PROT_READ | VM_PROT_WRITE); assert (kr == KERN_SUCCESS); return (void*)(intptr_t)kernel_addr; // return address in kernel map }
bool IOBufferMemoryDescriptor::initWithOptions( IOOptionBits options, vm_size_t capacity, vm_offset_t alignment, task_t inTask) { vm_map_t map = 0; IOOptionBits iomdOptions = kIOMemoryAsReference | kIOMemoryTypeVirtual; if (!capacity) return false; _options = options; _capacity = capacity; _physAddrs = 0; _physSegCount = 0; _buffer = 0; // Grab the direction and the Auto Prepare bits from the Buffer MD options iomdOptions |= options & (kIOMemoryDirectionMask | kIOMemoryAutoPrepare); if ((options & kIOMemorySharingTypeMask) && (alignment < page_size)) alignment = page_size; if ((inTask != kernel_task) && !(options & kIOMemoryPageable)) return false; _alignment = alignment; if (options & kIOMemoryPageable) { iomdOptions |= kIOMemoryBufferPageable; if (inTask == kernel_task) { /* Allocate some kernel address space. */ _buffer = IOMallocPageable(capacity, alignment); if (_buffer) map = IOPageableMapForAddress((vm_address_t) _buffer); } else { kern_return_t kr; if( !reserved) { reserved = IONew( ExpansionData, 1 ); if( !reserved) return( false ); } map = get_task_map(inTask); vm_map_reference(map); reserved->map = map; kr = vm_allocate( map, (vm_address_t *) &_buffer, round_page_32(capacity), VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_MEMORY_IOKIT) ); if( KERN_SUCCESS != kr) return( false ); // we have to make sure that these pages don't get copied on fork. kr = vm_inherit( map, (vm_address_t) _buffer, round_page_32(capacity), VM_INHERIT_NONE); if( KERN_SUCCESS != kr) return( false ); } } else { // @@@ gvdl: Need to remove this // Buffer should never auto prepare they should be prepared explicitly // But it never was enforced so what are you going to do? iomdOptions |= kIOMemoryAutoPrepare; /* Allocate a wired-down buffer inside kernel space. */ if (options & kIOMemoryPhysicallyContiguous) _buffer = IOMallocContiguous(capacity, alignment, 0); else if (alignment > 1) _buffer = IOMallocAligned(capacity, alignment); else _buffer = IOMalloc(capacity); } if (!_buffer) return false; _singleRange.v.address = (vm_address_t) _buffer; _singleRange.v.length = capacity; if (!super::initWithOptions(&_singleRange.v, 1, 0, inTask, iomdOptions, /* System mapper */ 0)) return false; if (options & kIOMemoryPageable) { kern_return_t kr; ipc_port_t sharedMem = (ipc_port_t) _memEntry; vm_size_t size = round_page_32(_ranges.v[0].length); // must create the entry before any pages are allocated if( 0 == sharedMem) { // set memory entry cache vm_prot_t memEntryCacheMode = VM_PROT_READ | VM_PROT_WRITE; switch (options & kIOMapCacheMask) { case kIOMapInhibitCache: SET_MAP_MEM(MAP_MEM_IO, memEntryCacheMode); break; case kIOMapWriteThruCache: SET_MAP_MEM(MAP_MEM_WTHRU, memEntryCacheMode); break; case kIOMapWriteCombineCache: SET_MAP_MEM(MAP_MEM_WCOMB, memEntryCacheMode); break; case kIOMapCopybackCache: SET_MAP_MEM(MAP_MEM_COPYBACK, memEntryCacheMode); break; case kIOMapDefaultCache: default: SET_MAP_MEM(MAP_MEM_NOOP, memEntryCacheMode); break; } kr = mach_make_memory_entry( map, &size, _ranges.v[0].address, memEntryCacheMode, &sharedMem, NULL ); if( (KERN_SUCCESS == kr) && (size != round_page_32(_ranges.v[0].length))) { ipc_port_release_send( sharedMem ); kr = kIOReturnVMError; } if( KERN_SUCCESS != kr) sharedMem = 0; _memEntry = (void *) sharedMem; } } setLength(capacity); return true; }
bool IOBufferMemoryDescriptor::initWithPhysicalMask( task_t inTask, IOOptionBits options, mach_vm_size_t capacity, mach_vm_address_t alignment, mach_vm_address_t physicalMask) { kern_return_t kr; task_t mapTask = NULL; vm_map_t vmmap = NULL; mach_vm_address_t highestMask = 0; IOOptionBits iomdOptions = kIOMemoryTypeVirtual64 | kIOMemoryAsReference; IODMAMapSpecification mapSpec; bool mapped = false; bool needZero; if (!capacity) return false; _options = options; _capacity = capacity; _internalFlags = 0; _internalReserved = 0; _buffer = 0; _ranges.v64 = IONew(IOAddressRange, 1); if (!_ranges.v64) return (false); _ranges.v64->address = 0; _ranges.v64->length = 0; // make sure super::free doesn't dealloc _ranges before super::init _flags = kIOMemoryAsReference; // Grab IOMD bits from the Buffer MD options iomdOptions |= (options & kIOBufferDescriptorMemoryFlags); if (!(kIOMemoryMapperNone & options)) { IOMapper::checkForSystemMapper(); mapped = (0 != IOMapper::gSystem); } needZero = mapped; if (physicalMask && (alignment <= 1)) { alignment = ((physicalMask ^ (-1ULL)) & (physicalMask - 1)); highestMask = (physicalMask | alignment); alignment++; if (alignment < page_size) alignment = page_size; } if ((options & (kIOMemorySharingTypeMask | kIOMapCacheMask | kIOMemoryClearEncrypt)) && (alignment < page_size)) alignment = page_size; if (alignment >= page_size) capacity = round_page(capacity); if (alignment > page_size) options |= kIOMemoryPhysicallyContiguous; _alignment = alignment; if ((inTask != kernel_task) && !(options & kIOMemoryPageable)) return false; bzero(&mapSpec, sizeof(mapSpec)); mapSpec.alignment = _alignment; mapSpec.numAddressBits = 64; if (highestMask && mapped) { if (highestMask <= 0xFFFFFFFF) mapSpec.numAddressBits = (32 - __builtin_clz((unsigned int) highestMask)); else mapSpec.numAddressBits = (64 - __builtin_clz((unsigned int) (highestMask >> 32))); highestMask = 0; } // set flags for entry + object create vm_prot_t memEntryCacheMode = VM_PROT_READ | VM_PROT_WRITE; // set memory entry cache mode switch (options & kIOMapCacheMask) { case kIOMapInhibitCache: SET_MAP_MEM(MAP_MEM_IO, memEntryCacheMode); break; case kIOMapWriteThruCache: SET_MAP_MEM(MAP_MEM_WTHRU, memEntryCacheMode); break; case kIOMapWriteCombineCache: SET_MAP_MEM(MAP_MEM_WCOMB, memEntryCacheMode); break; case kIOMapCopybackCache: SET_MAP_MEM(MAP_MEM_COPYBACK, memEntryCacheMode); break; case kIOMapCopybackInnerCache: SET_MAP_MEM(MAP_MEM_INNERWBACK, memEntryCacheMode); break; case kIOMapDefaultCache: default: SET_MAP_MEM(MAP_MEM_NOOP, memEntryCacheMode); break; } if (options & kIOMemoryPageable) { iomdOptions |= kIOMemoryBufferPageable; // must create the entry before any pages are allocated // set flags for entry + object create memEntryCacheMode |= MAP_MEM_NAMED_CREATE; if (options & kIOMemoryPurgeable) memEntryCacheMode |= MAP_MEM_PURGABLE; } else { memEntryCacheMode |= MAP_MEM_NAMED_REUSE; vmmap = kernel_map; // Buffer shouldn't auto prepare they should be prepared explicitly // But it never was enforced so what are you going to do? iomdOptions |= kIOMemoryAutoPrepare; /* Allocate a wired-down buffer inside kernel space. */ bool contig = (0 != (options & kIOMemoryHostPhysicallyContiguous)); if (!contig && (0 != (options & kIOMemoryPhysicallyContiguous))) { contig |= (!mapped); contig |= (0 != (kIOMemoryMapperNone & options)); #if 0 // treat kIOMemoryPhysicallyContiguous as kIOMemoryHostPhysicallyContiguous for now contig |= true; #endif } if (contig || highestMask || (alignment > page_size)) { _internalFlags |= kInternalFlagPhysical; if (highestMask) { _internalFlags |= kInternalFlagPageSized; capacity = round_page(capacity); } _buffer = (void *) IOKernelAllocateWithPhysicalRestrict( capacity, highestMask, alignment, contig); } else if (needZero && ((capacity + alignment) <= (page_size - kIOPageAllocChunkBytes))) { _internalFlags |= kInternalFlagPageAllocated; needZero = false; _buffer = (void *) iopa_alloc(capacity, alignment); } else if (alignment > 1) { _buffer = IOMallocAligned(capacity, alignment); } else { _buffer = IOMalloc(capacity); } if (!_buffer) { return false; } if (needZero) bzero(_buffer, capacity); } if( (options & (kIOMemoryPageable | kIOMapCacheMask))) { ipc_port_t sharedMem; vm_size_t size = round_page(capacity); kr = mach_make_memory_entry(vmmap, &size, (vm_offset_t)_buffer, memEntryCacheMode, &sharedMem, NULL ); if( (KERN_SUCCESS == kr) && (size != round_page(capacity))) { ipc_port_release_send( sharedMem ); kr = kIOReturnVMError; } if( KERN_SUCCESS != kr) return( false ); _memEntry = (void *) sharedMem; if( options & kIOMemoryPageable) { #if IOALLOCDEBUG debug_iomallocpageable_size += size; #endif mapTask = inTask; if (NULL == inTask) inTask = kernel_task; } else if (options & kIOMapCacheMask) { // Prefetch each page to put entries into the pmap volatile UInt8 * startAddr = (UInt8 *)_buffer; volatile UInt8 * endAddr = (UInt8 *)_buffer + capacity; while (startAddr < endAddr) { *startAddr; startAddr += page_size; } } } _ranges.v64->address = (mach_vm_address_t) _buffer;; _ranges.v64->length = _capacity; if (!super::initWithOptions(_ranges.v64, 1, 0, inTask, iomdOptions, /* System mapper */ 0)) return false; // give any system mapper the allocation params if (kIOReturnSuccess != dmaCommandOperation(kIOMDAddDMAMapSpec, &mapSpec, sizeof(mapSpec))) return false; if (mapTask) { if (!reserved) { reserved = IONew( ExpansionData, 1 ); if( !reserved) return( false ); } reserved->map = createMappingInTask(mapTask, 0, kIOMapAnywhere | (options & kIOMapCacheMask), 0, 0); if (!reserved->map) { _buffer = 0; return( false ); } release(); // map took a retain on this reserved->map->retain(); removeMapping(reserved->map); mach_vm_address_t buffer = reserved->map->getAddress(); _buffer = (void *) buffer; if (kIOMemoryTypeVirtual64 == (kIOMemoryTypeMask & iomdOptions)) _ranges.v64->address = buffer; } setLength(_capacity); return true; }