static IOReturn
IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable)
{
IOReturn err = kIOReturnSuccess;
int32_t idx = 0;
IOPolledInterface * poller;
AbsoluteTime deadline;
if (!vars->io) return (kIOReturnSuccess);
abortable &= vars->abortable;
clock_interval_to_deadline(2000, kMillisecondScale, &deadline);
while (-1 == vars->ioStatus)
{
for (idx = 0;
(poller = (IOPolledInterface *) vars->pollers->getObject(idx));
idx++)
{
IOReturn newErr;
newErr = poller->checkForWork();
if ((newErr == kIOReturnAborted) && !abortable)
newErr = kIOReturnSuccess;
if (kIOReturnSuccess == err)
err = newErr;
}
if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline)))
{
HIBLOG("IOPolledInterface::forced timeout\n");
vars->ioStatus = kIOReturnTimeout;
}
}
vars->io = false;
#if HIBERNATION
if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
{
err = kIOReturnAborted;
HIBLOG("IOPolledInterface::checkForWork sw abort\n");
}
#endif
if (err)
{
HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
}
else
{
err = vars->ioStatus;
if (kIOReturnSuccess != err) HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
}
return (err);
}
IOReturn
IOPolledFilePollersSetup(IOPolledFileIOVars * vars,
uint32_t openState)
{
IOReturn err;
err = kIOReturnSuccess;
do
{
if (!vars->pollers->openCount)
{
err = IOPolledFilePollersProbe(vars->pollers);
if (kIOReturnSuccess != err) break;
err = IOPolledFilePollersOpen(vars, openState, false);
if (kIOReturnSuccess != err) break;
vars->pollers->openState = openState;
}
vars->pollers->openCount++;
vars->pollers->io = false;
vars->buffer = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy();
vars->bufferHalf = 0;
vars->bufferOffset = 0;
vars->bufferSize = (vars->pollers->ioBuffer->getLength() >> 1);
if (vars->maxiobytes < vars->bufferSize) vars->bufferSize = vars->maxiobytes;
}
while (false);
if (kIOReturnSuccess != err) HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n", openState, err);
return (err);
}
static IOReturn
IOStartPolledIO(IOPolledFilePollers * vars,
uint32_t operation, uint32_t bufferOffset,
uint64_t deviceOffset, uint64_t length)
{
IOReturn err;
IOPolledInterface * poller;
IOPolledCompletion completion;
err = vars->ioStatus;
if (kIOReturnSuccess != err) return (err);
completion.target = 0;
completion.action = &IOPolledIOComplete;
completion.parameter = vars;
vars->ioStatus = -1;
poller = (IOPolledInterface *) vars->pollers->getObject(0);
err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion);
if (err)
HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
return (err);
}
void ml_ppc_sleep(void)
{
struct per_proc_info *proc_info;
boolean_t dohalt;
proc_info = getPerProc();
if (!proc_info->hibernate)
{
ml_ppc_do_sleep();
return;
}
{
uint64_t start, end, nsec;
HIBLOG("mapping_hibernate_flush start\n");
clock_get_uptime(&start);
mapping_hibernate_flush();
clock_get_uptime(&end);
absolutetime_to_nanoseconds(end - start, &nsec);
HIBLOG("mapping_hibernate_flush time: %qd ms\n", nsec / 1000000ULL);
}
dohalt = hibernate_write_image();
if (dohalt)
{
// off
HIBLOG("power off\n");
if (PE_halt_restart)
(*PE_halt_restart)(kPEHaltCPU);
}
else
{
// sleep
HIBLOG("sleep\n");
// should we come back via regular wake, set the state in memory.
PerProcTable[0].ppe_vaddr->hibernate = 0;
PE_cpu_machine_quiesce(proc_info->cpu_id);
return;
}
}
IOReturn
IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable)
{
IOPolledFilePollers * vars = filevars->pollers;
IOBufferMemoryDescriptor * ioBuffer;
IOPolledInterface * poller;
IOService * next;
IOReturn err = kIOReturnError;
int32_t idx;
vars->abortable = abortable;
ioBuffer = 0;
if (kIOPolledAfterSleepState == state)
{
vars->ioStatus = 0;
vars->io = false;
}
(void) IOPolledFilePollersIODone(vars, false);
if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state))
{
ioBuffer = vars->ioBuffer;
if (!ioBuffer)
{
vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionInOut,
2 * kDefaultIOSize, page_size);
if (!ioBuffer) return (kIOReturnNoMemory);
}
}
for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
{
poller = (IOPolledInterface *) vars->pollers->getObject(idx);
err = poller->open(state, ioBuffer);
if ((kIOReturnSuccess != err) && (kIOPolledPreflightCoreDumpState == state))
{
err = poller->open(kIOPolledPreflightState, ioBuffer);
}
if (kIOReturnSuccess != err)
{
HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
break;
}
}
if (kIOReturnSuccess == err)
{
next = vars->media;
while (next)
{
next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
next = next->getProvider();
}
}
return (err);
}
kern_return_t
hibernate_setup(IOHibernateImageHeader * header,
uint32_t free_page_ratio,
uint32_t free_page_time,
boolean_t vmflush,
hibernate_page_list_t ** page_list_ret,
hibernate_page_list_t ** page_list_wired_ret,
boolean_t * encryptedswap)
{
hibernate_page_list_t * page_list = NULL;
hibernate_page_list_t * page_list_wired = NULL;
uint32_t gobble_count;
*page_list_ret = NULL;
*page_list_wired_ret = NULL;
if (vmflush)
hibernate_flush_memory();
page_list = hibernate_page_list_allocate();
if (!page_list)
return (KERN_RESOURCE_SHORTAGE);
page_list_wired = hibernate_page_list_allocate();
if (!page_list_wired)
{
kfree(page_list, page_list->list_size);
return (KERN_RESOURCE_SHORTAGE);
}
*encryptedswap = dp_encryption;
// pages we could force out to reduce hibernate image size
gobble_count = (uint32_t)((((uint64_t) page_list->page_count) * ((uint64_t) free_page_ratio)) / 100);
// no failures hereafter
hibernate_processor_setup(header);
HIBLOG("hibernate_alloc_pages flags %08x, gobbling %d pages\n",
header->processorFlags, gobble_count);
if (gobble_count)
hibernate_gobble_pages(gobble_count, free_page_time);
*page_list_ret = page_list;
*page_list_wired_ret = page_list_wired;
return (KERN_SUCCESS);
}
IOReturn
IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state)
{
IOPolledFilePollers * vars = filevars->pollers;
IOPolledInterface * poller;
IORegistryEntry * next;
IOReturn err;
int32_t idx;
(void) IOPolledFilePollersIODone(vars, false);
if (kIOPolledPostflightState == state)
{
vars->openCount--;
if (vars->openCount)
{
// 21207427
IOPolledFilePollersOpen(filevars, vars->openState, vars->abortable);
return (kIOReturnSuccess);
}
}
for (idx = 0, err = kIOReturnSuccess;
(poller = (IOPolledInterface *) vars->pollers->getObject(idx));
idx++)
{
err = poller->close(state);
if (err) HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
}
if (kIOPolledPostflightState == state)
{
next = vars->media;
while (next)
{
next->removeProperty(kIOPolledInterfaceActiveKey);
next = next->getParentEntry(gIOServicePlane);
}
if (vars->ioBuffer)
{
vars->ioBuffer->release();
vars->ioBuffer = 0;
}
}
return (err);
}
static IOReturn
IOPolledFilePollersProbe(IOPolledFilePollers * vars)
{
IOReturn err = kIOReturnError;
int32_t idx;
IOPolledInterface * poller;
for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
{
poller = (IOPolledInterface *) vars->pollers->getObject(idx);
err = poller->probe(vars->media);
if (err)
{
HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
break;
}
}
return (err);
}
void
hibernate_page_list_setall_machine(hibernate_page_list_t * page_list,
hibernate_page_list_t * page_list_wired,
uint32_t * pagesOut)
{
uint32_t page, count, PCAsize;
/* Get total size of PCA table */
PCAsize = round_page((hash_table_size / PerProcTable[0].ppe_vaddr->pf.pfPTEG)
* sizeof(PCA_t));
page = atop_64(hash_table_base - PCAsize);
count = atop_64(hash_table_size + PCAsize);
hibernate_set_page_state(page_list, page_list_wired, page, count, 0);
pagesOut -= count;
HIBLOG("removed hash, pca: %d pages\n", count);
save_snapshot();
}
IOReturn
IOPolledFileRead(IOPolledFileIOVars * vars,
uint8_t * bytes, IOByteCount size,
IOPolledFileCryptVars * cryptvars)
{
IOReturn err = kIOReturnSuccess;
IOByteCount copy;
// bytesWritten += size;
do
{
copy = vars->bufferLimit - vars->bufferOffset;
if (copy > size)
copy = size;
if (bytes)
{
bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
bytes += copy;
}
size -= copy;
vars->bufferOffset += copy;
// vars->position += copy;
if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
{
if (!vars->pollers->io) cryptvars = 0;
err = IOPolledFilePollersIODone(vars->pollers, true);
if (kIOReturnSuccess != err)
break;
if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
vars->position += vars->lastRead;
vars->extentRemaining -= vars->lastRead;
vars->bufferLimit = vars->lastRead;
if (!vars->extentRemaining)
{
vars->currentExtent++;
vars->extentRemaining = vars->currentExtent->length;
vars->extentPosition = vars->position;
if (!vars->extentRemaining)
{
err = kIOReturnOverrun;
break;
}
}
uint64_t length;
uint64_t lastReadLength = vars->lastRead;
uint64_t offset = (vars->position
- vars->extentPosition + vars->currentExtent->start);
if (vars->extentRemaining <= vars->bufferSize)
length = vars->extentRemaining;
else
length = vars->bufferSize;
if ((length + vars->position) > vars->readEnd)
length = vars->readEnd - vars->position;
vars->lastRead = length;
if (length)
{
//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
err = IOStartPolledIO(vars->pollers, kIOPolledRead, vars->bufferHalf, offset, length);
if (kIOReturnSuccess != err)
break;
vars->pollers->io = true;
}
vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
vars->bufferOffset = 0;
#if CRYPTO
if (cryptvars)
{
uint8_t thisVector[AES_BLOCK_SIZE];
AbsoluteTime startTime, endTime;
// save initial vector for following decrypts
bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
&cryptvars->aes_iv[0], AES_BLOCK_SIZE);
// decrypt the buffer
clock_get_uptime(&startTime);
aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
&thisVector[0],
lastReadLength / AES_BLOCK_SIZE,
vars->buffer + vars->bufferHalf,
&cryptvars->ctx.decrypt);
clock_get_uptime(&endTime);
ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
vars->cryptBytes += lastReadLength;
}
#endif /* CRYPTO */
}
}
while (size);
return (err);
}
IOReturn
IOPolledFileWrite(IOPolledFileIOVars * vars,
const uint8_t * bytes, IOByteCount size,
IOPolledFileCryptVars * cryptvars)
{
IOReturn err = kIOReturnSuccess;
IOByteCount copy;
bool flush = false;
do
{
if (!bytes && !size)
{
// seek to end of block & flush
size = vars->position & (vars->blockSize - 1);
if (size)
size = vars->blockSize - size;
flush = true;
// use some garbage for the fill
bytes = vars->buffer + vars->bufferOffset;
}
copy = vars->bufferLimit - vars->bufferOffset;
if (copy > size)
copy = size;
else
flush = true;
if (bytes)
{
bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
bytes += copy;
}
else
bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
size -= copy;
vars->bufferOffset += copy;
vars->position += copy;
if (flush && vars->bufferOffset)
{
uint64_t offset = (vars->position - vars->bufferOffset
- vars->extentPosition + vars->currentExtent->start);
uint32_t length = (vars->bufferOffset);
#if CRYPTO
if (cryptvars && vars->encryptStart
&& (vars->position > vars->encryptStart)
&& ((vars->position - length) < vars->encryptEnd))
{
AbsoluteTime startTime, endTime;
uint64_t encryptLen, encryptStart;
encryptLen = vars->position - vars->encryptStart;
if (encryptLen > length)
encryptLen = length;
encryptStart = length - encryptLen;
if (vars->position > vars->encryptEnd)
encryptLen -= (vars->position - vars->encryptEnd);
clock_get_uptime(&startTime);
// encrypt the buffer
aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
&cryptvars->aes_iv[0],
encryptLen / AES_BLOCK_SIZE,
vars->buffer + vars->bufferHalf + encryptStart,
&cryptvars->ctx.encrypt);
clock_get_uptime(&endTime);
ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
vars->cryptBytes += encryptLen;
// save initial vector for following encrypts
bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
&cryptvars->aes_iv[0],
AES_BLOCK_SIZE);
}
#endif /* CRYPTO */
err = IOPolledFilePollersIODone(vars->pollers, true);
if (kIOReturnSuccess != err)
break;
if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length);
if (kIOReturnSuccess != err)
break;
vars->pollers->io = true;
vars->extentRemaining -= vars->bufferOffset;
if (!vars->extentRemaining)
{
vars->currentExtent++;
vars->extentRemaining = vars->currentExtent->length;
vars->extentPosition = vars->position;
}
vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
vars->bufferOffset = 0;
if (vars->bufferSize <= vars->extentRemaining)
vars->bufferLimit = vars->bufferSize;
else
vars->bufferLimit = vars->extentRemaining;
if (!vars->extentRemaining)
{
err = kIOReturnOverrun;
break;
}
flush = false;
}
}
while (size);
return (err);
}
IOReturn
IOPolledFileOpen(const char * filename,
uint64_t setFileSize, uint64_t fsFreeSize,
void * write_file_addr, size_t write_file_len,
IOPolledFileIOVars ** fileVars,
OSData ** imagePath,
uint8_t * volumeCryptKey, size_t keySize)
{
IOReturn err = kIOReturnSuccess;
IOPolledFileIOVars * vars;
_OpenFileContext ctx;
OSData * extentsData;
OSNumber * num;
IOService * part = 0;
dev_t block_dev;
dev_t image_dev;
AbsoluteTime startTime, endTime;
uint64_t nsec;
vars = IONew(IOPolledFileIOVars, 1);
if (!vars) return (kIOReturnNoMemory);
bzero(vars, sizeof(*vars));
vars->allocated = true;
do
{
extentsData = OSData::withCapacity(32);
ctx.extents = extentsData;
ctx.size = 0;
clock_get_uptime(&startTime);
vars->fileRef = kern_open_file_for_direct_io(filename,
(write_file_addr != NULL) || (0 != setFileSize),
&file_extent_callback, &ctx,
setFileSize,
fsFreeSize,
// write file:
0, write_file_addr, write_file_len,
// results
&block_dev,
&image_dev,
&vars->block0,
&vars->maxiobytes,
&vars->flags);
#if 0
uint32_t msDelay = (131071 & random());
HIBLOG("sleep %d\n", msDelay);
IOSleep(msDelay);
#endif
clock_get_uptime(&endTime);
SUB_ABSOLUTETIME(&endTime, &startTime);
absolutetime_to_nanoseconds(endTime, &nsec);
if (!vars->fileRef) err = kIOReturnNoSpace;
HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
if (kIOReturnSuccess != err) break;
HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size,
(extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
vars->maxiobytes, kIOPolledFileSSD & vars->flags);
assert(!vars->block0);
if (extentsData->getLength() < sizeof(IOPolledFileExtent))
{
err = kIOReturnNoSpace;
break;
}
vars->fileSize = ctx.size;
vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
part = IOCopyMediaForDev(image_dev);
if (!part)
{
err = kIOReturnNotFound;
break;
}
if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) break;
if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey))))
vars->blockSize = num->unsigned32BitValue();
if (vars->blockSize < 4096) vars->blockSize = 4096;
HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
major(image_dev), minor(image_dev), (long)vars->blockSize,
vars->pollers->pollers->getCount());
OSString * keyUUID = NULL;
if (volumeCryptKey)
{
err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
}
*fileVars = vars;
vars->fileExtents = extentsData;
// make imagePath
OSData * data;
if (imagePath)
{
#if defined(__i386__) || defined(__x86_64__)
char str2[24 + sizeof(uuid_string_t) + 2];
if (keyUUID)
snprintf(str2, sizeof(str2), "%qx:%s",
vars->extentMap[0].start, keyUUID->getCStringNoCopy());
else
snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
err = IOService::getPlatform()->callPlatformFunction(
gIOCreateEFIDevicePathSymbol, false,
(void *) part, (void *) str2,
(void *) (uintptr_t) true, (void *) &data);
#else
data = 0;
err = kIOReturnSuccess;
#endif
if (kIOReturnSuccess != err)
{
HIBLOG("error 0x%x getting path\n", err);
break;
}
*imagePath = data;
}
}
while (false);
if (kIOReturnSuccess != err)
{
HIBLOG("error 0x%x opening polled file\n", err);
IOPolledFileClose(&vars, 0, 0, 0, 0, 0);
}
if (part) part->release();
return (err);
}
