/**
* Resets the state of the task queue.
*
* This does not remove any tasks from the queue. It just resets the task queue
* such that all tasks are ready to be run again.
*/
BOOL
MPResetTaskQ(virt_ptr<MPTaskQueue> queue)
{
OSUninterruptibleSpinLock_Acquire(virt_addrof(queue->lock));
if (queue->state != MPTaskQueueState::Finished &&
queue->state != MPTaskQueueState::Stopped) {
OSUninterruptibleSpinLock_Release(virt_addrof(queue->lock));
return FALSE;
}
queue->state = MPTaskQueueState::Initialised;
queue->tasks = queue->queueSize;
queue->tasksReady = queue->queueSize;
queue->tasksRunning = 0u;
queue->tasksFinished = 0u;
queue->queueIndex = 0u;
for (auto i = 0u; i < queue->tasks; ++i) {
auto task = queue->queue[i];
task->result = 0u;
task->coreID = 3u;
task->duration = 0u;
task->state = MPTaskState::Ready;
}
OSUninterruptibleSpinLock_Release(virt_addrof(queue->lock));
return TRUE;
}
uint64_t
DMAECopyMem(virt_ptr<void> dst,
virt_ptr<void> src,
uint32_t numWords,
DMAEEndianSwapMode endian)
{
coreinit::OSLockMutex(virt_addrof(sRingData->mutex));
if (endian == DMAEEndianSwapMode::None) {
std::memcpy(dst.get(),
src.get(),
numWords * 4);
} else if (endian == DMAEEndianSwapMode::Swap8In16) {
auto dstWords = reinterpret_cast<uint16_t *>(dst.get());
auto srcWords = reinterpret_cast<uint16_t *>(src.get());
for (auto i = 0u; i < numWords * 2; ++i) {
*dstWords++ = byte_swap(*srcWords++);
}
} else if (endian == DMAEEndianSwapMode::Swap8In32) {
auto dstDwords = reinterpret_cast<uint32_t *>(dst.get());
auto srcDwords = reinterpret_cast<uint32_t *>(src.get());
for (auto i = 0u; i < numWords; ++i) {
*dstDwords++ = byte_swap(*srcDwords++);
}
}
auto timestamp = coreinit::OSGetTime();
sRingData->lastSubmittedTimestamp = timestamp;
coreinit::OSUnlockMutex(virt_addrof(sRingData->mutex));
return timestamp;
}
void
MEMFreeToExpHeap(MEMHeapHandle handle,
virt_ptr<void> mem)
{
auto heap = virt_cast<MEMExpHeap *>(handle);
decaf_check(heap->header.tag == MEMHeapTag::ExpandedHeap);
if (!mem) {
return;
}
internal::HeapLock lock { virt_addrof(heap->header) };
// Find the block
auto dataStart = virt_cast<uint8_t *>(mem);
auto block = virt_cast<MEMExpHeapBlock *>(dataStart - sizeof(MEMExpHeapBlock));
// Get the bounding region for this block
auto memStart = getBlockMemStart(block);
auto memEnd = getBlockMemEnd(block);
// Remove the block from the used list
removeBlock(virt_addrof(heap->usedList), block);
// Release the memory back to the heap free list
releaseMemory(heap, memStart, memEnd);
}
void
OSSendFatalError(virt_ptr<OSFatalError> error,
virt_ptr<const char> functionName,
uint32_t line)
{
if (error) {
if (functionName) {
string_copy(virt_addrof(error->functionName).get(),
error->functionName.size(),
functionName.get(),
error->functionName.size());
error->functionName[error->functionName.size() - 1] = char { 0 };
} else {
error->functionName[0] = char { 0 };
}
error->line = line;
}
// TODO: Kernel call 0x6C00 systemFatal
gLog->error("SystemFatal: messageType: {}", error->messageType);
gLog->error("SystemFatal: errorCode: {}", error->errorCode);
gLog->error("SystemFatal: internalErrorCode: {}", error->internalErrorCode);
gLog->error("SystemFatal: processId: {}", error->processId);
gLog->error("SystemFatal: functionName: {}", virt_addrof(error->functionName));
gLog->error("SystemFatal: line: {}", error->line);
ghs_exit(-1);
}
static void
lockMutexNoLock(virt_ptr<OSMutex> mutex)
{
auto thread = OSGetCurrentThread();
decaf_check(thread->state == OSThreadState::Running);
while (mutex->owner) {
if (mutex->owner == thread) {
mutex->count++;
return;
}
// Mark this thread as waiting on the mutex
thread->mutex = mutex;
// Promote mutex owner priority
internal::promoteThreadPriorityNoLock(mutex->owner, thread->priority);
// Wait for other owner to unlock
internal::sleepThreadNoLock(virt_addrof(mutex->queue));
internal::rescheduleSelfNoLock();
// We are no longer waiting on the mutex
thread->mutex = nullptr;
}
// Set current thread to owner of mutex
mutex->count++;
mutex->owner = thread;
MutexQueue::append(virt_addrof(thread->mutexQueue), mutex);
thread->cancelState |= OSThreadCancelState::DisabledByMutex;
}
DMAETimestamp
DMAEGetLastSubmittedTimeStamp()
{
coreinit::OSLockMutex(virt_addrof(sRingData->mutex));
auto timestamp = sRingData->lastSubmittedTimestamp;
coreinit::OSUnlockMutex(virt_addrof(sRingData->mutex));
return timestamp;
}
void
initialiseExceptionContext(cpu::Core *core)
{
auto context = virt_addrof(sExceptionData->exceptionThreadContext[core->id]);
std::memset(context.get(), 0, sizeof(Context));
auto stack = virt_addrof(sExceptionData->exceptionStackBuffer[core->id * ExceptionThreadStackSize]);
context->gpr[1] = virt_cast<virt_addr>(stack).getAddress() + ExceptionThreadStackSize - 8;
context->attr |= 1 << core->id;
setContextFiberEntry(context, exceptionContextFiberEntry, nullptr);
}
/**
* Get the status of a task queue.
*/
BOOL
MPGetTaskQInfo(virt_ptr<MPTaskQueue> queue,
virt_ptr<MPTaskQueueInfo> info)
{
OSUninterruptibleSpinLock_Acquire(virt_addrof(queue->lock));
info->state = queue->state;
info->tasks = queue->tasks;
info->tasksReady = queue->tasksReady;
info->tasksRunning = queue->tasksRunning;
info->tasksFinished = queue->tasksFinished;
OSUninterruptibleSpinLock_Release(virt_addrof(queue->lock));
return TRUE;
}
ios::Error
ipckDriverAllocateRequestBlock(RamPartitionId clientProcessId,
RamPartitionId loaderProcessId,
virt_ptr<IPCKDriver> driver,
virt_ptr<IPCKDriverRequestBlock> *outRequestBlock,
ios::Handle handle,
ios::Command command,
IPCKDriverHostAsyncCallbackFn asyncCallback,
virt_ptr<void> asyncCallbackData)
{
auto error = IPCKDriver_FIFOPop(virt_addrof(driver->freeFifo), outRequestBlock);
if (error < ios::Error::OK) {
if (error == ios::Error::QEmpty) {
return ios::Error::QFull;
}
return error;
}
auto requestBlock = *outRequestBlock;
requestBlock->flags = requestBlock->flags.value()
.unk_0x0C00(0)
.replyState(IPCKDriverRequestBlock::WaitReply)
.requestState(IPCKDriverRequestBlock::Allocated)
.clientProcessId(clientProcessId)
.loaderProcessId(loaderProcessId);
requestBlock->asyncCallback->func = asyncCallback;
requestBlock->asyncCallbackData = asyncCallbackData;
requestBlock->userRequest = nullptr;
auto request = requestBlock->request;
std::memset(virt_addrof(request->request.args).get(),
0,
sizeof(request->request.args));
request->request.clientPid = static_cast<int32_t>(clientProcessId);
request->request.handle = handle;
request->request.command = command;
request->request.flags = 0u;
request->request.reply = ios::Error::OK;
request->request.cpuId = static_cast<ios::CpuId>(cpu::this_core::id() + 1);
if (clientProcessId != RamPartitionId::Kernel) {
request->request.titleId = getCurrentTitleId();
}
request->prevHandle = handle;
request->prevCommand = command;
return ios::Error::OK;
}
/**
* Unlocks the mutex.
*
* Will decrease the recursion count, will only unlock the mutex when the
* recursion count reaches 0.
* If any other threads are waiting to lock the mutex they will be woken.
*
* Similar to <a href="http://en.cppreference.com/w/cpp/thread/recursive_mutex/unlock">std::recursive_mutex::unlock</a>.
*/
void
OSUnlockMutex(virt_ptr<OSMutex> mutex)
{
internal::lockScheduler();
auto thread = OSGetCurrentThread();
decaf_check(thread->state == OSThreadState::Running);
// Not the owner, ignore this call.
if (mutex->owner != thread) {
internal::unlockScheduler();
return;
}
// Decrement the mutexes lock count
mutex->count--;
// If we still own the mutex, lets just leave now
if (mutex->count > 0) {
internal::unlockScheduler();
return;
}
// Remove mutex from thread's mutex queue
MutexQueue::erase(virt_addrof(thread->mutexQueue), mutex);
// Clear the mutex owner
mutex->owner = nullptr;
// If we have a promoted priority, reset it.
if (thread->priority < thread->basePriority) {
thread->priority = internal::calculateThreadPriorityNoLock(thread);
}
// Clear the cancelState flag if we don't have any more mutexes locked
if (!thread->mutexQueue.head) {
thread->cancelState &= ~OSThreadCancelState::DisabledByMutex;
}
// Wakeup any threads trying to lock this mutex
internal::wakeupThreadNoLock(virt_addrof(mutex->queue));
// Check if we are meant to cancel now
internal::testThreadCancelNoLock();
// Reschedule everyone
internal::rescheduleAllCoreNoLock();
// Unlock our scheduler and continue
internal::unlockScheduler();
}
/**
* Starts a task queue.
*
* Sets the task state to Ready.
*
* \return Returns true if state was previously Initialised or Stopped.
*/
BOOL
MPStartTaskQ(virt_ptr<MPTaskQueue> queue)
{
OSUninterruptibleSpinLock_Acquire(virt_addrof(queue->lock));
if (queue->state == MPTaskQueueState::Initialised
|| queue->state == MPTaskQueueState::Stopped) {
queue->state = MPTaskQueueState::Ready;
OSUninterruptibleSpinLock_Release(virt_addrof(queue->lock));
return TRUE;
}
OSUninterruptibleSpinLock_Release(virt_addrof(queue->lock));
return FALSE;
}
void
unlockAllMutexNoLock(virt_ptr<OSThread> thread)
{
while (auto mutex = thread->mutexQueue.head) {
// Remove this mutex from our queue
MutexQueue::erase(virt_addrof(thread->mutexQueue), mutex);
// Release this mutex
mutex->count = 0;
mutex->owner = nullptr;
// Wakeup any threads trying to lock this mutex
internal::wakeupThreadNoLock(virt_addrof(mutex->queue));
}
}
/**
* Try to lock a mutex.
*
* If no one owns the mutex, set current thread as owner.
* If the lock is owned by the current thread, increase the recursion count.
* If the lock is owned by another thread, do not block, return FALSE.
*
* \return TRUE if the mutex is locked, FALSE if the mutex is owned by another thread.
*
* Similar to <a href="http://en.cppreference.com/w/cpp/thread/recursive_mutex/try_lock">std::recursive_mutex::try_lock</a>.
*/
BOOL
OSTryLockMutex(virt_ptr<OSMutex> mutex)
{
internal::lockScheduler();
auto thread = OSGetCurrentThread();
decaf_check(thread->state == OSThreadState::Running);
internal::testThreadCancelNoLock();
if (mutex->owner == thread) {
mutex->count++;
internal::unlockScheduler();
return TRUE;
} else if (mutex->owner) {
internal::unlockScheduler();
return FALSE;
}
// Set thread to owner of mutex
mutex->count++;
mutex->owner = thread;
MutexQueue::append(virt_addrof(thread->mutexQueue), mutex);
thread->cancelState |= OSThreadCancelState::DisabledByMutex;
internal::unlockScheduler();
return TRUE;
}
/**
* Dequeue 1 task at queueIndex
*
* Does not remove tasks from queue buffer.
*
* \return Returns dequeued task.
*/
virt_ptr<MPTask>
MPDequeTask(virt_ptr<MPTaskQueue> queue)
{
OSUninterruptibleSpinLock_Acquire(virt_addrof(queue->lock));
if (queue->state != MPTaskQueueState::Ready
|| queue->queueIndex == queue->queueSize) {
OSUninterruptibleSpinLock_Release(virt_addrof(queue->lock));
return nullptr;
}
auto task = queue->queue[queue->queueIndex];
queue->queueIndex++;
OSUninterruptibleSpinLock_Release(virt_addrof(queue->lock));
return task;
}
void
dumpExpandedHeap(virt_ptr<MEMExpHeap> heap)
{
internal::HeapLock lock { virt_addrof(heap->header) };
gLog->debug("MEMExpHeap(0x{:8x})", heap);
gLog->debug("Status Address Size Group");
for (auto block = heap->freeList.head; block; block = block->next) {
auto attribs = static_cast<MEMExpHeapBlockAttribs>(block->attribs);
gLog->debug("FREE 0x{:8x} 0x{:8x} {:d}",
block,
static_cast<uint32_t>(block->blockSize),
attribs.groupId());
}
for (auto block = heap->usedList.head; block; block = block->next) {
auto attribs = static_cast<MEMExpHeapBlockAttribs>(block->attribs);
gLog->debug("USED 0x{:8x} 0x{:8x} {:d}",
block,
static_cast<uint32_t>(block->blockSize),
attribs.groupId());
}
}
uint16_t
MEMGetGroupIDForExpHeap(MEMHeapHandle handle)
{
auto heap = virt_cast<MEMExpHeap *>(handle);
internal::HeapLock lock { virt_addrof(heap->header) };
return heap->groupId;
}
uint32_t
MEMGetAllocatableSizeForExpHeapEx(MEMHeapHandle handle,
int32_t alignment)
{
auto heap = virt_cast<MEMExpHeap *>(handle);
auto largestFree = 0u;
internal::HeapLock lock { virt_addrof(heap->header) };
if (alignment > 0) {
decaf_check((alignment & 0x3) == 0);
for (auto block = heap->freeList.head; block; block = block->next) {
auto alignedSize = getAlignedBlockSize(block, alignment, MEMExpHeapDirection::FromStart);
if (alignedSize > largestFree) {
largestFree = alignedSize;
}
}
} else {
alignment = -alignment;
decaf_check((alignment & 0x3) == 0);
for (auto block = heap->freeList.head; block; block = block->next) {
auto alignedSize = getAlignedBlockSize(block, alignment, MEMExpHeapDirection::FromEnd);
if (alignedSize > largestFree) {
largestFree = alignedSize;
}
}
}
return largestFree;
}
uint32_t
MEMAdjustExpHeap(MEMHeapHandle handle)
{
auto heap = virt_cast<MEMExpHeap *>(handle);
internal::HeapLock lock { virt_addrof(heap->header) };
auto lastFreeBlock = heap->freeList.tail;
if (!lastFreeBlock) {
return 0;
}
auto blockData = virt_cast<uint8_t *>(lastFreeBlock) + sizeof(MEMExpHeapBlock);
if (blockData + lastFreeBlock->blockSize != heap->header.dataEnd) {
// This block is not for the end of the heap
return 0;
}
// Remove the block from the free list
decaf_check(!lastFreeBlock->next);
if (lastFreeBlock->prev) {
lastFreeBlock->prev->next = nullptr;
}
// Move the heaps end pointer to the true start point of this block
heap->header.dataEnd = getBlockMemStart(lastFreeBlock);
auto heapMemStart = virt_cast<uint8_t *>(heap);
auto heapMemEnd = virt_cast<uint8_t *>(heap->header.dataEnd);
return static_cast<uint32_t>(heapMemEnd - heapMemStart);
}
static ios::Error
defensiveProcessIncomingMessagePointer(virt_ptr<IPCKDriver> driver,
virt_ptr<IPCKDriverRequest> request,
virt_ptr<IPCKDriverRequestBlock> *outRequestBlock)
{
auto index = request - driver->requestsBuffer;
if (index >= IPCKRequestsPerCore || index < 0) {
return ios::Error::Invalid;
}
if (driver->requestBlocks[index].request != request) {
return ios::Error::Invalid;
}
auto requestBlock = virt_addrof(driver->requestBlocks[index]);
auto flags = requestBlock->flags.value();
if (flags.requestState() == IPCKDriverRequestBlock::Unallocated) {
return ios::Error::Invalid;
}
requestBlock->flags =
flags.replyState(IPCKDriverRequestBlock::ReceivedReply);
*outRequestBlock = requestBlock;
return ios::Error::OK;
}
/**
* Run a specific task.
*
* The task must belong to a queue.
* The task must be in the Ready state.
*
* \return Returns TRUE if task was run.
*/
BOOL
MPRunTask(virt_ptr<MPTask> task)
{
auto queue = task->queue;
if (task->state != MPTaskState::Ready) {
return FALSE;
}
if (!queue
|| queue->state == MPTaskQueueState::Stopping
|| queue->state == MPTaskQueueState::Stopped) {
return FALSE;
}
OSUninterruptibleSpinLock_Acquire(virt_addrof(queue->lock));
queue->tasksReady--;
queue->tasksRunning++;
OSUninterruptibleSpinLock_Release(virt_addrof(queue->lock));
task->state = MPTaskState::Running;
task->coreID = OSGetCoreId();
auto start = OSGetTime();
task->result = cafe::invoke(cpu::this_core::state(),
task->func,
task->userArg1,
task->userArg2);
task->duration = OSGetTime() - start;
task->state = MPTaskState::Finished;
OSUninterruptibleSpinLock_Acquire(virt_addrof(queue->lock));
queue->tasksRunning--;
queue->tasksFinished++;
if (queue->state == MPTaskQueueState::Stopping && queue->tasksRunning == 0) {
queue->state = MPTaskQueueState::Stopped;
}
if (queue->tasks == queue->tasksFinished) {
queue->state = MPTaskQueueState::Finished;
}
OSUninterruptibleSpinLock_Release(virt_addrof(queue->lock));
return TRUE;
}
/**
* Initialise a condition variable structure with a name.
*/
void
OSInitCondEx(virt_ptr<OSCondition> condition,
virt_ptr<const char> name)
{
condition->tag = OSCondition::Tag;
condition->name = name;
OSInitThreadQueueEx(virt_addrof(condition->queue), condition);
}
MEMExpHeapMode
MEMGetAllocModeForExpHeap(MEMHeapHandle handle)
{
auto heap = virt_cast<MEMExpHeap *>(handle);
internal::HeapLock lock { virt_addrof(heap->header) };
auto expHeapAttribs = heap->attribs.value();
return expHeapAttribs.allocMode();
}
virt_ptr<void>
MEMDestroyExpHeap(MEMHeapHandle handle)
{
auto heap = virt_cast<MEMExpHeap *>(handle);
decaf_check(heap);
decaf_check(heap->header.tag == MEMHeapTag::ExpandedHeap);
internal::unregisterHeap(virt_addrof(heap->header));
return heap;
}
uint16_t
MEMSetGroupIDForExpHeap(MEMHeapHandle handle,
uint16_t id)
{
auto heap = virt_cast<MEMExpHeap *>(handle);
internal::HeapLock lock { virt_addrof(heap->header) };
auto originalGroupId = heap->groupId;
heap->groupId = id;
return originalGroupId;
}
uint64_t
DMAEFillMem(virt_ptr<void> dst,
uint32_t value,
uint32_t numDwords)
{
coreinit::OSLockMutex(virt_addrof(sRingData->mutex));
auto dstValue = byte_swap(value);
auto dstDwords = reinterpret_cast<uint32_t *>(dst.get());
for (auto i = 0u; i < numDwords; ++i) {
dstDwords[i] = dstValue;
}
auto timestamp = coreinit::OSGetTime();
sRingData->lastSubmittedTimestamp = timestamp;
coreinit::OSUnlockMutex(virt_addrof(sRingData->mutex));
return timestamp;
}
/**
* Stops a task queue.
*
* If there are tasks running the state is set to Stopping.
* If there are no tasks running the state is set to Stopped.
*
* \return Returns FALSE if the task queue is not in the Ready state.
*/
BOOL
MPStopTaskQ(virt_ptr<MPTaskQueue> queue)
{
OSUninterruptibleSpinLock_Acquire(virt_addrof(queue->lock));
if (queue->state != MPTaskQueueState::Ready) {
OSUninterruptibleSpinLock_Release(virt_addrof(queue->lock));
return FALSE;
}
if (queue->tasksRunning == 0) {
queue->state = MPTaskQueueState::Stopped;
} else {
queue->state = MPTaskQueueState::Stopping;
}
OSUninterruptibleSpinLock_Release(virt_addrof(queue->lock));
return TRUE;
}
/**
* Initialise a mutex structure with a name.
*/
void
OSInitMutexEx(virt_ptr<OSMutex> mutex,
virt_ptr<const char> name)
{
mutex->tag = OSMutex::Tag;
mutex->name = name;
mutex->owner = nullptr;
mutex->count = 0;
OSInitThreadQueueEx(virt_addrof(mutex->queue), mutex);
MutexQueue::initLink(mutex);
}
ios::Error
ipckDriverOpen()
{
auto driver = ipckDriverGetInstance();
if (driver->state != IPCKDriverState::Initialised &&
driver->state != IPCKDriverState::Unknown1) {
return ios::Error::NotReady;
}
auto error = initialiseResourceBuffers(driver);
if (error < ios::Error::OK) {
return error;
}
IPCKDriver_FIFOInit(virt_addrof(driver->freeFifo));
IPCKDriver_FIFOInit(virt_addrof(driver->outboundFIFO));
for (auto i = 0u; i < NumRamPartitions; ++i) {
IPCKDriver_FIFOInit(virt_addrof(driver->perProcessUserReply[i]));
IPCKDriver_FIFOInit(virt_addrof(driver->perProcessLoaderReply[i]));
}
for (auto i = 0u; i < IPCKRequestsPerCore; ++i) {
IPCKDriver_FIFOPush(virt_addrof(driver->freeFifo),
virt_addrof(driver->requestBlocks[i]));
}
driver->unk0x04++;
setKernelInterruptHandler(driver->interruptType,
ipckDriverHandleInterrupt);
driver->state = IPCKDriverState::Open;
return ios::Error::OK;
}
uint32_t
MEMGetTotalFreeSizeForExpHeap(MEMHeapHandle handle)
{
auto heap = virt_cast<MEMExpHeap *>(handle);
auto freeSize = 0u;
internal::HeapLock lock { virt_addrof(heap->header) };
for (auto block = heap->freeList.head; block; block = block->next) {
freeSize += block->blockSize;
}
return freeSize;
}
static BSPError
prepareIpcBuffer(std::size_t responseSize,
virt_ptr<BSPIpcBuffer> buffer)
{
if (responseSize > sizeof(ios::bsp::BSPResponse)) {
return BSPError::ResponseTooLarge;
}
std::memset(virt_addrof(buffer->request).get(),
0,
sizeof(BSPRequest));
return BSPError::OK;
}
