gceSTATUS gckKERNEL_SecurityExecute( IN gckKERNEL Kernel, IN gctPOINTER Buffer, IN gctUINT32 Bytes ) { gceSTATUS status; gcsTA_INTERFACE iface; gcmkHEADER(); iface.command = KERNEL_EXECUTE; iface.u.Execute.command_buffer = (gctUINT32 *)Buffer; iface.u.Execute.gpu = Kernel->core; iface.u.Execute.command_buffer_length = Bytes; #if defined(LINUX) gcmkONERROR(gckOS_GetPhysicalAddress(Kernel->os, Buffer, (gctUINT32 *)&iface.u.Execute.command_buffer)); #endif gcmkONERROR(gckKERNEL_SecurityCallService(Kernel->securityChannel, &iface)); /* Update queue tail pointer. */ gcmkONERROR(gckHARDWARE_UpdateQueueTail( Kernel->hardware, 0, 0 )); gcmkFOOTER_NO(); return gcvSTATUS_OK; OnError: gcmkFOOTER(); return status; }
/******************************************************************************* ** ** gckCOMMAND_Commit ** ** Commit a command buffer to the command queue. ** ** INPUT: ** ** gckCOMMAND Command ** Pointer to an gckCOMMAND object. ** ** gcoCMDBUF CommandBuffer ** Pointer to an gcoCMDBUF object. ** ** gcoCONTEXT Context ** Pointer to an gcoCONTEXT object. ** ** OUTPUT: ** ** Nothing. */ gceSTATUS gckCOMMAND_Commit( IN gckCOMMAND Command, IN gcoCMDBUF CommandBuffer, IN gcoCONTEXT Context, IN gctHANDLE Process ) { gcoCMDBUF commandBuffer; gcoCONTEXT context; gckHARDWARE hardware; gceSTATUS status; gctPOINTER initialLink, link; gctSIZE_T bytes, initialSize, lastRun; gcoCMDBUF buffer; gctPOINTER wait; gctSIZE_T waitSize; gctUINT32 offset; gctPOINTER fetchAddress; gctSIZE_T fetchSize; gctUINT8_PTR logical; gcsMAPPED_PTR stack = gcvNULL; gctINT acquired = 0; #if gcdSECURE_USER gctUINT32_PTR hint; #endif #if gcdDUMP_COMMAND gctPOINTER dataPointer; gctSIZE_T dataBytes; #endif gctPOINTER flushPointer; gctSIZE_T flushSize; gcmkHEADER_ARG("Command=0x%x CommandBuffer=0x%x Context=0x%x", Command, CommandBuffer, Context); /* Verify the arguments. */ gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); #if gcdNULL_DRIVER == 2 /* Do nothing with infinite hardware. */ gcmkFOOTER_NO(); return gcvSTATUS_OK; #endif gcmkONERROR( _AddMap(Command->os, CommandBuffer, gcmSIZEOF(struct _gcoCMDBUF), (gctPOINTER *) &commandBuffer, &stack)); gcmkVERIFY_OBJECT(commandBuffer, gcvOBJ_COMMANDBUFFER); gcmkONERROR( _AddMap(Command->os, Context, gcmSIZEOF(struct _gcoCONTEXT), (gctPOINTER *) &context, &stack)); gcmkVERIFY_OBJECT(context, gcvOBJ_CONTEXT); /* Extract the gckHARDWARE and gckEVENT objects. */ hardware = Command->kernel->hardware; gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); /* Acquire the context switching mutex. */ gcmkONERROR( gckOS_AcquireMutex(Command->os, Command->mutexContext, gcvINFINITE)); ++acquired; /* Reserved slot in the context or command buffer. */ gcmkONERROR( gckHARDWARE_PipeSelect(hardware, gcvNULL, 0, &bytes)); /* Test if we need to switch to this context. */ if ((context->id != 0) && (context->id != Command->currentContext) ) { /* Map the context buffer.*/ gcmkONERROR( _AddMap(Command->os, context->logical, context->bufferSize, (gctPOINTER *) &logical, &stack)); #if gcdSECURE_USER /* Map the hint array.*/ gcmkONERROR( _AddMap(Command->os, context->hintArray, context->hintCount * gcmSIZEOF(gctUINT32), (gctPOINTER *) &hint, &stack)); /* Loop while we have valid hints. */ while (*hint != 0) { /* Map handle into physical address. */ gcmkONERROR( gckKERNEL_MapLogicalToPhysical( Command->kernel, Process, (gctPOINTER *) (logical + *hint))); /* Next hint. */ ++hint; } #endif /* See if we have to check pipes. */ if (context->pipe2DIndex != 0) { /* See if we are in the correct pipe. */ if (context->initialPipe == Command->pipeSelect) { gctUINT32 reserved = bytes; gctUINT8_PTR nop = logical; /* Already in the correct pipe, fill context buffer with NOP. */ while (reserved > 0) { bytes = reserved; gcmkONERROR( gckHARDWARE_Nop(hardware, nop, &bytes)); gcmkASSERT(reserved >= bytes); reserved -= bytes; nop += bytes; } } else { /* Switch to the correct pipe. */ gcmkONERROR( gckHARDWARE_PipeSelect(hardware, logical, context->initialPipe, &bytes)); } } /* Save initial link pointer. */ initialLink = logical; initialSize = context->bufferSize; #if MRVL_PRINT_CMD_BUFFER _AddCmdBuffer( Command, initialLink, initialSize, gcvTRUE, gcvFALSE ); #endif /* Save initial buffer to flush. */ flushPointer = initialLink; flushSize = initialSize; /* Save pointer to next link. */ gcmkONERROR( _AddMap(Command->os, context->link, 8, &link, &stack)); /* Start parsing CommandBuffer. */ buffer = commandBuffer; /* Mark context buffer as used. */ if (context->inUse != gcvNULL) { gctBOOL_PTR inUse; gcmkONERROR( _AddMap(Command->os, (gctPOINTER) context->inUse, gcmSIZEOF(gctBOOL), (gctPOINTER *) &inUse, &stack)); *inUse = gcvTRUE; } } else { /* Test if this is a new context. */ if (context->id == 0) { /* Generate unique ID for the context buffer. */ context->id = ++ Command->contextCounter; if (context->id == 0) { /* Context counter overflow (wow!) */ gcmkONERROR(gcvSTATUS_TOO_COMPLEX); } } /* Map the command buffer. */ gcmkONERROR( _AddMap(Command->os, commandBuffer->logical, commandBuffer->offset, (gctPOINTER *) &logical, &stack)); #if gcdSECURE_USER /* Map the hint table. */ gcmkONERROR( _AddMap(Command->os, commandBuffer->hintCommit, commandBuffer->offset - commandBuffer->startOffset, (gctPOINTER *) &hint, &stack)); /* Walk while we have valid hints. */ while (*hint != 0) { /* Map the handle to a physical address. */ gcmkONERROR( gckKERNEL_MapLogicalToPhysical( Command->kernel, Process, (gctPOINTER *) (logical + *hint))); /* Next hint. */ ++hint; } #endif if (context->entryPipe == Command->pipeSelect) { gctUINT32 reserved = Command->reservedHead; gctUINT8_PTR nop = logical + commandBuffer->startOffset; /* Already in the correct pipe, fill context buffer with NOP. */ while (reserved > 0) { bytes = reserved; gcmkONERROR( gckHARDWARE_Nop(hardware, nop, &bytes)); gcmkASSERT(reserved >= bytes); reserved -= bytes; nop += bytes; } } else { /* Switch to the correct pipe. */ gcmkONERROR( gckHARDWARE_PipeSelect(hardware, logical + commandBuffer->startOffset, context->entryPipe, &bytes)); } /* Save initial link pointer. */ initialLink = logical + commandBuffer->startOffset; initialSize = commandBuffer->offset - commandBuffer->startOffset + Command->reservedTail; #if MRVL_PRINT_CMD_BUFFER _AddCmdBuffer( Command, initialLink, initialSize, gcvFALSE, gcvFALSE ); #endif /* Save initial buffer to flush. */ flushPointer = initialLink; flushSize = initialSize; /* Save pointer to next link. */ link = logical + commandBuffer->offset; /* No more data. */ buffer = gcvNULL; } #if MRVL_PRINT_CMD_BUFFER _AddLink(Command, Command->wait, initialLink); #endif #if gcdDUMP_COMMAND dataPointer = initialLink; dataBytes = initialSize; #endif /* Loop through all remaining command buffers. */ if (buffer != gcvNULL) { /* Map the command buffer. */ gcmkONERROR( _AddMap(Command->os, buffer->logical, buffer->offset + Command->reservedTail, (gctPOINTER *) &logical, &stack)); #if MRVL_PRINT_CMD_BUFFER _AddCmdBuffer( Command, (gctUINT32_PTR)logical, buffer->offset + Command->reservedTail, gcvFALSE, gcvFALSE ); #endif #if gcdSECURE_USER /* Map the hint table. */ gcmkONERROR( _AddMap(Command->os, buffer->hintCommit, buffer->offset - buffer->startOffset, (gctPOINTER *) &hint, &stack)); /* Walk while we have valid hints. */ while (*hint != 0) { /* Map the handle to a physical address. */ gcmkONERROR( gckKERNEL_MapLogicalToPhysical( Command->kernel, Process, (gctPOINTER *) (logical + *hint))); /* Next hint. */ ++hint; } #endif /* First slot becomes a NOP. */ { gctUINT32 reserved = Command->reservedHead; gctUINT8_PTR nop = logical + buffer->startOffset; /* Already in the correct pipe, fill context buffer with NOP. */ while (reserved > 0) { bytes = reserved; gcmkONERROR( gckHARDWARE_Nop(hardware, nop, &bytes)); gcmkASSERT(reserved >= bytes); reserved -= bytes; nop += bytes; } } /* Generate the LINK to this command buffer. */ gcmkONERROR( gckHARDWARE_Link(hardware, link, logical + buffer->startOffset, buffer->offset - buffer->startOffset + Command->reservedTail, &bytes)); #if MRVL_PRINT_CMD_BUFFER _AddLink(Command, link, (gctUINT32_PTR)logical); #endif /* Flush the initial buffer. */ gcmkONERROR(gckOS_CacheFlush(Command->os, Process, flushPointer, flushSize)); /* Save new flush pointer. */ flushPointer = logical + buffer->startOffset; flushSize = buffer->offset - buffer->startOffset + Command->reservedTail; #if gcdDUMP_COMMAND _DumpCommand(Command, dataPointer, dataBytes); dataPointer = logical + buffer->startOffset; dataBytes = buffer->offset - buffer->startOffset + Command->reservedTail; #endif /* Save pointer to next link. */ link = logical + buffer->offset; } /* Compute number of bytes required for WAIT/LINK. */ gcmkONERROR( gckHARDWARE_WaitLink(hardware, gcvNULL, Command->offset, &bytes, gcvNULL, gcvNULL)); lastRun = bytes; /* Grab the command queue mutex. */ gcmkONERROR( gckOS_AcquireMutex(Command->os, Command->mutexQueue, gcvINFINITE)); ++acquired; if (Command->kernel->notifyIdle) { /* Increase the commit stamp */ Command->commitStamp++; /* Set busy if idle */ if (Command->idle) { Command->idle = gcvFALSE; gcmkVERIFY_OK(gckOS_NotifyIdle(Command->os, gcvFALSE)); } } /* Compute number of bytes left in current command queue. */ bytes = Command->pageSize - Command->offset; if (bytes < lastRun) { /* Create a new command queue. */ gcmkONERROR(_NewQueue(Command, gcvTRUE)); /* Adjust run size with any extra commands inserted. */ lastRun += Command->offset; } /* Get current offset. */ offset = Command->offset; /* Append WAIT/LINK in command queue. */ bytes = Command->pageSize - offset; gcmkONERROR( gckHARDWARE_WaitLink(hardware, (gctUINT8 *) Command->logical + offset, offset, &bytes, &wait, &waitSize)); /* Flush the cache for the wait/link. */ gcmkONERROR(gckOS_CacheFlush(Command->os, gcvNULL, (gctUINT8 *) Command->logical + offset, bytes)); #if gcdDUMP_COMMAND _DumpCommand(Command, (gctUINT8 *) Command->logical + offset, bytes); #endif /* Adjust offset. */ offset += bytes; if (Command->newQueue) { /* Compute fetch location and size for a new command queue. */ fetchAddress = Command->logical; fetchSize = offset; } else { /* Compute fetch location and size for an existing command queue. */ fetchAddress = (gctUINT8 *) Command->logical + Command->offset; fetchSize = offset - Command->offset; } bytes = 8; /* Link in WAIT/LINK. */ gcmkONERROR( gckHARDWARE_Link(hardware, link, fetchAddress, fetchSize, &bytes)); #if MRVL_PRINT_CMD_BUFFER _AddLink(Command, link, fetchAddress); #endif /* Flush the cache for the command buffer. */ gcmkONERROR(gckOS_CacheFlush(Command->os, Process, flushPointer, flushSize)); #if gcdDUMP_COMMAND _DumpCommand(Command, dataPointer, dataBytes); #endif /* Execute the entire sequence. */ gcmkONERROR( gckHARDWARE_Link(hardware, Command->wait, initialLink, initialSize, &Command->waitSize)); /* Flush the cache for the link. */ gcmkONERROR(gckOS_CacheFlush(Command->os, gcvNULL, Command->wait, Command->waitSize)); #if gcdDUMP_COMMAND _DumpCommand(Command, Command->wait, Command->waitSize); #endif /* Update command queue offset. */ Command->offset = offset; Command->newQueue = gcvFALSE; /* Update address of last WAIT. */ Command->wait = wait; Command->waitSize = waitSize; /* Update context and pipe select. */ Command->currentContext = context->id; Command->pipeSelect = context->currentPipe; /* Update queue tail pointer. */ gcmkONERROR( gckHARDWARE_UpdateQueueTail(hardware, Command->logical, Command->offset)); #if gcdDUMP_COMMAND gcmkPRINT("@[kernel.commit]"); #endif /* Release the command queue mutex. */ gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexQueue)); --acquired; /* Release the context switching mutex. */ gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); --acquired; /* Submit events if asked for. */ if (Command->submit) { /* Submit events. */ status = gckEVENT_Submit(Command->kernel->event, gcvFALSE, gcvFALSE); if (gcmIS_SUCCESS(status)) { /* Success. */ Command->submit = gcvFALSE; } else { gcmkTRACE_ZONE(gcvLEVEL_WARNING, gcvZONE_COMMAND, "gckEVENT_Submit returned %d", status); } } /* Success. */ status = gcvSTATUS_OK; OnError: if (acquired > 1) { /* Release the command queue mutex. */ gcmkVERIFY_OK( gckOS_ReleaseMutex(Command->os, Command->mutexQueue)); } if (acquired > 0) { /* Release the context switching mutex. */ gcmkVERIFY_OK( gckOS_ReleaseMutex(Command->os, Command->mutexContext)); } /* Unmap all mapped pointers. */ while (stack != gcvNULL) { gcsMAPPED_PTR map = stack; stack = map->next; gcmkVERIFY_OK( gckOS_UnmapUserPointer(Command->os, map->pointer, map->bytes, map->kernelPointer)); gcmkVERIFY_OK( gckOS_Free(Command->os, map)); } /* Return status. */ gcmkFOOTER(); return status; }
/******************************************************************************* ** ** gckCOMMAND_Execute ** ** Execute a previously reserved command queue by appending a WAIT/LINK command ** sequence after it and modifying the last WAIT into a LINK command. The ** command FIFO mutex will be released whether this function succeeds or not. ** ** INPUT: ** ** gckCOMMAND Command ** Pointer to an gckCOMMAND object. ** ** gctSIZE_T RequestedBytes ** Number of bytes previously reserved. ** ** OUTPUT: ** ** Nothing. */ gceSTATUS gckCOMMAND_Execute( IN gckCOMMAND Command, IN gctSIZE_T RequestedBytes, IN gctBOOL Locking ) { gctUINT32 offset; gctPOINTER address; gctSIZE_T bytes; gceSTATUS status; gctPOINTER wait; gctSIZE_T waitBytes; gcmkHEADER_ARG("Command=0x%x RequestedBytes=%lu Locking=%d", Command, RequestedBytes, Locking); /* Verify the arguments. */ gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); if (Command->kernel->notifyIdle) { /* Increase the commit stamp */ Command->commitStamp++; /* Set busy if idle */ if (Command->idle) { Command->idle = gcvFALSE; gcmkVERIFY_OK(gckOS_NotifyIdle(Command->os, gcvFALSE)); } } /* Compute offset for WAIT/LINK. */ offset = Command->offset + RequestedBytes; /* Compute number of byts left in command queue. */ bytes = Command->pageSize - offset; /* Append WAIT/LINK in command queue. */ gcmkONERROR( gckHARDWARE_WaitLink(Command->kernel->hardware, (gctUINT8 *) Command->logical + offset, offset, &bytes, &wait, &waitBytes)); if (Command->newQueue) { /* For a new command queue, point to the start of the command ** queue and include both the commands inserted at the head of it ** and the WAIT/LINK. */ address = Command->logical; bytes += offset; } else { /* For an existing command queue, point to the current offset and ** include the WAIT/LINK. */ address = (gctUINT8 *) Command->logical + Command->offset; bytes += RequestedBytes; } /* Flush the cache. */ gcmkONERROR(gckOS_CacheFlush(Command->os, gcvNULL, address, bytes)); #if gcdDUMP_COMMAND _DumpCommand(Command, address, bytes); #endif /* Convert the last WAIT into a LINK. */ gcmkONERROR(gckHARDWARE_Link(Command->kernel->hardware, Command->wait, address, bytes, &Command->waitSize)); #if MRVL_PRINT_CMD_BUFFER _AddLink(Command, Command->wait, address); #endif /* Flush the cache. */ gcmkONERROR(gckOS_CacheFlush(Command->os, gcvNULL, Command->wait, Command->waitSize)); #if gcdDUMP_COMMAND _DumpCommand(Command, Command->wait, 8); #endif /* Update the pointer to the last WAIT. */ Command->wait = wait; Command->waitSize = waitBytes; /* Update the command queue. */ Command->offset += bytes; Command->newQueue = gcvFALSE; /* Update queue tail pointer. */ gcmkONERROR( gckHARDWARE_UpdateQueueTail(Command->kernel->hardware, Command->logical, Command->offset)); #if gcdDUMP_COMMAND gcmkPRINT("@[kernel.execute]"); #endif if (!Locking) { /* Release the command queue mutex. */ gcmkONERROR( gckOS_ReleaseMutex(Command->os, Command->mutexQueue)); } /* Submit events if asked for. */ if (Command->submit) { /* Submit events. */ status = gckEVENT_Submit(Command->kernel->event, gcvFALSE, gcvFALSE); if (gcmIS_SUCCESS(status)) { /* Success. */ Command->submit = gcvFALSE; } else { gcmkTRACE_ZONE(gcvLEVEL_WARNING, gcvZONE_COMMAND, "gckEVENT_Submit returned %d", status); } } /* Success. */ gcmkFOOTER_NO(); return gcvSTATUS_OK; OnError: /* Release the command queue mutex. */ gcmkVERIFY_OK( gckOS_ReleaseMutex(Command->os, Command->mutexQueue)); /* Return the status. */ gcmkFOOTER(); return status; }