/**
Call back function when the timer event is signaled.
@param[in] Event The Event this notify function registered to.
@param[in] Context Pointer to the context data registered to the
Event.
**/
VOID
EFIAPI
ProcessAsyncTaskList (
IN EFI_EVENT Event,
IN VOID* Context
)
{
NVME_CONTROLLER_PRIVATE_DATA *Private;
EFI_PCI_IO_PROTOCOL *PciIo;
NVME_CQ *Cq;
UINT16 QueueId;
UINT32 Data;
LIST_ENTRY *Link;
LIST_ENTRY *NextLink;
NVME_PASS_THRU_ASYNC_REQ *AsyncRequest;
NVME_BLKIO2_SUBTASK *Subtask;
NVME_BLKIO2_REQUEST *BlkIo2Request;
EFI_BLOCK_IO2_TOKEN *Token;
BOOLEAN HasNewItem;
EFI_STATUS Status;
Private = (NVME_CONTROLLER_PRIVATE_DATA*)Context;
QueueId = 2;
Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
HasNewItem = FALSE;
PciIo = Private->PciIo;
//
// Submit asynchronous subtasks to the NVMe Submission Queue
//
for (Link = GetFirstNode (&Private->UnsubmittedSubtasks);
!IsNull (&Private->UnsubmittedSubtasks, Link);
Link = NextLink) {
NextLink = GetNextNode (&Private->UnsubmittedSubtasks, Link);
Subtask = NVME_BLKIO2_SUBTASK_FROM_LINK (Link);
BlkIo2Request = Subtask->BlockIo2Request;
Token = BlkIo2Request->Token;
RemoveEntryList (Link);
BlkIo2Request->UnsubmittedSubtaskNum--;
//
// If any previous subtask fails, do not process subsequent ones.
//
if (Token->TransactionStatus != EFI_SUCCESS) {
if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
BlkIo2Request->LastSubtaskSubmitted &&
(BlkIo2Request->UnsubmittedSubtaskNum == 0)) {
//
// Remove the BlockIo2 request from the device asynchronous queue.
//
RemoveEntryList (&BlkIo2Request->Link);
FreePool (BlkIo2Request);
gBS->SignalEvent (Token->Event);
}
FreePool (Subtask->CommandPacket->NvmeCmd);
FreePool (Subtask->CommandPacket->NvmeCompletion);
FreePool (Subtask->CommandPacket);
FreePool (Subtask);
continue;
}
Status = Private->Passthru.PassThru (
&Private->Passthru,
Subtask->NamespaceId,
Subtask->CommandPacket,
Subtask->Event
);
if (Status == EFI_NOT_READY) {
InsertHeadList (&Private->UnsubmittedSubtasks, Link);
BlkIo2Request->UnsubmittedSubtaskNum++;
break;
} else if (EFI_ERROR (Status)) {
Token->TransactionStatus = EFI_DEVICE_ERROR;
if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
Subtask->IsLast) {
//
// Remove the BlockIo2 request from the device asynchronous queue.
//
RemoveEntryList (&BlkIo2Request->Link);
FreePool (BlkIo2Request);
gBS->SignalEvent (Token->Event);
}
FreePool (Subtask->CommandPacket->NvmeCmd);
FreePool (Subtask->CommandPacket->NvmeCompletion);
FreePool (Subtask->CommandPacket);
FreePool (Subtask);
} else {
InsertTailList (&BlkIo2Request->SubtasksQueue, Link);
if (Subtask->IsLast) {
BlkIo2Request->LastSubtaskSubmitted = TRUE;
}
}
}
while (Cq->Pt != Private->Pt[QueueId]) {
ASSERT (Cq->Sqid == QueueId);
HasNewItem = TRUE;
//
// Find the command with given Command Id.
//
for (Link = GetFirstNode (&Private->AsyncPassThruQueue);
!IsNull (&Private->AsyncPassThruQueue, Link);
Link = NextLink) {
NextLink = GetNextNode (&Private->AsyncPassThruQueue, Link);
AsyncRequest = NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link);
if (AsyncRequest->CommandId == Cq->Cid) {
//
// Copy the Respose Queue entry for this command to the callers
// response buffer.
//
CopyMem (
AsyncRequest->Packet->NvmeCompletion,
Cq,
sizeof(EFI_NVM_EXPRESS_COMPLETION)
);
//
// Free the resources allocated before cmd submission
//
if (AsyncRequest->MapData != NULL) {
PciIo->Unmap (PciIo, AsyncRequest->MapData);
}
if (AsyncRequest->MapMeta != NULL) {
PciIo->Unmap (PciIo, AsyncRequest->MapMeta);
}
if (AsyncRequest->MapPrpList != NULL) {
PciIo->Unmap (PciIo, AsyncRequest->MapPrpList);
}
if (AsyncRequest->PrpListHost != NULL) {
PciIo->FreeBuffer (
PciIo,
AsyncRequest->PrpListNo,
AsyncRequest->PrpListHost
);
}
RemoveEntryList (Link);
gBS->SignalEvent (AsyncRequest->CallerEvent);
FreePool (AsyncRequest);
//
// Update submission queue head.
//
Private->AsyncSqHead = Cq->Sqhd;
break;
}
}
Private->CqHdbl[QueueId].Cqh++;
if (Private->CqHdbl[QueueId].Cqh > NVME_ASYNC_CCQ_SIZE) {
Private->CqHdbl[QueueId].Cqh = 0;
Private->Pt[QueueId] ^= 1;
}
Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
}
if (HasNewItem) {
Data = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[QueueId]);
PciIo->Mem.Write (
PciIo,
EfiPciIoWidthUint32,
NVME_BAR,
NVME_CQHDBL_OFFSET(QueueId, Private->Cap.Dstrd),
1,
&Data
);
}
}
/**
Sends an NVM Express Command Packet to an NVM Express controller or namespace. This function supports
both blocking I/O and non-blocking I/O. The blocking I/O functionality is required, and the non-blocking
I/O functionality is optional.
@param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
@param[in] NamespaceId A 32 bit namespace ID as defined in the NVMe specification to which the NVM Express Command
Packet will be sent. A value of 0 denotes the NVM Express controller, a value of all 0xFF's
(all bytes are 0xFF) in the namespace ID specifies that the command packet should be sent to
all valid namespaces.
@param[in,out] Packet A pointer to the NVM Express Command Packet.
@param[in] Event If non-blocking I/O is not supported then Event is ignored, and blocking I/O is performed.
If Event is NULL, then blocking I/O is performed. If Event is not NULL and non-blocking I/O
is supported, then non-blocking I/O is performed, and Event will be signaled when the NVM
Express Command Packet completes.
@retval EFI_SUCCESS The NVM Express Command Packet was sent by the host. TransferLength bytes were transferred
to, or from DataBuffer.
@retval EFI_BAD_BUFFER_SIZE The NVM Express Command Packet was not executed. The number of bytes that could be transferred
is returned in TransferLength.
@retval EFI_NOT_READY The NVM Express Command Packet could not be sent because the controller is not ready. The caller
may retry again later.
@retval EFI_DEVICE_ERROR A device error occurred while attempting to send the NVM Express Command Packet.
@retval EFI_INVALID_PARAMETER NamespaceId or the contents of EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET are invalid. The NVM
Express Command Packet was not sent, so no additional status information is available.
@retval EFI_UNSUPPORTED The command described by the NVM Express Command Packet is not supported by the NVM Express
controller. The NVM Express Command Packet was not sent so no additional status information
is available.
@retval EFI_TIMEOUT A timeout occurred while waiting for the NVM Express Command Packet to execute.
**/
EFI_STATUS
EFIAPI
NvmExpressPassThru (
IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *This,
IN UINT32 NamespaceId,
IN OUT EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET *Packet,
IN EFI_EVENT Event OPTIONAL
)
{
NVME_CONTROLLER_PRIVATE_DATA *Private;
EFI_STATUS Status;
EFI_PCI_IO_PROTOCOL *PciIo;
NVME_SQ *Sq;
NVME_CQ *Cq;
UINT16 QueueId;
UINT32 Bytes;
UINT16 Offset;
EFI_EVENT TimerEvent;
EFI_PCI_IO_PROTOCOL_OPERATION Flag;
EFI_PHYSICAL_ADDRESS PhyAddr;
VOID *MapData;
VOID *MapMeta;
VOID *MapPrpList;
UINTN MapLength;
UINT64 *Prp;
VOID *PrpListHost;
UINTN PrpListNo;
UINT32 Attributes;
UINT32 IoAlign;
UINT32 MaxTransLen;
UINT32 Data;
NVME_PASS_THRU_ASYNC_REQ *AsyncRequest;
EFI_TPL OldTpl;
//
// check the data fields in Packet parameter.
//
if ((This == NULL) || (Packet == NULL)) {
return EFI_INVALID_PARAMETER;
}
if ((Packet->NvmeCmd == NULL) || (Packet->NvmeCompletion == NULL)) {
return EFI_INVALID_PARAMETER;
}
if (Packet->QueueType != NVME_ADMIN_QUEUE && Packet->QueueType != NVME_IO_QUEUE) {
return EFI_INVALID_PARAMETER;
}
//
// 'Attributes' with neither EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL nor
// EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL set is an illegal
// configuration.
//
Attributes = This->Mode->Attributes;
if ((Attributes & (EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL |
EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL)) == 0) {
return EFI_INVALID_PARAMETER;
}
//
// Buffer alignment check for TransferBuffer & MetadataBuffer.
//
IoAlign = This->Mode->IoAlign;
if (IoAlign > 0 && (((UINTN) Packet->TransferBuffer & (IoAlign - 1)) != 0)) {
return EFI_INVALID_PARAMETER;
}
if (IoAlign > 0 && (((UINTN) Packet->MetadataBuffer & (IoAlign - 1)) != 0)) {
return EFI_INVALID_PARAMETER;
}
Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
//
// Check NamespaceId is valid or not.
//
if ((NamespaceId > Private->ControllerData->Nn) &&
(NamespaceId != (UINT32) -1)) {
return EFI_INVALID_PARAMETER;
}
//
// Check whether TransferLength exceeds the maximum data transfer size.
//
if (Private->ControllerData->Mdts != 0) {
MaxTransLen = (1 << (Private->ControllerData->Mdts)) *
(1 << (Private->Cap.Mpsmin + 12));
if (Packet->TransferLength > MaxTransLen) {
Packet->TransferLength = MaxTransLen;
return EFI_BAD_BUFFER_SIZE;
}
}
PciIo = Private->PciIo;
MapData = NULL;
MapMeta = NULL;
MapPrpList = NULL;
PrpListHost = NULL;
PrpListNo = 0;
Prp = NULL;
TimerEvent = NULL;
Status = EFI_SUCCESS;
if (Packet->QueueType == NVME_ADMIN_QUEUE) {
QueueId = 0;
} else {
if (Event == NULL) {
QueueId = 1;
} else {
QueueId = 2;
//
// Submission queue full check.
//
if ((Private->SqTdbl[QueueId].Sqt + 1) % (NVME_ASYNC_CSQ_SIZE + 1) ==
Private->AsyncSqHead) {
return EFI_NOT_READY;
}
}
}
Sq = Private->SqBuffer[QueueId] + Private->SqTdbl[QueueId].Sqt;
Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
if (Packet->NvmeCmd->Nsid != NamespaceId) {
return EFI_INVALID_PARAMETER;
}
ZeroMem (Sq, sizeof (NVME_SQ));
Sq->Opc = (UINT8)Packet->NvmeCmd->Cdw0.Opcode;
Sq->Fuse = (UINT8)Packet->NvmeCmd->Cdw0.FusedOperation;
Sq->Cid = Private->Cid[QueueId]++;
Sq->Nsid = Packet->NvmeCmd->Nsid;
//
// Currently we only support PRP for data transfer, SGL is NOT supported.
//
ASSERT (Sq->Psdt == 0);
if (Sq->Psdt != 0) {
DEBUG ((EFI_D_ERROR, "NvmExpressPassThru: doesn't support SGL mechanism\n"));
return EFI_UNSUPPORTED;
}
Sq->Prp[0] = (UINT64)(UINTN)Packet->TransferBuffer;
//
// If the NVMe cmd has data in or out, then mapping the user buffer to the PCI controller specific addresses.
// Note here we don't handle data buffer for CreateIOSubmitionQueue and CreateIOCompletionQueue cmds because
// these two cmds are special which requires their data buffer must support simultaneous access by both the
// processor and a PCI Bus Master. It's caller's responsbility to ensure this.
//
if (((Sq->Opc & (BIT0 | BIT1)) != 0) && (Sq->Opc != NVME_ADMIN_CRIOCQ_CMD) && (Sq->Opc != NVME_ADMIN_CRIOSQ_CMD)) {
if ((Packet->TransferLength == 0) || (Packet->TransferBuffer == NULL)) {
return EFI_INVALID_PARAMETER;
}
if ((Sq->Opc & BIT0) != 0) {
Flag = EfiPciIoOperationBusMasterRead;
} else {
Flag = EfiPciIoOperationBusMasterWrite;
}
MapLength = Packet->TransferLength;
Status = PciIo->Map (
PciIo,
Flag,
Packet->TransferBuffer,
&MapLength,
&PhyAddr,
&MapData
);
if (EFI_ERROR (Status) || (Packet->TransferLength != MapLength)) {
return EFI_OUT_OF_RESOURCES;
}
Sq->Prp[0] = PhyAddr;
Sq->Prp[1] = 0;
if((Packet->MetadataLength != 0) && (Packet->MetadataBuffer != NULL)) {
MapLength = Packet->MetadataLength;
Status = PciIo->Map (
PciIo,
Flag,
Packet->MetadataBuffer,
&MapLength,
&PhyAddr,
&MapMeta
);
if (EFI_ERROR (Status) || (Packet->MetadataLength != MapLength)) {
PciIo->Unmap (
PciIo,
MapData
);
return EFI_OUT_OF_RESOURCES;
}
Sq->Mptr = PhyAddr;
}
}
//
// If the buffer size spans more than two memory pages (page size as defined in CC.Mps),
// then build a PRP list in the second PRP submission queue entry.
//
Offset = ((UINT16)Sq->Prp[0]) & (EFI_PAGE_SIZE - 1);
Bytes = Packet->TransferLength;
if ((Offset + Bytes) > (EFI_PAGE_SIZE * 2)) {
//
// Create PrpList for remaining data buffer.
//
PhyAddr = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
Prp = NvmeCreatePrpList (PciIo, PhyAddr, EFI_SIZE_TO_PAGES(Offset + Bytes) - 1, &PrpListHost, &PrpListNo, &MapPrpList);
if (Prp == NULL) {
goto EXIT;
}
Sq->Prp[1] = (UINT64)(UINTN)Prp;
} else if ((Offset + Bytes) > EFI_PAGE_SIZE) {
Sq->Prp[1] = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
}
if(Packet->NvmeCmd->Flags & CDW2_VALID) {
Sq->Rsvd2 = (UINT64)Packet->NvmeCmd->Cdw2;
}
if(Packet->NvmeCmd->Flags & CDW3_VALID) {
Sq->Rsvd2 |= LShiftU64 ((UINT64)Packet->NvmeCmd->Cdw3, 32);
}
if(Packet->NvmeCmd->Flags & CDW10_VALID) {
Sq->Payload.Raw.Cdw10 = Packet->NvmeCmd->Cdw10;
}
if(Packet->NvmeCmd->Flags & CDW11_VALID) {
Sq->Payload.Raw.Cdw11 = Packet->NvmeCmd->Cdw11;
}
if(Packet->NvmeCmd->Flags & CDW12_VALID) {
Sq->Payload.Raw.Cdw12 = Packet->NvmeCmd->Cdw12;
}
if(Packet->NvmeCmd->Flags & CDW13_VALID) {
Sq->Payload.Raw.Cdw13 = Packet->NvmeCmd->Cdw13;
}
if(Packet->NvmeCmd->Flags & CDW14_VALID) {
Sq->Payload.Raw.Cdw14 = Packet->NvmeCmd->Cdw14;
}
if(Packet->NvmeCmd->Flags & CDW15_VALID) {
Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
}
//
// Ring the submission queue doorbell.
//
if ((Event != NULL) && (QueueId != 0)) {
Private->SqTdbl[QueueId].Sqt =
(Private->SqTdbl[QueueId].Sqt + 1) % (NVME_ASYNC_CSQ_SIZE + 1);
} else {
Private->SqTdbl[QueueId].Sqt ^= 1;
}
Data = ReadUnaligned32 ((UINT32*)&Private->SqTdbl[QueueId]);
Status = PciIo->Mem.Write (
PciIo,
EfiPciIoWidthUint32,
NVME_BAR,
NVME_SQTDBL_OFFSET(QueueId, Private->Cap.Dstrd),
1,
&Data
);
if (EFI_ERROR (Status)) {
goto EXIT;
}
//
// For non-blocking requests, return directly if the command is placed
// in the submission queue.
//
if ((Event != NULL) && (QueueId != 0)) {
AsyncRequest = AllocateZeroPool (sizeof (NVME_PASS_THRU_ASYNC_REQ));
if (AsyncRequest == NULL) {
Status = EFI_DEVICE_ERROR;
goto EXIT;
}
AsyncRequest->Signature = NVME_PASS_THRU_ASYNC_REQ_SIG;
AsyncRequest->Packet = Packet;
AsyncRequest->CommandId = Sq->Cid;
AsyncRequest->CallerEvent = Event;
AsyncRequest->MapData = MapData;
AsyncRequest->MapMeta = MapMeta;
AsyncRequest->MapPrpList = MapPrpList;
AsyncRequest->PrpListNo = PrpListNo;
AsyncRequest->PrpListHost = PrpListHost;
OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
InsertTailList (&Private->AsyncPassThruQueue, &AsyncRequest->Link);
gBS->RestoreTPL (OldTpl);
return EFI_SUCCESS;
}
Status = gBS->CreateEvent (
EVT_TIMER,
TPL_CALLBACK,
NULL,
NULL,
&TimerEvent
);
if (EFI_ERROR (Status)) {
goto EXIT;
}
Status = gBS->SetTimer(TimerEvent, TimerRelative, Packet->CommandTimeout);
if (EFI_ERROR(Status)) {
goto EXIT;
}
//
// Wait for completion queue to get filled in.
//
Status = EFI_TIMEOUT;
while (EFI_ERROR (gBS->CheckEvent (TimerEvent))) {
if (Cq->Pt != Private->Pt[QueueId]) {
Status = EFI_SUCCESS;
break;
}
}
//
// Check the NVMe cmd execution result
//
if (Status != EFI_TIMEOUT) {
if ((Cq->Sct == 0) && (Cq->Sc == 0)) {
Status = EFI_SUCCESS;
} else {
Status = EFI_DEVICE_ERROR;
//
// Copy the Respose Queue entry for this command to the callers response buffer
//
CopyMem(Packet->NvmeCompletion, Cq, sizeof(EFI_NVM_EXPRESS_COMPLETION));
//
// Dump every completion entry status for debugging.
//
DEBUG_CODE_BEGIN();
NvmeDumpStatus(Cq);
DEBUG_CODE_END();
}
} else {
//
// Timeout occurs for an NVMe command. Reset the controller to abort the
// outstanding commands.
//
DEBUG ((DEBUG_ERROR, "NvmExpressPassThru: Timeout occurs for an NVMe command.\n"));
//
// Disable the timer to trigger the process of async transfers temporarily.
//
Status = gBS->SetTimer (Private->TimerEvent, TimerCancel, 0);
if (EFI_ERROR (Status)) {
goto EXIT;
}
//
// Reset the NVMe controller.
//
Status = NvmeControllerInit (Private);
if (!EFI_ERROR (Status)) {
Status = AbortAsyncPassThruTasks (Private);
if (!EFI_ERROR (Status)) {
//
// Re-enable the timer to trigger the process of async transfers.
//
Status = gBS->SetTimer (Private->TimerEvent, TimerPeriodic, NVME_HC_ASYNC_TIMER);
if (!EFI_ERROR (Status)) {
//
// Return EFI_TIMEOUT to indicate a timeout occurs for NVMe PassThru command.
//
Status = EFI_TIMEOUT;
}
}
} else {
Status = EFI_DEVICE_ERROR;
}
goto EXIT;
}
if ((Private->CqHdbl[QueueId].Cqh ^= 1) == 0) {
Private->Pt[QueueId] ^= 1;
}
Data = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[QueueId]);
Status = PciIo->Mem.Write (
PciIo,
EfiPciIoWidthUint32,
NVME_BAR,
NVME_CQHDBL_OFFSET(QueueId, Private->Cap.Dstrd),
1,
&Data
);
//
// For now, the code does not support the non-blocking feature for admin queue.
// If Event is not NULL for admin queue, signal the caller's event here.
//
if (Event != NULL) {
ASSERT (QueueId == 0);
gBS->SignalEvent (Event);
}
EXIT:
if (MapData != NULL) {
PciIo->Unmap (
PciIo,
MapData
);
}
if (MapMeta != NULL) {
PciIo->Unmap (
PciIo,
MapMeta
);
}
if (MapPrpList != NULL) {
PciIo->Unmap (
PciIo,
MapPrpList
);
}
if (Prp != NULL) {
PciIo->FreeBuffer (PciIo, PrpListNo, PrpListHost);
}
if (TimerEvent != NULL) {
gBS->CloseEvent (TimerEvent);
}
return Status;
}
