/**
Sends an NVM Express Command Packet to an NVM Express controller or namespace. This function only
supports blocking execution of the command.
@param[in] Private The pointer to the NVME_CONTEXT Data structure.
@param[in] NamespaceId Is a 32 bit Namespace ID to which the Express HCI command packet will
be sent.
A Value of 0 denotes the NVM Express controller, a Value of all 0FFh in
the namespace ID specifies that the command packet should be sent to all
valid namespaces.
@param[in,out] Packet A pointer to the EDKII PEI NVM Express PassThru Command Packet to send
to the NVMe namespace specified by NamespaceId.
@retval EFI_SUCCESS The EDKII PEI NVM Express Command Packet was sent by the host.
TransferLength bytes were transferred to, or from DataBuffer.
@retval EFI_NOT_READY The EDKII PEI NVM Express Command Packet could not be sent because
the controller is not ready. The caller may retry again later.
@retval EFI_DEVICE_ERROR A device error occurred while attempting to send the EDKII PEI NVM
Express Command Packet.
@retval EFI_INVALID_PARAMETER Namespace, or the contents of EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
are invalid.
The EDKII PEI NVM Express Command Packet was not sent, so no
additional status information is available.
@retval EFI_UNSUPPORTED The command described by the EDKII PEI NVM Express Command Packet
is not supported by the host adapter.
The EDKII PEI NVM Express Command Packet was not sent, so no
additional status information is available.
@retval EFI_TIMEOUT A timeout occurred while waiting for the EDKII PEI NVM Express Command
Packet to execute.
**/
EFI_STATUS
NvmePassThru (
IN PEI_NVME_CONTROLLER_PRIVATE_DATA *Private,
IN UINT32 NamespaceId,
IN OUT EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET *Packet
)
{
EFI_STATUS Status;
NVME_SQ *Sq;
NVME_CQ *Cq;
UINT8 QueueId;
UINTN SqSize;
UINTN CqSize;
EDKII_IOMMU_OPERATION MapOp;
UINTN MapLength;
EFI_PHYSICAL_ADDRESS PhyAddr;
VOID *MapData;
VOID *MapMeta;
UINT32 Bytes;
UINT32 Offset;
UINT32 Data32;
UINT64 Timer;
//
// Check the data fields in Packet parameter
//
if (Packet == NULL) {
DEBUG ((
DEBUG_ERROR,
"%a, Invalid parameter: Packet(%lx)\n",
__FUNCTION__,
(UINTN)Packet
));
return EFI_INVALID_PARAMETER;
}
if ((Packet->NvmeCmd == NULL) || (Packet->NvmeCompletion == NULL)) {
DEBUG ((
DEBUG_ERROR,
"%a, Invalid parameter: NvmeCmd (%lx)/NvmeCompletion(%lx)\n",
__FUNCTION__,
(UINTN)Packet->NvmeCmd,
(UINTN)Packet->NvmeCompletion
));
return EFI_INVALID_PARAMETER;
}
if (Packet->QueueType != NVME_ADMIN_QUEUE && Packet->QueueType != NVME_IO_QUEUE) {
DEBUG ((
DEBUG_ERROR,
"%a, Invalid parameter: QueueId(%lx)\n",
__FUNCTION__,
(UINTN)Packet->QueueType
));
return EFI_INVALID_PARAMETER;
}
QueueId = Packet->QueueType;
Sq = Private->SqBuffer[QueueId] + Private->SqTdbl[QueueId].Sqt;
Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
if (QueueId == NVME_ADMIN_QUEUE) {
SqSize = NVME_ASQ_SIZE + 1;
CqSize = NVME_ACQ_SIZE + 1;
} else {
SqSize = NVME_CSQ_SIZE + 1;
CqSize = NVME_CCQ_SIZE + 1;
}
if (Packet->NvmeCmd->Nsid != NamespaceId) {
DEBUG ((
DEBUG_ERROR,
"%a: Nsid mismatch (%x, %x)\n",
__FUNCTION__,
Packet->NvmeCmd->Nsid,
NamespaceId
));
return EFI_INVALID_PARAMETER;
}
ZeroMem (Sq, sizeof (NVME_SQ));
Sq->Opc = Packet->NvmeCmd->Cdw0.Opcode;
Sq->Fuse = Packet->NvmeCmd->Cdw0.FusedOperation;
Sq->Cid = Packet->NvmeCmd->Cdw0.Cid;
Sq->Nsid = Packet->NvmeCmd->Nsid;
//
// Currently we only support PRP for data transfer, SGL is NOT supported
//
ASSERT (Sq->Psdt == 0);
if (Sq->Psdt != 0) {
DEBUG ((DEBUG_ERROR, "%a: Does not support SGL mechanism.\n", __FUNCTION__));
return EFI_UNSUPPORTED;
}
Sq->Prp[0] = (UINT64)(UINTN)Packet->TransferBuffer;
Sq->Prp[1] = 0;
MapData = NULL;
MapMeta = NULL;
Status = EFI_SUCCESS;
//
// If the NVMe cmd has data in or out, then mapping the user buffer to the PCI controller
// specific addresses.
//
if ((Sq->Opc & (BIT0 | BIT1)) != 0) {
if (((Packet->TransferLength != 0) && (Packet->TransferBuffer == NULL)) ||
((Packet->TransferLength == 0) && (Packet->TransferBuffer != NULL))) {
return EFI_INVALID_PARAMETER;
}
//
// Currently, we only support creating IO submission/completion queues that are
// allocated internally by the driver.
//
if ((Packet->QueueType == NVME_ADMIN_QUEUE) &&
((Sq->Opc == NVME_ADMIN_CRIOCQ_CMD) || (Sq->Opc == NVME_ADMIN_CRIOSQ_CMD))) {
if ((Packet->TransferBuffer != Private->SqBuffer[NVME_IO_QUEUE]) &&
(Packet->TransferBuffer != Private->CqBuffer[NVME_IO_QUEUE])) {
DEBUG ((
DEBUG_ERROR,
"%a: Does not support external IO queues creation request.\n",
__FUNCTION__
));
return EFI_UNSUPPORTED;
}
} else {
if ((Sq->Opc & BIT0) != 0) {
MapOp = EdkiiIoMmuOperationBusMasterRead;
} else {
MapOp = EdkiiIoMmuOperationBusMasterWrite;
}
if ((Packet->TransferLength != 0) && (Packet->TransferBuffer != NULL)) {
MapLength = Packet->TransferLength;
Status = IoMmuMap (
MapOp,
Packet->TransferBuffer,
&MapLength,
&PhyAddr,
&MapData
);
if (EFI_ERROR (Status) || (MapLength != Packet->TransferLength)) {
Status = EFI_OUT_OF_RESOURCES;
DEBUG ((DEBUG_ERROR, "%a: Fail to map data buffer.\n", __FUNCTION__));
goto Exit;
}
Sq->Prp[0] = PhyAddr;
}
if((Packet->MetadataLength != 0) && (Packet->MetadataBuffer != NULL)) {
MapLength = Packet->MetadataLength;
Status = IoMmuMap (
MapOp,
Packet->MetadataBuffer,
&MapLength,
&PhyAddr,
&MapMeta
);
if (EFI_ERROR (Status) || (MapLength != Packet->MetadataLength)) {
Status = EFI_OUT_OF_RESOURCES;
DEBUG ((DEBUG_ERROR, "%a: Fail to map meta data buffer.\n", __FUNCTION__));
goto Exit;
}
Sq->Mptr = PhyAddr;
}
}
}
//
// If the Buffer Size spans more than two memory pages (page Size as defined in CC.Mps),
// then build a PRP list in the second PRP submission queue entry.
//
Offset = ((UINT32)Sq->Prp[0]) & (EFI_PAGE_SIZE - 1);
Bytes = Packet->TransferLength;
if ((Offset + Bytes) > (EFI_PAGE_SIZE * 2)) {
//
// Create PrpList for remaining Data Buffer.
//
PhyAddr = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
Sq->Prp[1] = NvmeCreatePrpList (
Private,
PhyAddr,
EFI_SIZE_TO_PAGES(Offset + Bytes) - 1
);
if (Sq->Prp[1] == 0) {
Status = EFI_OUT_OF_RESOURCES;
DEBUG ((DEBUG_ERROR, "%a: Create PRP list fail, Status - %r\n", __FUNCTION__, Status));
goto Exit;
}
} else if ((Offset + Bytes) > EFI_PAGE_SIZE) {
Sq->Prp[1] = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
}
if (Packet->NvmeCmd->Flags & CDW10_VALID) {
Sq->Payload.Raw.Cdw10 = Packet->NvmeCmd->Cdw10;
}
if (Packet->NvmeCmd->Flags & CDW11_VALID) {
Sq->Payload.Raw.Cdw11 = Packet->NvmeCmd->Cdw11;
}
if (Packet->NvmeCmd->Flags & CDW12_VALID) {
Sq->Payload.Raw.Cdw12 = Packet->NvmeCmd->Cdw12;
}
if (Packet->NvmeCmd->Flags & CDW13_VALID) {
Sq->Payload.Raw.Cdw13 = Packet->NvmeCmd->Cdw13;
}
if (Packet->NvmeCmd->Flags & CDW14_VALID) {
Sq->Payload.Raw.Cdw14 = Packet->NvmeCmd->Cdw14;
}
if (Packet->NvmeCmd->Flags & CDW15_VALID) {
Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
}
//
// Ring the submission queue doorbell.
//
Private->SqTdbl[QueueId].Sqt++;
if (Private->SqTdbl[QueueId].Sqt == SqSize) {
Private->SqTdbl[QueueId].Sqt = 0;
}
Data32 = ReadUnaligned32 ((UINT32 *)&Private->SqTdbl[QueueId]);
Status = NVME_SET_SQTDBL (Private, QueueId, &Data32);
if (EFI_ERROR (Status)) {
DEBUG ((DEBUG_ERROR, "%a: NVME_SET_SQTDBL fail, Status - %r\n", __FUNCTION__, Status));
goto Exit;
}
//
// Wait for completion queue to get filled in.
//
Status = EFI_TIMEOUT;
Timer = 0;
while (Timer < Packet->CommandTimeout) {
if (Cq->Pt != Private->Pt[QueueId]) {
Status = EFI_SUCCESS;
break;
}
MicroSecondDelay (NVME_POLL_INTERVAL);
Timer += NVME_POLL_INTERVAL;
}
if (Status == EFI_TIMEOUT) {
//
// Timeout occurs for an NVMe command, reset the controller to abort the outstanding command
//
DEBUG ((DEBUG_ERROR, "%a: Timeout occurs for the PassThru command.\n", __FUNCTION__));
Status = NvmeControllerInit (Private);
if (EFI_ERROR (Status)) {
Status = EFI_DEVICE_ERROR;
} else {
//
// Return EFI_TIMEOUT to indicate a timeout occurs for PassThru command
//
Status = EFI_TIMEOUT;
}
goto Exit;
}
//
// Move forward the Completion Queue head
//
Private->CqHdbl[QueueId].Cqh++;
if (Private->CqHdbl[QueueId].Cqh == CqSize) {
Private->CqHdbl[QueueId].Cqh = 0;
Private->Pt[QueueId] ^= 1;
}
//
// Copy the Respose Queue entry for this command to the callers response buffer
//
CopyMem (Packet->NvmeCompletion, Cq, sizeof (EDKII_PEI_NVM_EXPRESS_COMPLETION));
//
// Check the NVMe cmd execution result
//
Status = NvmeCheckCqStatus (Cq);
NVME_SET_CQHDBL (Private, QueueId, &Private->CqHdbl[QueueId]);
Exit:
if (MapMeta != NULL) {
IoMmuUnmap (MapMeta);
}
if (MapData != NULL) {
IoMmuUnmap (MapData);
}
return Status;
}
/**
Dispatch function for a Software SMI handler.
@param[in] DispatchHandle The unique handle assigned to this handler by SmiHandlerRegister().
@param[in] RegisterContext Points to an optional handler context which was specified when the
handler was registered.
@param[in, out] CommBuffer A pointer to a collection of Data in memory that will
be conveyed from a non-SMM environment into an SMM environment.
@param[in, out] CommBufferSize The Size of the CommBuffer.
@retval EFI_SUCCESS The interrupt was handled and quiesced. No other handlers
should still be called.
@retval Others Other execution results.
**/
EFI_STATUS
EFIAPI
SmmUnlockOpalPassword (
IN EFI_HANDLE DispatchHandle,
IN CONST VOID *RegisterContext,
IN OUT VOID *CommBuffer,
IN OUT UINTN *CommBufferSize
)
{
EFI_STATUS Status;
OPAL_SMM_DEVICE *OpalDev;
LIST_ENTRY *Entry;
UINT8 BaseClassCode;
UINT8 SubClassCode;
UINT8 ProgInt;
TCG_RESULT Result;
UINT8 SataCmdSt;
UINT8 *StorePcieConfDataList[16];
UINTN RpBase;
UINTN MemoryBase;
UINTN MemoryLength;
OPAL_SESSION Session;
BOOLEAN BlockSidSupport;
ZeroMem (StorePcieConfDataList, sizeof (StorePcieConfDataList));
Status = EFI_DEVICE_ERROR;
//
// try to unlock all locked hdd disks.
//
for (Entry = mSmmDeviceList.ForwardLink; Entry != &mSmmDeviceList; Entry = Entry->ForwardLink) {
OpalDev = BASE_CR(Entry, OPAL_SMM_DEVICE, Link);
RpBase = 0;
SataCmdSt = 0;
///
/// Configure RootPort for PCIe AHCI/NVME devices.
///
if (OpalDev->DeviceType == OPAL_DEVICE_TYPE_NVME) {
///
/// Save original RootPort configuration space to heap
///
RpBase = SaveRestoreRootportConfSpace (
OpalDev,
TRUE,
StorePcieConfDataList
);
MemoryBase = mNvmeContext.Nbar;
MemoryLength = 0;
ConfigureRootPortForPcieNand (RpBase, OpalDev->BusNum, (UINT32) MemoryBase, (UINT32) MemoryLength);
///
/// Enable PCIE decode for RootPort
///
SataCmdSt = PciRead8 (RpBase + NVME_PCIE_PCICMD);
PciWrite8 (RpBase + NVME_PCIE_PCICMD, 0x6);
} else {
SataCmdSt = PciRead8 (PCI_LIB_ADDRESS (OpalDev->BusNum, OpalDev->DevNum, OpalDev->FuncNum, NVME_PCIE_PCICMD));
PciWrite8 (PCI_LIB_ADDRESS (OpalDev->BusNum, OpalDev->DevNum, OpalDev->FuncNum, NVME_PCIE_PCICMD), 0x6);
}
BaseClassCode = PciRead8 (PCI_LIB_ADDRESS (OpalDev->BusNum, OpalDev->DevNum, OpalDev->FuncNum, 0x0B));
SubClassCode = PciRead8 (PCI_LIB_ADDRESS (OpalDev->BusNum, OpalDev->DevNum, OpalDev->FuncNum, 0x0A));
ProgInt = PciRead8 (PCI_LIB_ADDRESS (OpalDev->BusNum, OpalDev->DevNum, OpalDev->FuncNum, 0x09));
if (BaseClassCode != PCI_CLASS_MASS_STORAGE) {
Status = EFI_INVALID_PARAMETER;
break;
}
Status = EFI_DEVICE_ERROR;
if (OpalDev->DeviceType == OPAL_DEVICE_TYPE_SATA) {
if ((SubClassCode == PCI_CLASS_MASS_STORAGE_AHCI) || (SubClassCode == PCI_CLASS_MASS_STORAGE_RAID)) {
Status = GetAhciBaseAddress (OpalDev->BusNum, OpalDev->DevNum, OpalDev->FuncNum);
if (EFI_ERROR (Status)) {
DEBUG ((DEBUG_ERROR, "GetAhciBaseAddress error, Status: %r\n", Status));
goto done;
}
Status = AhciModeInitialize ((UINT8)OpalDev->SataPort);
ASSERT_EFI_ERROR (Status);
if (EFI_ERROR (Status)) {
DEBUG ((DEBUG_ERROR, "AhciModeInitialize error, Status: %r\n", Status));
goto done;
}
} else {
DEBUG ((DEBUG_ERROR, "SubClassCode not support for SATA device\n"));
}
} else if (OpalDev->DeviceType == OPAL_DEVICE_TYPE_NVME) {
if (SubClassCode == PCI_CLASS_MASS_STORAGE_NVM) {
if (ProgInt != PCI_IF_NVMHCI) {
DEBUG ((DEBUG_ERROR, "PI not support, skipped\n"));
Status = EFI_NOT_FOUND;
goto done;
}
mNvmeContext.PciBase = PCI_LIB_ADDRESS (OpalDev->BusNum, OpalDev->DevNum, OpalDev->FuncNum, 0x0);
mNvmeContext.NvmeInitWaitTime = 0;
mNvmeContext.Nsid = OpalDev->NvmeNamespaceId;
Status = NvmeControllerInit (&mNvmeContext);
} else {
DEBUG ((DEBUG_ERROR, "SubClassCode not support for NVME device\n"));
}
} else {
DEBUG ((DEBUG_ERROR, "Invalid Devicetype\n"));
goto done;
}
Status = EFI_DEVICE_ERROR;
BlockSidSupport = FALSE;
if (IsOpalDeviceLocked (OpalDev, &BlockSidSupport)) {
ZeroMem(&Session, sizeof(Session));
Session.Sscp = &OpalDev->Sscp;
Session.MediaId = 0;
Session.OpalBaseComId = OpalDev->OpalBaseComId;
Result = OpalSupportUnlock (&Session, OpalDev->Password, OpalDev->PasswordLength, NULL);
if (Result == TcgResultSuccess) {
Status = EFI_SUCCESS;
}
}
if (mSendBlockSID && BlockSidSupport) {
Result = OpalBlockSid (&Session, TRUE);
if (Result != TcgResultSuccess) {
break;
}
}
if (OpalDev->DeviceType == OPAL_DEVICE_TYPE_NVME) {
if (SubClassCode == PCI_CLASS_MASS_STORAGE_NVM) {
Status = NvmeControllerExit (&mNvmeContext);
}
}
done:
if (OpalDev->DeviceType == OPAL_DEVICE_TYPE_NVME) {
ASSERT (RpBase != 0);
PciWrite8 (RpBase + NVME_PCIE_PCICMD, 0);
RpBase = SaveRestoreRootportConfSpace (
OpalDev,
FALSE, // restore
StorePcieConfDataList
);
PciWrite8 (RpBase + NVME_PCIE_PCICMD, SataCmdSt);
} else {
PciWrite8 (PCI_LIB_ADDRESS (OpalDev->BusNum, OpalDev->DevNum, OpalDev->FuncNum, NVME_PCIE_PCICMD), SataCmdSt);
}
if (EFI_ERROR (Status)) {
break;
}
}
return Status;
}
/**
Entry point of the PEIM.
@param[in] FileHandle Handle of the file being invoked.
@param[in] PeiServices Describes the list of possible PEI Services.
@retval EFI_SUCCESS PPI successfully installed.
**/
EFI_STATUS
EFIAPI
NvmExpressPeimEntry (
IN EFI_PEI_FILE_HANDLE FileHandle,
IN CONST EFI_PEI_SERVICES **PeiServices
)
{
EFI_STATUS Status;
EFI_BOOT_MODE BootMode;
EDKII_NVM_EXPRESS_HOST_CONTROLLER_PPI *NvmeHcPpi;
UINT8 Controller;
UINTN MmioBase;
UINTN DevicePathLength;
EFI_DEVICE_PATH_PROTOCOL *DevicePath;
PEI_NVME_CONTROLLER_PRIVATE_DATA *Private;
EFI_PHYSICAL_ADDRESS DeviceAddress;
DEBUG ((DEBUG_INFO, "%a: Enters.\n", __FUNCTION__));
//
// Get the current boot mode.
//
Status = PeiServicesGetBootMode (&BootMode);
if (EFI_ERROR (Status)) {
DEBUG ((DEBUG_ERROR, "%a: Fail to get the current boot mode.\n", __FUNCTION__));
return Status;
}
//
// Locate the NVME host controller PPI
//
Status = PeiServicesLocatePpi (
&gEdkiiPeiNvmExpressHostControllerPpiGuid,
0,
NULL,
(VOID **) &NvmeHcPpi
);
if (EFI_ERROR (Status)) {
DEBUG ((DEBUG_ERROR, "%a: Fail to locate NvmeHostControllerPpi.\n", __FUNCTION__));
return EFI_UNSUPPORTED;
}
Controller = 0;
MmioBase = 0;
while (TRUE) {
Status = NvmeHcPpi->GetNvmeHcMmioBar (
NvmeHcPpi,
Controller,
&MmioBase
);
//
// When status is error, meant no controller is found
//
if (EFI_ERROR (Status)) {
break;
}
Status = NvmeHcPpi->GetNvmeHcDevicePath (
NvmeHcPpi,
Controller,
&DevicePathLength,
&DevicePath
);
if (EFI_ERROR (Status)) {
DEBUG ((
DEBUG_ERROR, "%a: Fail to allocate get the device path for Controller %d.\n",
__FUNCTION__, Controller
));
return Status;
}
//
// Check validity of the device path of the NVM Express controller.
//
Status = NvmeIsHcDevicePathValid (DevicePath, DevicePathLength);
if (EFI_ERROR (Status)) {
DEBUG ((
DEBUG_ERROR, "%a: The device path is invalid for Controller %d.\n",
__FUNCTION__, Controller
));
Controller++;
continue;
}
//
// For S3 resume performance consideration, not all NVM Express controllers
// will be initialized. The driver consumes the content within
// S3StorageDeviceInitList LockBox to see if a controller will be skipped
// during S3 resume.
//
if ((BootMode == BOOT_ON_S3_RESUME) &&
(NvmeS3SkipThisController (DevicePath, DevicePathLength))) {
DEBUG ((
DEBUG_ERROR, "%a: Controller %d is skipped during S3.\n",
__FUNCTION__, Controller
));
Controller++;
continue;
}
//
// Memory allocation for controller private data
//
Private = AllocateZeroPool (sizeof (PEI_NVME_CONTROLLER_PRIVATE_DATA));
if (Private == NULL) {
DEBUG ((
DEBUG_ERROR, "%a: Fail to allocate private data for Controller %d.\n",
__FUNCTION__, Controller
));
return EFI_OUT_OF_RESOURCES;
}
//
// Memory allocation for transfer-related data
//
Status = IoMmuAllocateBuffer (
NVME_MEM_MAX_PAGES,
&Private->Buffer,
&DeviceAddress,
&Private->BufferMapping
);
if (EFI_ERROR (Status)) {
DEBUG ((
DEBUG_ERROR, "%a: Fail to allocate DMA buffers for Controller %d.\n",
__FUNCTION__, Controller
));
return Status;
}
ASSERT (DeviceAddress == ((EFI_PHYSICAL_ADDRESS) (UINTN) Private->Buffer));
DEBUG ((DEBUG_INFO, "%a: DMA buffer base at 0x%x\n", __FUNCTION__, Private->Buffer));
//
// Initialize controller private data
//
Private->Signature = NVME_PEI_CONTROLLER_PRIVATE_DATA_SIGNATURE;
Private->MmioBase = MmioBase;
Private->DevicePathLength = DevicePathLength;
Private->DevicePath = DevicePath;
//
// Initialize the NVME controller
//
Status = NvmeControllerInit (Private);
if (EFI_ERROR (Status)) {
DEBUG ((
DEBUG_ERROR,
"%a: Controller initialization fail for Controller %d with Status - %r.\n",
__FUNCTION__, Controller, Status
));
NvmeFreeDmaResource (Private);
Controller++;
continue;
}
//
// Enumerate the NVME namespaces on the controller
//
Status = NvmeDiscoverNamespaces (Private);
if (EFI_ERROR (Status)) {
//
// No active namespace was found on the controller
//
DEBUG ((
DEBUG_ERROR,
"%a: Namespaces discovery fail for Controller %d with Status - %r.\n",
__FUNCTION__, Controller, Status
));
NvmeFreeDmaResource (Private);
Controller++;
continue;
}
Private->BlkIoPpi.GetNumberOfBlockDevices = NvmeBlockIoPeimGetDeviceNo;
Private->BlkIoPpi.GetBlockDeviceMediaInfo = NvmeBlockIoPeimGetMediaInfo;
Private->BlkIoPpi.ReadBlocks = NvmeBlockIoPeimReadBlocks;
CopyMem (
&Private->BlkIoPpiList,
&mNvmeBlkIoPpiListTemplate,
sizeof (EFI_PEI_PPI_DESCRIPTOR)
);
Private->BlkIoPpiList.Ppi = &Private->BlkIoPpi;
Private->BlkIo2Ppi.Revision = EFI_PEI_RECOVERY_BLOCK_IO2_PPI_REVISION;
Private->BlkIo2Ppi.GetNumberOfBlockDevices = NvmeBlockIoPeimGetDeviceNo2;
Private->BlkIo2Ppi.GetBlockDeviceMediaInfo = NvmeBlockIoPeimGetMediaInfo2;
Private->BlkIo2Ppi.ReadBlocks = NvmeBlockIoPeimReadBlocks2;
CopyMem (
&Private->BlkIo2PpiList,
&mNvmeBlkIo2PpiListTemplate,
sizeof (EFI_PEI_PPI_DESCRIPTOR)
);
Private->BlkIo2PpiList.Ppi = &Private->BlkIo2Ppi;
PeiServicesInstallPpi (&Private->BlkIoPpiList);
//
// Check if the NVME controller supports the Security Receive/Send commands
//
if ((Private->ControllerData->Oacs & SECURITY_SEND_RECEIVE_SUPPORTED) != 0) {
DEBUG ((
DEBUG_INFO,
"%a: Security Security Command PPI will be produced for Controller %d.\n",
__FUNCTION__, Controller
));
Private->StorageSecurityPpi.Revision = EDKII_STORAGE_SECURITY_PPI_REVISION;
Private->StorageSecurityPpi.GetNumberofDevices = NvmeStorageSecurityGetDeviceNo;
Private->StorageSecurityPpi.GetDevicePath = NvmeStorageSecurityGetDevicePath;
Private->StorageSecurityPpi.ReceiveData = NvmeStorageSecurityReceiveData;
Private->StorageSecurityPpi.SendData = NvmeStorageSecuritySendData;
CopyMem (
&Private->StorageSecurityPpiList,
&mNvmeStorageSecurityPpiListTemplate,
sizeof (EFI_PEI_PPI_DESCRIPTOR)
);
Private->StorageSecurityPpiList.Ppi = &Private->StorageSecurityPpi;
PeiServicesInstallPpi (&Private->StorageSecurityPpiList);
}
CopyMem (
&Private->EndOfPeiNotifyList,
&mNvmeEndOfPeiNotifyListTemplate,
sizeof (EFI_PEI_NOTIFY_DESCRIPTOR)
);
PeiServicesNotifyPpi (&Private->EndOfPeiNotifyList);
DEBUG ((
DEBUG_INFO, "%a: Controller %d has been successfully initialized.\n",
__FUNCTION__, Controller
));
Controller++;
}
return EFI_SUCCESS;
}
/**
Starts a device controller or a bus controller.
The Start() function is designed to be invoked from the EFI boot service ConnectController().
As a result, much of the error checking on the parameters to Start() has been moved into this
common boot service. It is legal to call Start() from other locations,
but the following calling restrictions must be followed or the system behavior will not be deterministic.
1. ControllerHandle must be a valid EFI_HANDLE.
2. If RemainingDevicePath is not NULL, then it must be a pointer to a naturally aligned
EFI_DEVICE_PATH_PROTOCOL.
3. Prior to calling Start(), the Supported() function for the driver specified by This must
have been called with the same calling parameters, and Supported() must have returned EFI_SUCCESS.
@param[in] This A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
@param[in] ControllerHandle The handle of the controller to start. This handle
must support a protocol interface that supplies
an I/O abstraction to the driver.
@param[in] RemainingDevicePath A pointer to the remaining portion of a device path. This
parameter is ignored by device drivers, and is optional for bus
drivers. For a bus driver, if this parameter is NULL, then handles
for all the children of Controller are created by this driver.
If this parameter is not NULL and the first Device Path Node is
not the End of Device Path Node, then only the handle for the
child device specified by the first Device Path Node of
RemainingDevicePath is created by this driver.
If the first Device Path Node of RemainingDevicePath is
the End of Device Path Node, no child handle is created by this
driver.
@retval EFI_SUCCESS The device was started.
@retval EFI_DEVICE_ERROR The device could not be started due to a device error.Currently not implemented.
@retval EFI_OUT_OF_RESOURCES The request could not be completed due to a lack of resources.
@retval Others The driver failded to start the device.
**/
EFI_STATUS
EFIAPI
NvmExpressDriverBindingStart (
IN EFI_DRIVER_BINDING_PROTOCOL *This,
IN EFI_HANDLE Controller,
IN EFI_DEVICE_PATH_PROTOCOL *RemainingDevicePath
)
{
EFI_STATUS Status;
EFI_PCI_IO_PROTOCOL *PciIo;
NVME_CONTROLLER_PRIVATE_DATA *Private;
EFI_DEVICE_PATH_PROTOCOL *ParentDevicePath;
UINT32 NamespaceId;
EFI_PHYSICAL_ADDRESS MappedAddr;
UINTN Bytes;
EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *Passthru;
DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: start\n"));
Private = NULL;
Passthru = NULL;
ParentDevicePath = NULL;
Status = gBS->OpenProtocol (
Controller,
&gEfiDevicePathProtocolGuid,
(VOID **) &ParentDevicePath,
This->DriverBindingHandle,
Controller,
EFI_OPEN_PROTOCOL_BY_DRIVER
);
if ((EFI_ERROR (Status)) && (Status != EFI_ALREADY_STARTED)) {
return Status;
}
Status = gBS->OpenProtocol (
Controller,
&gEfiPciIoProtocolGuid,
(VOID **) &PciIo,
This->DriverBindingHandle,
Controller,
EFI_OPEN_PROTOCOL_BY_DRIVER
);
if (EFI_ERROR (Status) && (Status != EFI_ALREADY_STARTED)) {
return Status;
}
//
// Check EFI_ALREADY_STARTED to reuse the original NVME_CONTROLLER_PRIVATE_DATA.
//
if (Status != EFI_ALREADY_STARTED) {
Private = AllocateZeroPool (sizeof (NVME_CONTROLLER_PRIVATE_DATA));
if (Private == NULL) {
DEBUG ((EFI_D_ERROR, "NvmExpressDriverBindingStart: allocating pool for Nvme Private Data failed!\n"));
Status = EFI_OUT_OF_RESOURCES;
goto Exit;
}
//
// 6 x 4kB aligned buffers will be carved out of this buffer.
// 1st 4kB boundary is the start of the admin submission queue.
// 2nd 4kB boundary is the start of the admin completion queue.
// 3rd 4kB boundary is the start of I/O submission queue #1.
// 4th 4kB boundary is the start of I/O completion queue #1.
// 5th 4kB boundary is the start of I/O submission queue #2.
// 6th 4kB boundary is the start of I/O completion queue #2.
//
// Allocate 6 pages of memory, then map it for bus master read and write.
//
Status = PciIo->AllocateBuffer (
PciIo,
AllocateAnyPages,
EfiBootServicesData,
6,
(VOID**)&Private->Buffer,
0
);
if (EFI_ERROR (Status)) {
goto Exit;
}
Bytes = EFI_PAGES_TO_SIZE (6);
Status = PciIo->Map (
PciIo,
EfiPciIoOperationBusMasterCommonBuffer,
Private->Buffer,
&Bytes,
&MappedAddr,
&Private->Mapping
);
if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (6))) {
goto Exit;
}
Private->BufferPciAddr = (UINT8 *)(UINTN)MappedAddr;
Private->Signature = NVME_CONTROLLER_PRIVATE_DATA_SIGNATURE;
Private->ControllerHandle = Controller;
Private->ImageHandle = This->DriverBindingHandle;
Private->DriverBindingHandle = This->DriverBindingHandle;
Private->PciIo = PciIo;
Private->ParentDevicePath = ParentDevicePath;
Private->Passthru.Mode = &Private->PassThruMode;
Private->Passthru.PassThru = NvmExpressPassThru;
Private->Passthru.GetNextNamespace = NvmExpressGetNextNamespace;
Private->Passthru.BuildDevicePath = NvmExpressBuildDevicePath;
Private->Passthru.GetNamespace = NvmExpressGetNamespace;
CopyMem (&Private->PassThruMode, &gEfiNvmExpressPassThruMode, sizeof (EFI_NVM_EXPRESS_PASS_THRU_MODE));
InitializeListHead (&Private->AsyncPassThruQueue);
InitializeListHead (&Private->UnsubmittedSubtasks);
Status = NvmeControllerInit (Private);
if (EFI_ERROR(Status)) {
goto Exit;
}
//
// Start the asynchronous I/O completion monitor
//
Status = gBS->CreateEvent (
EVT_TIMER | EVT_NOTIFY_SIGNAL,
TPL_NOTIFY,
ProcessAsyncTaskList,
Private,
&Private->TimerEvent
);
if (EFI_ERROR (Status)) {
goto Exit;
}
Status = gBS->SetTimer (
Private->TimerEvent,
TimerPeriodic,
NVME_HC_ASYNC_TIMER
);
if (EFI_ERROR (Status)) {
goto Exit;
}
Status = gBS->InstallMultipleProtocolInterfaces (
&Controller,
&gEfiNvmExpressPassThruProtocolGuid,
&Private->Passthru,
NULL
);
if (EFI_ERROR (Status)) {
goto Exit;
}
NvmeRegisterShutdownNotification ();
} else {
Status = gBS->OpenProtocol (
Controller,
&gEfiNvmExpressPassThruProtocolGuid,
(VOID **) &Passthru,
This->DriverBindingHandle,
Controller,
EFI_OPEN_PROTOCOL_GET_PROTOCOL
);
if (EFI_ERROR (Status)) {
goto Exit;
}
Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (Passthru);
}
if (RemainingDevicePath == NULL) {
//
// Enumerate all NVME namespaces in the controller
//
Status = DiscoverAllNamespaces (
Private
);
} else if (!IsDevicePathEnd (RemainingDevicePath)) {
//
// Enumerate the specified NVME namespace
//
Status = Private->Passthru.GetNamespace (
&Private->Passthru,
RemainingDevicePath,
&NamespaceId
);
if (!EFI_ERROR (Status)) {
Status = EnumerateNvmeDevNamespace (
Private,
NamespaceId
);
}
}
DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: end successfully\n"));
return EFI_SUCCESS;
Exit:
if ((Private != NULL) && (Private->Mapping != NULL)) {
PciIo->Unmap (PciIo, Private->Mapping);
}
if ((Private != NULL) && (Private->Buffer != NULL)) {
PciIo->FreeBuffer (PciIo, 6, Private->Buffer);
}
if ((Private != NULL) && (Private->ControllerData != NULL)) {
FreePool (Private->ControllerData);
}
if (Private != NULL) {
if (Private->TimerEvent != NULL) {
gBS->CloseEvent (Private->TimerEvent);
}
FreePool (Private);
}
gBS->CloseProtocol (
Controller,
&gEfiPciIoProtocolGuid,
This->DriverBindingHandle,
Controller
);
gBS->CloseProtocol (
Controller,
&gEfiDevicePathProtocolGuid,
This->DriverBindingHandle,
Controller
);
DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: end with %r\n", Status));
return Status;
}
/**
Sends an NVM Express Command Packet to an NVM Express controller or namespace. This function supports
both blocking I/O and non-blocking I/O. The blocking I/O functionality is required, and the non-blocking
I/O functionality is optional.
@param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
@param[in] NamespaceId A 32 bit namespace ID as defined in the NVMe specification to which the NVM Express Command
Packet will be sent. A value of 0 denotes the NVM Express controller, a value of all 0xFF's
(all bytes are 0xFF) in the namespace ID specifies that the command packet should be sent to
all valid namespaces.
@param[in,out] Packet A pointer to the NVM Express Command Packet.
@param[in] Event If non-blocking I/O is not supported then Event is ignored, and blocking I/O is performed.
If Event is NULL, then blocking I/O is performed. If Event is not NULL and non-blocking I/O
is supported, then non-blocking I/O is performed, and Event will be signaled when the NVM
Express Command Packet completes.
@retval EFI_SUCCESS The NVM Express Command Packet was sent by the host. TransferLength bytes were transferred
to, or from DataBuffer.
@retval EFI_BAD_BUFFER_SIZE The NVM Express Command Packet was not executed. The number of bytes that could be transferred
is returned in TransferLength.
@retval EFI_NOT_READY The NVM Express Command Packet could not be sent because the controller is not ready. The caller
may retry again later.
@retval EFI_DEVICE_ERROR A device error occurred while attempting to send the NVM Express Command Packet.
@retval EFI_INVALID_PARAMETER NamespaceId or the contents of EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET are invalid. The NVM
Express Command Packet was not sent, so no additional status information is available.
@retval EFI_UNSUPPORTED The command described by the NVM Express Command Packet is not supported by the NVM Express
controller. The NVM Express Command Packet was not sent so no additional status information
is available.
@retval EFI_TIMEOUT A timeout occurred while waiting for the NVM Express Command Packet to execute.
**/
EFI_STATUS
EFIAPI
NvmExpressPassThru (
IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *This,
IN UINT32 NamespaceId,
IN OUT EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET *Packet,
IN EFI_EVENT Event OPTIONAL
)
{
NVME_CONTROLLER_PRIVATE_DATA *Private;
EFI_STATUS Status;
EFI_PCI_IO_PROTOCOL *PciIo;
NVME_SQ *Sq;
NVME_CQ *Cq;
UINT16 QueueId;
UINT32 Bytes;
UINT16 Offset;
EFI_EVENT TimerEvent;
EFI_PCI_IO_PROTOCOL_OPERATION Flag;
EFI_PHYSICAL_ADDRESS PhyAddr;
VOID *MapData;
VOID *MapMeta;
VOID *MapPrpList;
UINTN MapLength;
UINT64 *Prp;
VOID *PrpListHost;
UINTN PrpListNo;
UINT32 Attributes;
UINT32 IoAlign;
UINT32 MaxTransLen;
UINT32 Data;
NVME_PASS_THRU_ASYNC_REQ *AsyncRequest;
EFI_TPL OldTpl;
//
// check the data fields in Packet parameter.
//
if ((This == NULL) || (Packet == NULL)) {
return EFI_INVALID_PARAMETER;
}
if ((Packet->NvmeCmd == NULL) || (Packet->NvmeCompletion == NULL)) {
return EFI_INVALID_PARAMETER;
}
if (Packet->QueueType != NVME_ADMIN_QUEUE && Packet->QueueType != NVME_IO_QUEUE) {
return EFI_INVALID_PARAMETER;
}
//
// 'Attributes' with neither EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL nor
// EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL set is an illegal
// configuration.
//
Attributes = This->Mode->Attributes;
if ((Attributes & (EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL |
EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL)) == 0) {
return EFI_INVALID_PARAMETER;
}
//
// Buffer alignment check for TransferBuffer & MetadataBuffer.
//
IoAlign = This->Mode->IoAlign;
if (IoAlign > 0 && (((UINTN) Packet->TransferBuffer & (IoAlign - 1)) != 0)) {
return EFI_INVALID_PARAMETER;
}
if (IoAlign > 0 && (((UINTN) Packet->MetadataBuffer & (IoAlign - 1)) != 0)) {
return EFI_INVALID_PARAMETER;
}
Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
//
// Check NamespaceId is valid or not.
//
if ((NamespaceId > Private->ControllerData->Nn) &&
(NamespaceId != (UINT32) -1)) {
return EFI_INVALID_PARAMETER;
}
//
// Check whether TransferLength exceeds the maximum data transfer size.
//
if (Private->ControllerData->Mdts != 0) {
MaxTransLen = (1 << (Private->ControllerData->Mdts)) *
(1 << (Private->Cap.Mpsmin + 12));
if (Packet->TransferLength > MaxTransLen) {
Packet->TransferLength = MaxTransLen;
return EFI_BAD_BUFFER_SIZE;
}
}
PciIo = Private->PciIo;
MapData = NULL;
MapMeta = NULL;
MapPrpList = NULL;
PrpListHost = NULL;
PrpListNo = 0;
Prp = NULL;
TimerEvent = NULL;
Status = EFI_SUCCESS;
if (Packet->QueueType == NVME_ADMIN_QUEUE) {
QueueId = 0;
} else {
if (Event == NULL) {
QueueId = 1;
} else {
QueueId = 2;
//
// Submission queue full check.
//
if ((Private->SqTdbl[QueueId].Sqt + 1) % (NVME_ASYNC_CSQ_SIZE + 1) ==
Private->AsyncSqHead) {
return EFI_NOT_READY;
}
}
}
Sq = Private->SqBuffer[QueueId] + Private->SqTdbl[QueueId].Sqt;
Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
if (Packet->NvmeCmd->Nsid != NamespaceId) {
return EFI_INVALID_PARAMETER;
}
ZeroMem (Sq, sizeof (NVME_SQ));
Sq->Opc = (UINT8)Packet->NvmeCmd->Cdw0.Opcode;
Sq->Fuse = (UINT8)Packet->NvmeCmd->Cdw0.FusedOperation;
Sq->Cid = Private->Cid[QueueId]++;
Sq->Nsid = Packet->NvmeCmd->Nsid;
//
// Currently we only support PRP for data transfer, SGL is NOT supported.
//
ASSERT (Sq->Psdt == 0);
if (Sq->Psdt != 0) {
DEBUG ((EFI_D_ERROR, "NvmExpressPassThru: doesn't support SGL mechanism\n"));
return EFI_UNSUPPORTED;
}
Sq->Prp[0] = (UINT64)(UINTN)Packet->TransferBuffer;
//
// If the NVMe cmd has data in or out, then mapping the user buffer to the PCI controller specific addresses.
// Note here we don't handle data buffer for CreateIOSubmitionQueue and CreateIOCompletionQueue cmds because
// these two cmds are special which requires their data buffer must support simultaneous access by both the
// processor and a PCI Bus Master. It's caller's responsbility to ensure this.
//
if (((Sq->Opc & (BIT0 | BIT1)) != 0) && (Sq->Opc != NVME_ADMIN_CRIOCQ_CMD) && (Sq->Opc != NVME_ADMIN_CRIOSQ_CMD)) {
if ((Packet->TransferLength == 0) || (Packet->TransferBuffer == NULL)) {
return EFI_INVALID_PARAMETER;
}
if ((Sq->Opc & BIT0) != 0) {
Flag = EfiPciIoOperationBusMasterRead;
} else {
Flag = EfiPciIoOperationBusMasterWrite;
}
MapLength = Packet->TransferLength;
Status = PciIo->Map (
PciIo,
Flag,
Packet->TransferBuffer,
&MapLength,
&PhyAddr,
&MapData
);
if (EFI_ERROR (Status) || (Packet->TransferLength != MapLength)) {
return EFI_OUT_OF_RESOURCES;
}
Sq->Prp[0] = PhyAddr;
Sq->Prp[1] = 0;
if((Packet->MetadataLength != 0) && (Packet->MetadataBuffer != NULL)) {
MapLength = Packet->MetadataLength;
Status = PciIo->Map (
PciIo,
Flag,
Packet->MetadataBuffer,
&MapLength,
&PhyAddr,
&MapMeta
);
if (EFI_ERROR (Status) || (Packet->MetadataLength != MapLength)) {
PciIo->Unmap (
PciIo,
MapData
);
return EFI_OUT_OF_RESOURCES;
}
Sq->Mptr = PhyAddr;
}
}
//
// If the buffer size spans more than two memory pages (page size as defined in CC.Mps),
// then build a PRP list in the second PRP submission queue entry.
//
Offset = ((UINT16)Sq->Prp[0]) & (EFI_PAGE_SIZE - 1);
Bytes = Packet->TransferLength;
if ((Offset + Bytes) > (EFI_PAGE_SIZE * 2)) {
//
// Create PrpList for remaining data buffer.
//
PhyAddr = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
Prp = NvmeCreatePrpList (PciIo, PhyAddr, EFI_SIZE_TO_PAGES(Offset + Bytes) - 1, &PrpListHost, &PrpListNo, &MapPrpList);
if (Prp == NULL) {
goto EXIT;
}
Sq->Prp[1] = (UINT64)(UINTN)Prp;
} else if ((Offset + Bytes) > EFI_PAGE_SIZE) {
Sq->Prp[1] = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
}
if(Packet->NvmeCmd->Flags & CDW2_VALID) {
Sq->Rsvd2 = (UINT64)Packet->NvmeCmd->Cdw2;
}
if(Packet->NvmeCmd->Flags & CDW3_VALID) {
Sq->Rsvd2 |= LShiftU64 ((UINT64)Packet->NvmeCmd->Cdw3, 32);
}
if(Packet->NvmeCmd->Flags & CDW10_VALID) {
Sq->Payload.Raw.Cdw10 = Packet->NvmeCmd->Cdw10;
}
if(Packet->NvmeCmd->Flags & CDW11_VALID) {
Sq->Payload.Raw.Cdw11 = Packet->NvmeCmd->Cdw11;
}
if(Packet->NvmeCmd->Flags & CDW12_VALID) {
Sq->Payload.Raw.Cdw12 = Packet->NvmeCmd->Cdw12;
}
if(Packet->NvmeCmd->Flags & CDW13_VALID) {
Sq->Payload.Raw.Cdw13 = Packet->NvmeCmd->Cdw13;
}
if(Packet->NvmeCmd->Flags & CDW14_VALID) {
Sq->Payload.Raw.Cdw14 = Packet->NvmeCmd->Cdw14;
}
if(Packet->NvmeCmd->Flags & CDW15_VALID) {
Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
}
//
// Ring the submission queue doorbell.
//
if ((Event != NULL) && (QueueId != 0)) {
Private->SqTdbl[QueueId].Sqt =
(Private->SqTdbl[QueueId].Sqt + 1) % (NVME_ASYNC_CSQ_SIZE + 1);
} else {
Private->SqTdbl[QueueId].Sqt ^= 1;
}
Data = ReadUnaligned32 ((UINT32*)&Private->SqTdbl[QueueId]);
Status = PciIo->Mem.Write (
PciIo,
EfiPciIoWidthUint32,
NVME_BAR,
NVME_SQTDBL_OFFSET(QueueId, Private->Cap.Dstrd),
1,
&Data
);
if (EFI_ERROR (Status)) {
goto EXIT;
}
//
// For non-blocking requests, return directly if the command is placed
// in the submission queue.
//
if ((Event != NULL) && (QueueId != 0)) {
AsyncRequest = AllocateZeroPool (sizeof (NVME_PASS_THRU_ASYNC_REQ));
if (AsyncRequest == NULL) {
Status = EFI_DEVICE_ERROR;
goto EXIT;
}
AsyncRequest->Signature = NVME_PASS_THRU_ASYNC_REQ_SIG;
AsyncRequest->Packet = Packet;
AsyncRequest->CommandId = Sq->Cid;
AsyncRequest->CallerEvent = Event;
AsyncRequest->MapData = MapData;
AsyncRequest->MapMeta = MapMeta;
AsyncRequest->MapPrpList = MapPrpList;
AsyncRequest->PrpListNo = PrpListNo;
AsyncRequest->PrpListHost = PrpListHost;
OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
InsertTailList (&Private->AsyncPassThruQueue, &AsyncRequest->Link);
gBS->RestoreTPL (OldTpl);
return EFI_SUCCESS;
}
Status = gBS->CreateEvent (
EVT_TIMER,
TPL_CALLBACK,
NULL,
NULL,
&TimerEvent
);
if (EFI_ERROR (Status)) {
goto EXIT;
}
Status = gBS->SetTimer(TimerEvent, TimerRelative, Packet->CommandTimeout);
if (EFI_ERROR(Status)) {
goto EXIT;
}
//
// Wait for completion queue to get filled in.
//
Status = EFI_TIMEOUT;
while (EFI_ERROR (gBS->CheckEvent (TimerEvent))) {
if (Cq->Pt != Private->Pt[QueueId]) {
Status = EFI_SUCCESS;
break;
}
}
//
// Check the NVMe cmd execution result
//
if (Status != EFI_TIMEOUT) {
if ((Cq->Sct == 0) && (Cq->Sc == 0)) {
Status = EFI_SUCCESS;
} else {
Status = EFI_DEVICE_ERROR;
//
// Copy the Respose Queue entry for this command to the callers response buffer
//
CopyMem(Packet->NvmeCompletion, Cq, sizeof(EFI_NVM_EXPRESS_COMPLETION));
//
// Dump every completion entry status for debugging.
//
DEBUG_CODE_BEGIN();
NvmeDumpStatus(Cq);
DEBUG_CODE_END();
}
} else {
//
// Timeout occurs for an NVMe command. Reset the controller to abort the
// outstanding commands.
//
DEBUG ((DEBUG_ERROR, "NvmExpressPassThru: Timeout occurs for an NVMe command.\n"));
//
// Disable the timer to trigger the process of async transfers temporarily.
//
Status = gBS->SetTimer (Private->TimerEvent, TimerCancel, 0);
if (EFI_ERROR (Status)) {
goto EXIT;
}
//
// Reset the NVMe controller.
//
Status = NvmeControllerInit (Private);
if (!EFI_ERROR (Status)) {
Status = AbortAsyncPassThruTasks (Private);
if (!EFI_ERROR (Status)) {
//
// Re-enable the timer to trigger the process of async transfers.
//
Status = gBS->SetTimer (Private->TimerEvent, TimerPeriodic, NVME_HC_ASYNC_TIMER);
if (!EFI_ERROR (Status)) {
//
// Return EFI_TIMEOUT to indicate a timeout occurs for NVMe PassThru command.
//
Status = EFI_TIMEOUT;
}
}
} else {
Status = EFI_DEVICE_ERROR;
}
goto EXIT;
}
if ((Private->CqHdbl[QueueId].Cqh ^= 1) == 0) {
Private->Pt[QueueId] ^= 1;
}
Data = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[QueueId]);
Status = PciIo->Mem.Write (
PciIo,
EfiPciIoWidthUint32,
NVME_BAR,
NVME_CQHDBL_OFFSET(QueueId, Private->Cap.Dstrd),
1,
&Data
);
//
// For now, the code does not support the non-blocking feature for admin queue.
// If Event is not NULL for admin queue, signal the caller's event here.
//
if (Event != NULL) {
ASSERT (QueueId == 0);
gBS->SignalEvent (Event);
}
EXIT:
if (MapData != NULL) {
PciIo->Unmap (
PciIo,
MapData
);
}
if (MapMeta != NULL) {
PciIo->Unmap (
PciIo,
MapMeta
);
}
if (MapPrpList != NULL) {
PciIo->Unmap (
PciIo,
MapPrpList
);
}
if (Prp != NULL) {
PciIo->FreeBuffer (PciIo, PrpListNo, PrpListHost);
}
if (TimerEvent != NULL) {
gBS->CloseEvent (TimerEvent);
}
return Status;
}
