VOID
ReleaseSlottedCommand(
_In_ PAHCI_CHANNEL_EXTENSION ChannelExtension,
_In_ UCHAR SlotNumber,
_In_ BOOLEAN AtDIRQL
)
/*++
Performs Completion Data marshaling back into the SRB
It assumes:
SRB STATUS has already been filled out
ATA status from the AHCI registers is valid for this command
SlotNumber points to a fully filled out Slot entry
Called by:
AhciHwStartIo
AhciNonQueuedErrorRecovery
AhciCompleteIssuedSRBs
It performs:
(overview)
1 Initialize
2 Marshal completion data
3 Complete the Command
(details)
1.1 Keep track of the completion for our debugging records
2.1 Handle Request Sense marshaling
2.2.1 Handle everything else's marshaling
2.2.2 If this was marked as needing return data, fill in the return Task File
3.1 Make the slot available again
3.2 Perform the Slot's last request, if there was one
3.3.1 IO is completing, that IO may have paused the queue so unpause the queue
3.3.2 Complete the command
Affected Variables/Registers:
SRB
--*/
{
PSLOT_CONTENT slotContent;
PAHCI_SRB_EXTENSION srbExtension;
PSTORAGE_REQUEST_BLOCK srbToRelease;
BOOLEAN isSenseSrb;
BOOLEAN retrySrb = FALSE;
slotContent = &ChannelExtension->Slot[SlotNumber];
if (slotContent->Srb == NULL) {
return;
}
srbExtension = GetSrbExtension(slotContent->Srb);
isSenseSrb = IsRequestSenseSrb(srbExtension->AtaFunction);
if (LogExecuteFullDetail(ChannelExtension->AdapterExtension->LogFlags)) {
RecordExecutionHistory(ChannelExtension, 0x10000053);//ReleaseSlottedCommand
}
//2/2 Log command completion part for our debugging records
if (LogCommand(ChannelExtension->AdapterExtension->LogFlags)) {
PCOMMAND_HISTORY cmdHistory;
slotContent->CommandHistoryIndex %= 64; // should not be 64 or bigger. do it anyway to make sure we are safe
cmdHistory = &ChannelExtension->CommandHistory[slotContent->CommandHistoryIndex];
StorPortCopyMemory((PVOID)&cmdHistory->CompletionFIS, (PVOID)&ChannelExtension->ReceivedFIS->D2hRegisterFis, sizeof(AHCI_D2H_REGISTER_FIS));
cmdHistory->CompletionPx[0] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->CLB.AsUlong);
cmdHistory->CompletionPx[1] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->CLBU);
cmdHistory->CompletionPx[2] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->FB.AsUlong);
cmdHistory->CompletionPx[3] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->FBU);
cmdHistory->CompletionPx[4] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->IS.AsUlong);
cmdHistory->CompletionPx[5] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->IE.AsUlong);
cmdHistory->CompletionPx[6] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->CMD.AsUlong);
cmdHistory->CompletionPx[7] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->DW7_Reserved);
cmdHistory->CompletionPx[8] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->TFD.AsUlong);
cmdHistory->CompletionPx[9] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SIG.AsUlong);
cmdHistory->CompletionPx[10] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SSTS.AsUlong);
cmdHistory->CompletionPx[11] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SCTL.AsUlong);
cmdHistory->CompletionPx[12] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SERR.AsUlong);
cmdHistory->CompletionPx[13] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SACT);
cmdHistory->CompletionPx[14] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->CI);
cmdHistory->CompletionPx[15] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SNTF.AsUlong);
cmdHistory->SrbStatus = slotContent->Srb->SrbStatus;
}
//2. Then complete the command
if (isSenseSrb) {
//2.1 Handle Request Sense marshaling
srbToRelease = (PSTORAGE_REQUEST_BLOCK)SrbGetOriginalRequest(slotContent->Srb);
//that original SRB must have SRB_STATUS_AUTOSENSE_VALID set appropriately
if (slotContent->Srb->SrbStatus == SRB_STATUS_SUCCESS) {
srbToRelease->SrbStatus |= SRB_STATUS_AUTOSENSE_VALID;
SrbSetScsiStatus(srbToRelease, SCSISTAT_CHECK_CONDITION);
} else {
srbToRelease->SrbStatus &= ~SRB_STATUS_AUTOSENSE_VALID;
}
//finish use of Sense.Srb
srbExtension->AtaFunction = 0;
} else {
//2.2.1 Handle everything else's marshaling
srbToRelease = slotContent->Srb;
if ((srbExtension->Flags & ATA_FLAGS_SENSEDATA_SET) == 0) {
//Record error and status
srbExtension->AtaStatus = ChannelExtension->TaskFileData.STS.AsUchar;
srbExtension->AtaError = ChannelExtension->TaskFileData.ERR;
//2.2.2 If this was marked as needing return data, fill in the return Task File.
if( IsReturnResults(srbExtension->Flags) ) {
SetReturnRegisterValues(ChannelExtension, srbToRelease, NULL);
}
// interpret ATA error to be SCSI error and fill SenseBuffer
// also log IO Record
if (IsAtaCommand(srbExtension->AtaFunction) ||
IsAtaCfisPayload(srbExtension->AtaFunction)) {
// IO Record for ATA device is logged in AtaMapError()
AtaMapError(ChannelExtension, srbToRelease, slotContent->StateFlags.FUA);
} else {
if (SRB_STATUS(srbToRelease->SrbStatus) == SRB_STATUS_SUCCESS) {
ChannelExtension->DeviceExtension[0].IoRecord.SuccessCount++;
} else {
ChannelExtension->DeviceExtension[0].IoRecord.OtherErrorCount++;
}
}
}
}
//3.1 Make the slot available again
slotContent->CmdHeader = NULL;
slotContent->CommandHistoryIndex = 0;
slotContent->Srb = NULL;
slotContent->StateFlags.FUA = FALSE;
//Clear the CommandsToComplete bit
ChannelExtension->SlotManager.CommandsToComplete &= ~(1 << SlotNumber);
ChannelExtension->SlotManager.HighPriorityAttribute &= ~(1 << SlotNumber);
//3.3.1 IO is completing, that IO may have paused the queue so unpause the queue
// During NCQ Error Recovery process, this flag will be reset in the error recovery completion process, either through reset or through slot release again.
if (ChannelExtension->StateFlags.NcqErrorRecoveryInProcess == 0) {
ChannelExtension->StateFlags.QueuePaused = FALSE;
}
//3.3.2 Complete the command
if ( (srbToRelease->SrbStatus != SRB_STATUS_SUCCESS) && !isSenseSrb && (srbExtension->RetryCount == 0) ) {
if ( (ChannelExtension->StateFlags.HybridInfoEnabledOnHiberFile == 1) &&
IsNCQWriteCommand(srbExtension) ) {
// NCQ WRITE with Hybrid Information failed, retry once without Hybrid Information.
ChannelExtension->StateFlags.HybridInfoEnabledOnHiberFile = 0;
retrySrb = TRUE;
} else if (srbToRelease->SrbStatus == SRB_STATUS_PARITY_ERROR) {
// if a command encounters CRC error, retry once
retrySrb = TRUE;
}
}
if (retrySrb) {
srbToRelease->SrbStatus = SRB_STATUS_PENDING;
srbExtension->RetryCount++;
AhciProcessIo(ChannelExtension, srbToRelease, AtDIRQL);
} else {
// otherwise, complete it.
AhciCompleteRequest(ChannelExtension, srbToRelease, AtDIRQL);
}
}
VOID
AhciCompleteJustSlottedRequest(
_In_ PAHCI_CHANNEL_EXTENSION ChannelExtension,
_In_ PSTORAGE_REQUEST_BLOCK Srb,
_In_ BOOLEAN AtDIRQL
)
/*
This routine is to complete a request that occupies a slot but has not been issued to adapter yet.
Srb status should be set before calling this routine.
*/
{
PSLOT_CONTENT slotContent;
PAHCI_SRB_EXTENSION srbExtension;
BOOLEAN isSenseSrb;
PSTORAGE_REQUEST_BLOCK srbToComplete;
srbExtension = GetSrbExtension(Srb);
slotContent = &ChannelExtension->Slot[srbExtension->QueueTag];
NT_ASSERT(slotContent->Srb == Srb);
isSenseSrb = IsRequestSenseSrb(srbExtension->AtaFunction);
if (isSenseSrb) {
// Handle Request Sense marshaling
srbToComplete = (PSTORAGE_REQUEST_BLOCK)SrbGetOriginalRequest(Srb);
//Sense Srb doesn't have chance to run yet, clear Sense Valid flag from original SRB.
srbToComplete->SrbStatus &= ~SRB_STATUS_AUTOSENSE_VALID;
//finish use of Sense.Srb
srbExtension->AtaFunction = 0;
} else {
srbToComplete = Srb;
}
//1. Make the slot available again
slotContent->CmdHeader = NULL;
slotContent->CommandHistoryIndex = 0;
slotContent->Srb = NULL;
slotContent->StateFlags.FUA = FALSE;
//2. this function can be called from places that don't call ActivateQueue yet. Clear the bit in IO Slices
if ((ChannelExtension->SlotManager.HighPriorityAttribute & (1 << srbExtension->QueueTag)) != 0) {
ChannelExtension->SlotManager.HighPriorityAttribute &= ~(1 << srbExtension->QueueTag);
}
if ((ChannelExtension->SlotManager.NCQueueSlice & (1 << srbExtension->QueueTag)) != 0) {
ChannelExtension->SlotManager.NCQueueSlice &= ~(1 << srbExtension->QueueTag);
}
if ((ChannelExtension->SlotManager.NormalQueueSlice & (1 << srbExtension->QueueTag)) != 0) {
ChannelExtension->SlotManager.NormalQueueSlice &= ~(1 << srbExtension->QueueTag);
}
if ((ChannelExtension->SlotManager.SingleIoSlice & (1 << srbExtension->QueueTag)) != 0) {
ChannelExtension->SlotManager.SingleIoSlice &= ~(1 << srbExtension->QueueTag);
}
if ((ChannelExtension->SlotManager.CommandsToComplete & (1 << srbExtension->QueueTag)) != 0) {
ChannelExtension->SlotManager.CommandsToComplete &= ~(1 << srbExtension->QueueTag);
}
//3. Complete the command
AhciCompleteRequest(ChannelExtension, srbToComplete, AtDIRQL);
return;
}
VOID
ReleaseSlottedCommand(
_In_ PAHCI_CHANNEL_EXTENSION ChannelExtension,
_In_ UCHAR SlotNumber,
_In_ BOOLEAN AtDIRQL
)
/*++
Performs Completion Data marshalling back into the SRB
It assumes:
SRB STATUS has already been filled out
ATA status from the AHCI registers is valid for this command
SlotNumber points to a fully filled out Slot entry
Called by:
AhciHwStartIo
AhciNonQueuedErrorRecovery
AhciCompleteIssuedSRBs
It performs:
(overview)
1 Intialize
2 Marshal completion data
3 Complete the Command
(details)
1.1 Keep track of the completion for our debugging records
2.1 Handle Request Sense marshalling
2.2.1 Handle everything else's marshalling
2.2.2 If this was marked as needing return data, fill in the return Task File
3.1 Make the slot available again
3.2 Perform the Slot's last request, if there was one
3.3.1 IO is completing, that IO may have paused the queue so unpause the queue
3.3.2 Complete the command
Affected Variables/Registers:
SRB
--*/
{
PSLOT_CONTENT slotContent;
PAHCI_SRB_EXTENSION srbExtension;
PSTORAGE_REQUEST_BLOCK srbToRelease;
PATA_TASK_FILE returnTaskFile;
BOOLEAN isSenseSrb;
BOOLEAN retrySrb = FALSE;
slotContent = &ChannelExtension->Slot[SlotNumber];
if (slotContent->Srb == NULL) {
return;
}
srbExtension = GetSrbExtension(slotContent->Srb);
isSenseSrb = IsRequestSenseSrb(srbExtension->AtaFunction);
if (LogExecuteFullDetail(ChannelExtension->AdapterExtension->LogFlags)) {
RecordExecutionHistory(ChannelExtension, 0x10000053);//ReleaseSlottedCommand
}
//2/2 Log command completion part for our debugging records
if (LogCommand(ChannelExtension->AdapterExtension->LogFlags)) {
PCOMMAND_HISTORY cmdHistory;
slotContent->CommandHistoryIndex %= 64; // should not be 64 or bigger. do it anyway to make sure we are safe
cmdHistory = &ChannelExtension->CommandHistory[slotContent->CommandHistoryIndex];
StorPortCopyMemory((PVOID)&cmdHistory->CompletionFIS, (PVOID)&ChannelExtension->ReceivedFIS->D2hRegisterFis, sizeof(AHCI_D2H_REGISTER_FIS));
StorPortCopyMemory((PVOID)&cmdHistory->CompletionPx, (PVOID)ChannelExtension->Px, 0x40);
cmdHistory->SrbStatus = slotContent->Srb->SrbStatus;
}
//2. Then complete the command
if (isSenseSrb) {
//2.1 Handle Request Sense marshalling
srbToRelease = (PSTORAGE_REQUEST_BLOCK)SrbGetOriginalRequest(slotContent->Srb);
//that original SRB must have SRB_STATUS_AUTOSENSE_VALID set appropriately
if (slotContent->Srb->SrbStatus == SRB_STATUS_SUCCESS) {
srbToRelease->SrbStatus |= SRB_STATUS_AUTOSENSE_VALID;
SrbSetScsiStatus(srbToRelease, SCSISTAT_CHECK_CONDITION);
} else {
srbToRelease->SrbStatus &= ~SRB_STATUS_AUTOSENSE_VALID;
}
//finish use of Sense.Srb
srbExtension->AtaFunction = 0;
} else {
//2.2.1 Handle everything else's marshalling
srbToRelease = slotContent->Srb;
//Record error and status
srbExtension->AtaStatus = ChannelExtension->TaskFileData.STS.AsUchar;
srbExtension->AtaError = ChannelExtension->TaskFileData.ERR;
//2.2.2 If this was marked as needing return data, fill in the return Task File.
if( IsReturnResults(srbExtension->Flags) ) {
PCDB cdb = SrbGetCdb(srbToRelease);
if ((cdb != NULL) && (cdb->CDB10.OperationCode == SCSIOP_ATA_PASSTHROUGH16)) {
// for ATA PASS THROUGH 16 command, return Descriptor Format Sense Data, including ATA Status Return info.
if (srbExtension->ResultBufferLength >= sizeof(DESCRIPTOR_SENSE_DATA)) {
PDESCRIPTOR_SENSE_DATA descriptorSenseData = (PDESCRIPTOR_SENSE_DATA)srbExtension->ResultBuffer;
PSCSI_SENSE_DESCRIPTOR_ATA_STATUS_RETURN ataStatus = (PSCSI_SENSE_DESCRIPTOR_ATA_STATUS_RETURN)((PUCHAR)descriptorSenseData + FIELD_OFFSET(DESCRIPTOR_SENSE_DATA, DescriptorBuffer));
AhciZeroMemory((PCHAR)srbExtension->ResultBuffer, srbExtension->ResultBufferLength);
// fill sense data header, leave SenseKey, ASC, ASCQ as zero.
descriptorSenseData->ErrorCode = SCSI_SENSE_ERRORCODE_DESCRIPTOR_CURRENT;
descriptorSenseData->AdditionalSenseLength = sizeof(SCSI_SENSE_DESCRIPTOR_ATA_STATUS_RETURN);
// fill ATA Status Return Info.
ataStatus->Header.DescriptorType = SCSI_SENSE_DESCRIPTOR_TYPE_ATA_STATUS_RETURN;
ataStatus->Header.AdditionalLength = 0x0C;
ataStatus->Extend = Is48BitCommand(srbExtension->Flags) ? 1 : 0;
ataStatus->Error = ChannelExtension->ReceivedFIS->D2hRegisterFis.Error;
ataStatus->SectorCount7_0 = ChannelExtension->ReceivedFIS->D2hRegisterFis.SectorCount;
ataStatus->LbaLow7_0 = ChannelExtension->ReceivedFIS->D2hRegisterFis.SectorNumber;
ataStatus->LbaMid7_0 = ChannelExtension->ReceivedFIS->D2hRegisterFis.CylLow;
ataStatus->LbaHigh7_0 = ChannelExtension->ReceivedFIS->D2hRegisterFis.CylHigh;
ataStatus->Device = ChannelExtension->ReceivedFIS->D2hRegisterFis.Dev_Head;
ataStatus->Status = ChannelExtension->ReceivedFIS->D2hRegisterFis.Status;
if (Is48BitCommand(srbExtension->Flags)) {
ataStatus->SectorCount15_8 = ChannelExtension->ReceivedFIS->D2hRegisterFis.SectorCount_Exp;
ataStatus->LbaLow15_8 = ChannelExtension->ReceivedFIS->D2hRegisterFis.SectorNum_Exp;
ataStatus->LbaMid15_8 = ChannelExtension->ReceivedFIS->D2hRegisterFis.CylLow_Exp;
ataStatus->LbaHigh15_8 = ChannelExtension->ReceivedFIS->D2hRegisterFis.CylHigh_Exp;
}
} else {
NT_ASSERT(FALSE);
}
} else {
returnTaskFile = (PATA_TASK_FILE)srbExtension->ResultBuffer;
returnTaskFile->Current.bCommandReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.Status;
returnTaskFile->Current.bFeaturesReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.Error;
returnTaskFile->Current.bCylHighReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.CylHigh;
returnTaskFile->Current.bCylLowReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.CylLow;
returnTaskFile->Current.bDriveHeadReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.Dev_Head;
returnTaskFile->Current.bSectorCountReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.SectorCount;
returnTaskFile->Current.bSectorNumberReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.SectorNumber;
//if 48bit, get all of it
if(Is48BitCommand(srbExtension->Flags) && (srbExtension->ResultBufferLength >= sizeof(ATA_TASK_FILE))) {
returnTaskFile->Previous.bCommandReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.Status;
returnTaskFile->Previous.bFeaturesReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.Error;
returnTaskFile->Previous.bCylHighReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.CylHigh_Exp;
returnTaskFile->Previous.bCylLowReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.CylLow_Exp;
returnTaskFile->Previous.bDriveHeadReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.Dev_Head;
returnTaskFile->Previous.bSectorCountReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.SectorCount_Exp;
returnTaskFile->Previous.bSectorNumberReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.SectorNum_Exp;
}
}
// set flag SRB_STATUS_AUTOSENSE_VALID so that Storport will copy it back to original Sense Buffer
srbToRelease->SrbStatus |= SRB_STATUS_AUTOSENSE_VALID;
}
// interpret ATA error to be SCSI error and fill SenseBuffer
// also log IO Record
if (IsAtaCommand(srbExtension->AtaFunction) ||
IsAtaCfisPayload(srbExtension->AtaFunction)) {
// IO Record for ATA device is logged in AtaMapError()
AtaMapError(ChannelExtension, srbToRelease, slotContent->StateFlags.FUA);
} else {
if (SRB_STATUS(srbToRelease->SrbStatus) == SRB_STATUS_SUCCESS) {
ChannelExtension->DeviceExtension[0].IoRecord.SuccessCount++;
} else {
ChannelExtension->DeviceExtension[0].IoRecord.OtherErrorCount++;
}
}
}
//3.1 Make the slot available again
slotContent->CmdHeader = NULL;
slotContent->CommandHistoryIndex = 0;
slotContent->Srb = NULL;
slotContent->StateFlags.FUA = FALSE;
//Clear the CommandsToComplete bit
ChannelExtension->SlotManager.CommandsToComplete &= ~(1 << SlotNumber);
ChannelExtension->SlotManager.HighPriorityAttribute &= ~(1 << SlotNumber);
//3.3.1 IO is completing, that IO may have paused the queue so unpause the queue
ChannelExtension->StateFlags.QueuePaused = FALSE;
//3.3.2 Complete the command
if ( (srbToRelease->SrbStatus != SRB_STATUS_SUCCESS) && !isSenseSrb && (srbExtension->RetryCount == 0) ) {
if ( (ChannelExtension->StateFlags.HybridInfoEnabledOnHiberFile == 1) &&
IsNCQWriteCommand(srbExtension) ) {
// NCQ WRITE with Hybrid Information failed, retry once without Hybrid Information.
ChannelExtension->StateFlags.HybridInfoEnabledOnHiberFile = 0;
retrySrb = TRUE;
} else if (srbToRelease->SrbStatus == SRB_STATUS_PARITY_ERROR) {
// if a command encounters CRC error, retry once
retrySrb = TRUE;
}
}
if (retrySrb) {
srbToRelease->SrbStatus = SRB_STATUS_PENDING;
srbExtension->RetryCount++;
AhciProcessIo(ChannelExtension, srbToRelease, AtDIRQL);
} else {
// otherwise, complete it.
AhciCompleteRequest(ChannelExtension, srbToRelease, AtDIRQL);
}
}
