Exemplo n.º 1
0
VOID
GetAvailableSlot(
    PAHCI_CHANNEL_EXTENSION ChannelExtension,
    PSTORAGE_REQUEST_BLOCK  Srb
    )
/*++
    Returns the next available slot that can be used for an ATA command in the QueueTag field of the SRB
It assumes:

Called by:

It performs:
    1.1 Initialize variables
    2.1 Special case the slot for the local SRB
    2.2 Chose the slot circularly starting with CurrentCommandSlot
    3.1 Update CurrentCommandSlot

Affected Variables/Registers:

Return Value:
    Next available slot number.  If SRB is localSRB then slot is 0.
    If no slot is available tag returned is 0xFF.
--*/
{
    ULONG               allocated;
    UCHAR               limit;
    UCHAR               i;
    PAHCI_SRB_EXTENSION srbExtension;

    srbExtension = GetSrbExtension(Srb);

  //1.1 Initialize variables
    srbExtension->QueueTag = 0xFF;
    limit = ChannelExtension->CurrentCommandSlot;

    allocated = GetOccupiedSlots(ChannelExtension);

  //2.1 Use slot 0 for internal commands, don't increment CCS
    if (Srb == (PSTORAGE_REQUEST_BLOCK)&ChannelExtension->Local.Srb ) {
        if ((allocated & (1 << 0)) > 0) {
            srbExtension->QueueTag = 0xFF;
        } else {
            srbExtension->QueueTag = 0;
        }
        return;
    }

  //2.2 Chose the slot circularly starting with CCS
    for (i = limit; i <= ChannelExtension->AdapterExtension->CAP.NCS; i++) {
        if ( (allocated & (1 << i)) == 0 ) {
            srbExtension->QueueTag = i;
            goto getout;
        }
    }

    for (i = 1; i < limit; i++) {
        if ( (allocated & (1 << i)) == 0 ) {
            srbExtension->QueueTag = i;
            goto getout;
        }
    }

getout:
  //3.1 Update CurrentCommandSlot
    if (IsRequestSenseSrb(srbExtension->AtaFunction)) {
      //If this SRB is for Request Sense, make sure it is given the next chance to run during ActivateQueue by not incrementing CCS.
        return;
    }

    ChannelExtension->CurrentCommandSlot++;
    if (ChannelExtension->CurrentCommandSlot == (ChannelExtension->AdapterExtension->CAP.NCS + 1)) {
        ChannelExtension->CurrentCommandSlot = 1;
    }

    return;
}
Exemplo n.º 2
0
VOID
ReleaseSlottedCommand(
    _In_ PAHCI_CHANNEL_EXTENSION ChannelExtension,
    _In_ UCHAR SlotNumber,
    _In_ BOOLEAN AtDIRQL
    )
/*++
    Performs Completion Data marshaling back into the SRB
It assumes:
    SRB STATUS has already been filled out
    ATA status from the AHCI registers is valid for this command
    SlotNumber points to a fully filled out Slot entry
Called by:
    AhciHwStartIo
    AhciNonQueuedErrorRecovery
    AhciCompleteIssuedSRBs

It performs:
    (overview)
    1 Initialize
    2 Marshal completion data
    3 Complete the Command
    (details)
    1.1 Keep track of the completion for our debugging records
    2.1 Handle Request Sense marshaling
    2.2.1 Handle everything else's marshaling
    2.2.2 If this was marked as needing return data, fill in the return Task File
    3.1 Make the slot available again
    3.2 Perform the Slot's last request, if there was one
    3.3.1 IO is completing, that IO may have paused the queue so unpause the queue
    3.3.2  Complete the command
Affected Variables/Registers:
    SRB
--*/
{
    PSLOT_CONTENT           slotContent;
    PAHCI_SRB_EXTENSION     srbExtension;
    PSTORAGE_REQUEST_BLOCK  srbToRelease;
    BOOLEAN                 isSenseSrb;
    BOOLEAN                 retrySrb = FALSE;

    slotContent = &ChannelExtension->Slot[SlotNumber];
    if (slotContent->Srb == NULL) {
        return;
    }

    srbExtension = GetSrbExtension(slotContent->Srb);
    isSenseSrb = IsRequestSenseSrb(srbExtension->AtaFunction);

    if (LogExecuteFullDetail(ChannelExtension->AdapterExtension->LogFlags)) {
        RecordExecutionHistory(ChannelExtension, 0x10000053);//ReleaseSlottedCommand
    }

  //2/2 Log command completion part for our debugging records
    if (LogCommand(ChannelExtension->AdapterExtension->LogFlags)) {
        PCOMMAND_HISTORY         cmdHistory;
        slotContent->CommandHistoryIndex %= 64; // should not be 64 or bigger. do it anyway to make sure we are safe

        cmdHistory = &ChannelExtension->CommandHistory[slotContent->CommandHistoryIndex];
        StorPortCopyMemory((PVOID)&cmdHistory->CompletionFIS, (PVOID)&ChannelExtension->ReceivedFIS->D2hRegisterFis, sizeof(AHCI_D2H_REGISTER_FIS));

        cmdHistory->CompletionPx[0] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->CLB.AsUlong);
        cmdHistory->CompletionPx[1] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->CLBU);
        cmdHistory->CompletionPx[2] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->FB.AsUlong);
        cmdHistory->CompletionPx[3] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->FBU);

        cmdHistory->CompletionPx[4] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->IS.AsUlong);
        cmdHistory->CompletionPx[5] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->IE.AsUlong);
        cmdHistory->CompletionPx[6] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->CMD.AsUlong);
        cmdHistory->CompletionPx[7] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->DW7_Reserved);

        cmdHistory->CompletionPx[8] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->TFD.AsUlong);
        cmdHistory->CompletionPx[9] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SIG.AsUlong);
        cmdHistory->CompletionPx[10] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SSTS.AsUlong);
        cmdHistory->CompletionPx[11] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SCTL.AsUlong);

        cmdHistory->CompletionPx[12] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SERR.AsUlong);
        cmdHistory->CompletionPx[13] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SACT);
        cmdHistory->CompletionPx[14] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->CI);
        cmdHistory->CompletionPx[15] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SNTF.AsUlong);

        cmdHistory->SrbStatus = slotContent->Srb->SrbStatus;

    }

  //2. Then complete the command
    if (isSenseSrb) {
      //2.1 Handle Request Sense marshaling
        srbToRelease = (PSTORAGE_REQUEST_BLOCK)SrbGetOriginalRequest(slotContent->Srb);

        //that original SRB must have SRB_STATUS_AUTOSENSE_VALID set appropriately
        if (slotContent->Srb->SrbStatus == SRB_STATUS_SUCCESS) {
            srbToRelease->SrbStatus |= SRB_STATUS_AUTOSENSE_VALID;
            SrbSetScsiStatus(srbToRelease, SCSISTAT_CHECK_CONDITION);
        } else {
            srbToRelease->SrbStatus &= ~SRB_STATUS_AUTOSENSE_VALID;
        }

        //finish use of Sense.Srb
        srbExtension->AtaFunction = 0;
    } else {
      //2.2.1 Handle everything else's marshaling
        srbToRelease = slotContent->Srb;

        if ((srbExtension->Flags & ATA_FLAGS_SENSEDATA_SET) == 0) {
           //Record error and status
            srbExtension->AtaStatus = ChannelExtension->TaskFileData.STS.AsUchar;
            srbExtension->AtaError = ChannelExtension->TaskFileData.ERR;

          //2.2.2 If this was marked as needing return data, fill in the return Task File.
            if( IsReturnResults(srbExtension->Flags) ) {
                SetReturnRegisterValues(ChannelExtension, srbToRelease, NULL);
            }

            // interpret ATA error to be SCSI error and fill SenseBuffer
            // also log IO Record
            if (IsAtaCommand(srbExtension->AtaFunction) ||
                IsAtaCfisPayload(srbExtension->AtaFunction)) {
                // IO Record for ATA device is logged in AtaMapError()
                AtaMapError(ChannelExtension, srbToRelease, slotContent->StateFlags.FUA);
            } else {
                if (SRB_STATUS(srbToRelease->SrbStatus) == SRB_STATUS_SUCCESS) {
                    ChannelExtension->DeviceExtension[0].IoRecord.SuccessCount++;
                } else {
                    ChannelExtension->DeviceExtension[0].IoRecord.OtherErrorCount++;
                }
            }
        }
    }


  //3.1 Make the slot available again
    slotContent->CmdHeader = NULL;
    slotContent->CommandHistoryIndex = 0;
    slotContent->Srb = NULL;
    slotContent->StateFlags.FUA = FALSE;

    //Clear the CommandsToComplete bit
    ChannelExtension->SlotManager.CommandsToComplete &= ~(1 << SlotNumber);
    ChannelExtension->SlotManager.HighPriorityAttribute &= ~(1 << SlotNumber);

  //3.3.1 IO is completing, that IO may have paused the queue so unpause the queue
  //      During NCQ Error Recovery process, this flag will be reset in the error recovery completion process, either through reset or through slot release again.
    if (ChannelExtension->StateFlags.NcqErrorRecoveryInProcess == 0) {
        ChannelExtension->StateFlags.QueuePaused = FALSE;
    }

  //3.3.2 Complete the command
    if ( (srbToRelease->SrbStatus != SRB_STATUS_SUCCESS) && !isSenseSrb && (srbExtension->RetryCount == 0) ) {
        if ( (ChannelExtension->StateFlags.HybridInfoEnabledOnHiberFile == 1) && 
             IsNCQWriteCommand(srbExtension) ) {
            // NCQ WRITE with Hybrid Information failed, retry once without Hybrid Information.
            ChannelExtension->StateFlags.HybridInfoEnabledOnHiberFile = 0;
            retrySrb = TRUE;
        } else if (srbToRelease->SrbStatus == SRB_STATUS_PARITY_ERROR) {
            // if a command encounters CRC error, retry once
            retrySrb = TRUE;
        }
    }

    if (retrySrb) {
        srbToRelease->SrbStatus = SRB_STATUS_PENDING;
        srbExtension->RetryCount++;
        AhciProcessIo(ChannelExtension, srbToRelease, AtDIRQL);
    } else {
        // otherwise, complete it.
        AhciCompleteRequest(ChannelExtension, srbToRelease, AtDIRQL);
    }
}
Exemplo n.º 3
0
VOID
AhciCompleteJustSlottedRequest(
    _In_ PAHCI_CHANNEL_EXTENSION ChannelExtension,
    _In_ PSTORAGE_REQUEST_BLOCK  Srb,
    _In_ BOOLEAN AtDIRQL
    )
/*
    This routine is to complete a request that occupies a slot but has not been issued to adapter yet.
    Srb status should be set before calling this routine.
*/
{
    PSLOT_CONTENT           slotContent;
    PAHCI_SRB_EXTENSION     srbExtension;
    BOOLEAN                 isSenseSrb;
    PSTORAGE_REQUEST_BLOCK  srbToComplete;

    srbExtension = GetSrbExtension(Srb);
    slotContent = &ChannelExtension->Slot[srbExtension->QueueTag];

    NT_ASSERT(slotContent->Srb == Srb);

    isSenseSrb = IsRequestSenseSrb(srbExtension->AtaFunction);

    if (isSenseSrb) {
      // Handle Request Sense marshaling
        srbToComplete = (PSTORAGE_REQUEST_BLOCK)SrbGetOriginalRequest(Srb);

        //Sense Srb doesn't have chance to run yet, clear Sense Valid flag from original SRB.
        srbToComplete->SrbStatus &= ~SRB_STATUS_AUTOSENSE_VALID;

        //finish use of Sense.Srb
        srbExtension->AtaFunction = 0;
    } else {
        srbToComplete = Srb;
    }

  //1. Make the slot available again
    slotContent->CmdHeader = NULL;
    slotContent->CommandHistoryIndex = 0;
    slotContent->Srb = NULL;
    slotContent->StateFlags.FUA = FALSE;

  //2. this function can be called from places that don't call ActivateQueue yet. Clear the bit in IO Slices
    if ((ChannelExtension->SlotManager.HighPriorityAttribute & (1 << srbExtension->QueueTag)) != 0) {
        ChannelExtension->SlotManager.HighPriorityAttribute &= ~(1 << srbExtension->QueueTag);
    }
    if ((ChannelExtension->SlotManager.NCQueueSlice & (1 << srbExtension->QueueTag)) != 0) {
        ChannelExtension->SlotManager.NCQueueSlice &= ~(1 << srbExtension->QueueTag);
    }
    if ((ChannelExtension->SlotManager.NormalQueueSlice & (1 << srbExtension->QueueTag)) != 0) {
        ChannelExtension->SlotManager.NormalQueueSlice &= ~(1 << srbExtension->QueueTag);
    }
    if ((ChannelExtension->SlotManager.SingleIoSlice & (1 << srbExtension->QueueTag)) != 0) {
        ChannelExtension->SlotManager.SingleIoSlice &= ~(1 << srbExtension->QueueTag);
    }
    if ((ChannelExtension->SlotManager.CommandsToComplete & (1 << srbExtension->QueueTag)) != 0) {
        ChannelExtension->SlotManager.CommandsToComplete &= ~(1 << srbExtension->QueueTag);
    }

  //3. Complete the command
    AhciCompleteRequest(ChannelExtension, srbToComplete, AtDIRQL);

    return;
}
Exemplo n.º 4
0
VOID
ReleaseSlottedCommand(
    _In_ PAHCI_CHANNEL_EXTENSION ChannelExtension,
    _In_ UCHAR SlotNumber,
    _In_ BOOLEAN AtDIRQL
    )
/*++
    Performs Completion Data marshalling back into the SRB
It assumes:
    SRB STATUS has already been filled out
    ATA status from the AHCI registers is valid for this command
    SlotNumber points to a fully filled out Slot entry
Called by:
    AhciHwStartIo
    AhciNonQueuedErrorRecovery
    AhciCompleteIssuedSRBs

It performs:
    (overview)
    1 Intialize
    2 Marshal completion data
    3 Complete the Command
    (details)
    1.1 Keep track of the completion for our debugging records
    2.1 Handle Request Sense marshalling
    2.2.1 Handle everything else's marshalling
    2.2.2 If this was marked as needing return data, fill in the return Task File
    3.1 Make the slot available again
    3.2 Perform the Slot's last request, if there was one
    3.3.1 IO is completing, that IO may have paused the queue so unpause the queue
    3.3.2  Complete the command
Affected Variables/Registers:
    SRB
--*/
{
    PSLOT_CONTENT           slotContent;
    PAHCI_SRB_EXTENSION     srbExtension;
    PSTORAGE_REQUEST_BLOCK  srbToRelease;
    PATA_TASK_FILE          returnTaskFile;
    BOOLEAN                 isSenseSrb;
    BOOLEAN                 retrySrb = FALSE;

    slotContent = &ChannelExtension->Slot[SlotNumber];
    if (slotContent->Srb == NULL) {
        return;
    }

    srbExtension = GetSrbExtension(slotContent->Srb);
    isSenseSrb = IsRequestSenseSrb(srbExtension->AtaFunction);

    if (LogExecuteFullDetail(ChannelExtension->AdapterExtension->LogFlags)) {
        RecordExecutionHistory(ChannelExtension, 0x10000053);//ReleaseSlottedCommand
    }

  //2/2 Log command completion part for our debugging records
    if (LogCommand(ChannelExtension->AdapterExtension->LogFlags)) {
        PCOMMAND_HISTORY         cmdHistory;
        slotContent->CommandHistoryIndex %= 64; // should not be 64 or bigger. do it anyway to make sure we are safe

        cmdHistory = &ChannelExtension->CommandHistory[slotContent->CommandHistoryIndex];
        StorPortCopyMemory((PVOID)&cmdHistory->CompletionFIS, (PVOID)&ChannelExtension->ReceivedFIS->D2hRegisterFis, sizeof(AHCI_D2H_REGISTER_FIS));
        StorPortCopyMemory((PVOID)&cmdHistory->CompletionPx, (PVOID)ChannelExtension->Px, 0x40);
        cmdHistory->SrbStatus = slotContent->Srb->SrbStatus;

    }

  //2. Then complete the command
    if (isSenseSrb) {
      //2.1 Handle Request Sense marshalling
        srbToRelease = (PSTORAGE_REQUEST_BLOCK)SrbGetOriginalRequest(slotContent->Srb);

        //that original SRB must have SRB_STATUS_AUTOSENSE_VALID set appropriately
        if (slotContent->Srb->SrbStatus == SRB_STATUS_SUCCESS) {
            srbToRelease->SrbStatus |= SRB_STATUS_AUTOSENSE_VALID;
            SrbSetScsiStatus(srbToRelease, SCSISTAT_CHECK_CONDITION);
        } else {
            srbToRelease->SrbStatus &= ~SRB_STATUS_AUTOSENSE_VALID;
        }

        //finish use of Sense.Srb
        srbExtension->AtaFunction = 0;
    } else {
      //2.2.1 Handle everything else's marshalling
        srbToRelease = slotContent->Srb;

        //Record error and status
        srbExtension->AtaStatus = ChannelExtension->TaskFileData.STS.AsUchar;
        srbExtension->AtaError = ChannelExtension->TaskFileData.ERR;

        //2.2.2 If this was marked as needing return data, fill in the return Task File.
        if( IsReturnResults(srbExtension->Flags) ) {
            PCDB cdb = SrbGetCdb(srbToRelease);

            if ((cdb != NULL) && (cdb->CDB10.OperationCode == SCSIOP_ATA_PASSTHROUGH16)) {
                // for ATA PASS THROUGH 16 command, return Descriptor Format Sense Data, including ATA Status Return info.
                if (srbExtension->ResultBufferLength >= sizeof(DESCRIPTOR_SENSE_DATA)) {
                    PDESCRIPTOR_SENSE_DATA descriptorSenseData = (PDESCRIPTOR_SENSE_DATA)srbExtension->ResultBuffer;
                    PSCSI_SENSE_DESCRIPTOR_ATA_STATUS_RETURN ataStatus = (PSCSI_SENSE_DESCRIPTOR_ATA_STATUS_RETURN)((PUCHAR)descriptorSenseData + FIELD_OFFSET(DESCRIPTOR_SENSE_DATA, DescriptorBuffer));

                    AhciZeroMemory((PCHAR)srbExtension->ResultBuffer, srbExtension->ResultBufferLength);

                    // fill sense data header, leave SenseKey, ASC, ASCQ as zero.
                    descriptorSenseData->ErrorCode = SCSI_SENSE_ERRORCODE_DESCRIPTOR_CURRENT;
                    descriptorSenseData->AdditionalSenseLength = sizeof(SCSI_SENSE_DESCRIPTOR_ATA_STATUS_RETURN);

                    // fill ATA Status Return Info.
                    ataStatus->Header.DescriptorType = SCSI_SENSE_DESCRIPTOR_TYPE_ATA_STATUS_RETURN;
                    ataStatus->Header.AdditionalLength = 0x0C;
                    ataStatus->Extend = Is48BitCommand(srbExtension->Flags) ? 1 : 0;

                    ataStatus->Error = ChannelExtension->ReceivedFIS->D2hRegisterFis.Error;
                    ataStatus->SectorCount7_0 = ChannelExtension->ReceivedFIS->D2hRegisterFis.SectorCount;
                    ataStatus->LbaLow7_0 = ChannelExtension->ReceivedFIS->D2hRegisterFis.SectorNumber;
                    ataStatus->LbaMid7_0 = ChannelExtension->ReceivedFIS->D2hRegisterFis.CylLow;
                    ataStatus->LbaHigh7_0 = ChannelExtension->ReceivedFIS->D2hRegisterFis.CylHigh;
                    ataStatus->Device = ChannelExtension->ReceivedFIS->D2hRegisterFis.Dev_Head;
                    ataStatus->Status = ChannelExtension->ReceivedFIS->D2hRegisterFis.Status;

                    if (Is48BitCommand(srbExtension->Flags)) {
                        ataStatus->SectorCount15_8 = ChannelExtension->ReceivedFIS->D2hRegisterFis.SectorCount_Exp;
                        ataStatus->LbaLow15_8 = ChannelExtension->ReceivedFIS->D2hRegisterFis.SectorNum_Exp;
                        ataStatus->LbaMid15_8 = ChannelExtension->ReceivedFIS->D2hRegisterFis.CylLow_Exp;
                        ataStatus->LbaHigh15_8 = ChannelExtension->ReceivedFIS->D2hRegisterFis.CylHigh_Exp;
                    }

                } else {
                    NT_ASSERT(FALSE);
                }
            } else {
                returnTaskFile = (PATA_TASK_FILE)srbExtension->ResultBuffer;

                returnTaskFile->Current.bCommandReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.Status;
                returnTaskFile->Current.bFeaturesReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.Error;
                returnTaskFile->Current.bCylHighReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.CylHigh;
                returnTaskFile->Current.bCylLowReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.CylLow;
                returnTaskFile->Current.bDriveHeadReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.Dev_Head;
                returnTaskFile->Current.bSectorCountReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.SectorCount;
                returnTaskFile->Current.bSectorNumberReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.SectorNumber;
                //if 48bit, get all of it
                if(Is48BitCommand(srbExtension->Flags) && (srbExtension->ResultBufferLength >= sizeof(ATA_TASK_FILE))) {
                    returnTaskFile->Previous.bCommandReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.Status;
                    returnTaskFile->Previous.bFeaturesReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.Error;
                    returnTaskFile->Previous.bCylHighReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.CylHigh_Exp;
                    returnTaskFile->Previous.bCylLowReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.CylLow_Exp;
                    returnTaskFile->Previous.bDriveHeadReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.Dev_Head;
                    returnTaskFile->Previous.bSectorCountReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.SectorCount_Exp;
                    returnTaskFile->Previous.bSectorNumberReg = ChannelExtension->ReceivedFIS->D2hRegisterFis.SectorNum_Exp;
                }
            }
            // set flag SRB_STATUS_AUTOSENSE_VALID so that Storport will copy it back to original Sense Buffer
            srbToRelease->SrbStatus |= SRB_STATUS_AUTOSENSE_VALID;

        }

        // interpret ATA error to be SCSI error and fill SenseBuffer
        // also log IO Record
        if (IsAtaCommand(srbExtension->AtaFunction) ||
            IsAtaCfisPayload(srbExtension->AtaFunction)) {
            // IO Record for ATA device is logged in AtaMapError()
            AtaMapError(ChannelExtension, srbToRelease, slotContent->StateFlags.FUA);
        } else {
            if (SRB_STATUS(srbToRelease->SrbStatus) == SRB_STATUS_SUCCESS) {
                ChannelExtension->DeviceExtension[0].IoRecord.SuccessCount++;
            } else {
                ChannelExtension->DeviceExtension[0].IoRecord.OtherErrorCount++;
            }
        }
    }


  //3.1 Make the slot available again
    slotContent->CmdHeader = NULL;
    slotContent->CommandHistoryIndex = 0;
    slotContent->Srb = NULL;
    slotContent->StateFlags.FUA = FALSE;

    //Clear the CommandsToComplete bit
    ChannelExtension->SlotManager.CommandsToComplete &= ~(1 << SlotNumber);
    ChannelExtension->SlotManager.HighPriorityAttribute &= ~(1 << SlotNumber);

  //3.3.1 IO is completing, that IO may have paused the queue so unpause the queue
    ChannelExtension->StateFlags.QueuePaused = FALSE;

  //3.3.2 Complete the command
    if ( (srbToRelease->SrbStatus != SRB_STATUS_SUCCESS) && !isSenseSrb && (srbExtension->RetryCount == 0) ) {
        if ( (ChannelExtension->StateFlags.HybridInfoEnabledOnHiberFile == 1) && 
             IsNCQWriteCommand(srbExtension) ) {
            // NCQ WRITE with Hybrid Information failed, retry once without Hybrid Information.
            ChannelExtension->StateFlags.HybridInfoEnabledOnHiberFile = 0;
            retrySrb = TRUE;
        } else if (srbToRelease->SrbStatus == SRB_STATUS_PARITY_ERROR) {
            // if a command encounters CRC error, retry once
            retrySrb = TRUE;
        }
    }

    if (retrySrb) {
        srbToRelease->SrbStatus = SRB_STATUS_PENDING;
        srbExtension->RetryCount++;
        AhciProcessIo(ChannelExtension, srbToRelease, AtDIRQL);
    } else {
        // otherwise, complete it.
        AhciCompleteRequest(ChannelExtension, srbToRelease, AtDIRQL);
    }
}