NTSTATUS TransferPktComplete(IN PDEVICE_OBJECT NullFdo, IN PIRP Irp, IN PVOID Context)
{
PTRANSFER_PACKET pkt = (PTRANSFER_PACKET)Context;
PFUNCTIONAL_DEVICE_EXTENSION fdoExt = pkt->Fdo->DeviceExtension;
PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
PIO_STACK_LOCATION origCurrentSp = IoGetCurrentIrpStackLocation(pkt->OriginalIrp);
BOOLEAN packetDone = FALSE;
/*
* Put all the assertions and spew in here so we don't have to look at them.
*/
DBGCHECKRETURNEDPKT(pkt);
if (SRB_STATUS(pkt->Srb.SrbStatus) == SRB_STATUS_SUCCESS){
fdoData->LoggedTURFailureSinceLastIO = FALSE;
/*
* The port driver should not have allocated a sense buffer
* if the SRB succeeded.
*/
ASSERT(!PORT_ALLOCATED_SENSE(fdoExt, &pkt->Srb));
/*
* Add this packet's transferred length to the original IRP's.
*/
InterlockedExchangeAdd((PLONG)&pkt->OriginalIrp->IoStatus.Information,
(LONG)pkt->Srb.DataTransferLength);
if (pkt->InLowMemRetry){
packetDone = StepLowMemRetry(pkt);
}
else {
packetDone = TRUE;
}
}
else {
/*
* The packet failed. We may retry it if possible.
*/
BOOLEAN shouldRetry;
/*
* Make sure IRP status matches SRB error status (since we propagate it).
*/
if (NT_SUCCESS(Irp->IoStatus.Status)){
Irp->IoStatus.Status = STATUS_UNSUCCESSFUL;
}
/*
* Interpret the SRB error (to a meaningful IRP status)
* and determine if we should retry this packet.
* This call looks at the returned SENSE info to figure out what to do.
*/
shouldRetry = InterpretTransferPacketError(pkt);
/*
* Sometimes the port driver can allocates a new 'sense' buffer
* to report transfer errors, e.g. when the default sense buffer
* is too small. If so, it is up to us to free it.
* Now that we're done interpreting the sense info, free it if appropriate.
*/
if (PORT_ALLOCATED_SENSE(fdoExt, &pkt->Srb)) {
DBGTRACE(ClassDebugSenseInfo, ("Freeing port-allocated sense buffer for pkt %ph.", pkt));
FREE_PORT_ALLOCATED_SENSE_BUFFER(fdoExt, &pkt->Srb);
pkt->Srb.SenseInfoBuffer = &pkt->SrbErrorSenseData;
pkt->Srb.SenseInfoBufferLength = sizeof(SENSE_DATA);
}
/*
* If the SRB queue is locked-up, release it.
* Do this after calling the error handler.
*/
if (pkt->Srb.SrbStatus & SRB_STATUS_QUEUE_FROZEN){
ClassReleaseQueue(pkt->Fdo);
}
if (shouldRetry && (pkt->NumRetries > 0)){
packetDone = RetryTransferPacket(pkt);
}
else {
packetDone = TRUE;
}
}
/*
* If the packet is completed, put it back in the free list.
* If it is the last packet servicing the original request, complete the original irp.
*/
if (packetDone){
LONG numPacketsRemaining;
PIRP deferredIrp;
PDEVICE_OBJECT Fdo = pkt->Fdo;
UCHAR uniqueAddr;
/*
* In case a remove is pending, bump the lock count so we don't get freed
* right after we complete the original irp.
*/
ClassAcquireRemoveLock(Fdo, (PIRP)&uniqueAddr);
/*
* The original IRP should get an error code
* if any one of the packets failed.
*/
if (!NT_SUCCESS(Irp->IoStatus.Status)){
pkt->OriginalIrp->IoStatus.Status = Irp->IoStatus.Status;
/*
* If the original I/O originated in user space (i.e. it is thread-queued),
* and the error is user-correctable (e.g. media is missing, for removable media),
* alert the user.
* Since this is only one of possibly several packets completing for the original IRP,
* we may do this more than once for a single request. That's ok; this allows
* us to test each returned status with IoIsErrorUserInduced().
*/
if (IoIsErrorUserInduced(Irp->IoStatus.Status) &&
pkt->CompleteOriginalIrpWhenLastPacketCompletes &&
pkt->OriginalIrp->Tail.Overlay.Thread){
IoSetHardErrorOrVerifyDevice(pkt->OriginalIrp, pkt->Fdo);
}
}
/*
* We use a field in the original IRP to count
* down the transfer pieces as they complete.
*/
numPacketsRemaining = InterlockedDecrement(
(PLONG)&pkt->OriginalIrp->Tail.Overlay.DriverContext[0]);
if (numPacketsRemaining > 0){
/*
* More transfer pieces remain for the original request.
* Wait for them to complete before completing the original irp.
*/
}
else {
/*
* All the transfer pieces are done.
* Complete the original irp if appropriate.
*/
ASSERT(numPacketsRemaining == 0);
if (pkt->CompleteOriginalIrpWhenLastPacketCompletes){
if (NT_SUCCESS(pkt->OriginalIrp->IoStatus.Status)){
ASSERT((ULONG)pkt->OriginalIrp->IoStatus.Information == origCurrentSp->Parameters.Read.Length);
ClasspPerfIncrementSuccessfulIo(fdoExt);
}
ClassReleaseRemoveLock(pkt->Fdo, pkt->OriginalIrp);
ClassCompleteRequest(pkt->Fdo, pkt->OriginalIrp, IO_DISK_INCREMENT);
/*
* We may have been called by one of the class drivers (e.g. cdrom)
* via the legacy API ClassSplitRequest.
* This is the only case for which the packet engine is called for an FDO
* with a StartIo routine; in that case, we have to call IoStartNextPacket
* now that the original irp has been completed.
*/
if (fdoExt->CommonExtension.DriverExtension->InitData.ClassStartIo) {
if (TEST_FLAG(pkt->Srb.SrbFlags, SRB_FLAGS_DONT_START_NEXT_PACKET)){
DBGTRAP(("SRB_FLAGS_DONT_START_NEXT_PACKET should never be set here (?)"));
}
else {
KIRQL oldIrql;
KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
IoStartNextPacket(pkt->Fdo, FALSE);
KeLowerIrql(oldIrql);
}
}
}
}
/*
* If the packet was synchronous, write the final
* result back to the issuer's status buffer and
* signal his event.
*/
if (pkt->SyncEventPtr){
KeSetEvent(pkt->SyncEventPtr, 0, FALSE);
pkt->SyncEventPtr = NULL;
}
/*
* Free the completed packet.
*/
pkt->OriginalIrp = NULL;
pkt->InLowMemRetry = FALSE;
EnqueueFreeTransferPacket(pkt->Fdo, pkt);
/*
* Now that we have freed some resources,
* try again to send one of the previously deferred irps.
*/
deferredIrp = DequeueDeferredClientIrp(fdoData);
if (deferredIrp){
DBGWARN(("... retrying deferred irp %xh.", deferredIrp));
ServiceTransferRequest(pkt->Fdo, deferredIrp);
}
ClassReleaseRemoveLock(Fdo, (PIRP)&uniqueAddr);
}
return STATUS_MORE_PROCESSING_REQUIRED;
}
/*
* RetryTransferPacket
*
* Retry sending a TRANSFER_PACKET.
*
* Return TRUE iff the packet is complete.
* (if so the status in pkt->irp is the final status).
*/
BOOLEAN NTAPI RetryTransferPacket(PTRANSFER_PACKET Pkt)
{
BOOLEAN packetDone;
DBGTRACE(ClassDebugTrace, ("retrying failed transfer (pkt=%ph, op=%s)", Pkt, DBGGETSCSIOPSTR(&Pkt->Srb)));
ASSERT(Pkt->NumRetries > 0);
Pkt->NumRetries--;
/*
* Tone down performance on the retry.
* This increases the chance for success on the retry.
* We've seen instances of drives that fail consistently but then start working
* once this scale-down is applied.
*/
SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_DISABLE_DISCONNECT);
SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_DISABLE_SYNCH_TRANSFER);
CLEAR_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_QUEUE_ACTION_ENABLE);
Pkt->Srb.QueueTag = SP_UNTAGGED;
if (Pkt->Irp->IoStatus.Status == STATUS_INSUFFICIENT_RESOURCES){
PCDB pCdb = (PCDB)Pkt->Srb.Cdb;
BOOLEAN isReadWrite = ((pCdb->CDB10.OperationCode == SCSIOP_READ) ||
(pCdb->CDB10.OperationCode == SCSIOP_WRITE));
if (Pkt->InLowMemRetry || !isReadWrite){
/*
* This should never happen.
* The memory manager guarantees that at least four pages will
* be available to allow forward progress in the port driver.
* So a one-page transfer should never fail with insufficient resources.
*/
ASSERT(isReadWrite && !Pkt->InLowMemRetry);
packetDone = TRUE;
}
else {
/*
* We are in low-memory stress.
* Start the low-memory retry state machine, which tries to
* resend the packet in little one-page chunks.
*/
InitLowMemRetry( Pkt,
Pkt->BufPtrCopy,
Pkt->BufLenCopy,
Pkt->TargetLocationCopy);
StepLowMemRetry(Pkt);
packetDone = FALSE;
}
}
else {
/*
* Retry the packet by simply resending it after a delay.
* Put the packet back in the pending queue and
* schedule a timer to retry the transfer.
*
* Do not call SetupReadWriteTransferPacket again because:
* (1) The minidriver may have set some bits
* in the SRB that it needs again and
* (2) doing so would reset numRetries.
*
* BECAUSE we do not call SetupReadWriteTransferPacket again,
* we have to reset a couple fields in the SRB that
* some miniports overwrite when they fail an SRB.
*/
Pkt->Srb.DataBuffer = Pkt->BufPtrCopy;
Pkt->Srb.DataTransferLength = Pkt->BufLenCopy;
if (Pkt->RetryIntervalSec == 0){
/*
* Always delay by at least a little when retrying.
* Some problems (e.g. CRC errors) are not recoverable without a slight delay.
*/
LARGE_INTEGER timerPeriod;
timerPeriod.HighPart = -1;
timerPeriod.LowPart = -(LONG)((ULONG)MINIMUM_RETRY_UNITS*KeQueryTimeIncrement());
KeInitializeTimer(&Pkt->RetryTimer);
KeInitializeDpc(&Pkt->RetryTimerDPC, TransferPacketRetryTimerDpc, Pkt);
KeSetTimer(&Pkt->RetryTimer, timerPeriod, &Pkt->RetryTimerDPC);
}
else {
LARGE_INTEGER timerPeriod;
ASSERT(Pkt->RetryIntervalSec < 100); // sanity check
timerPeriod.HighPart = -1;
timerPeriod.LowPart = Pkt->RetryIntervalSec*-10000000;
KeInitializeTimer(&Pkt->RetryTimer);
KeInitializeDpc(&Pkt->RetryTimerDPC, TransferPacketRetryTimerDpc, Pkt);
KeSetTimer(&Pkt->RetryTimer, timerPeriod, &Pkt->RetryTimerDPC);
}
packetDone = FALSE;
}
return packetDone;
}
/*
* RetryTransferPacket
*
* Retry sending a TRANSFER_PACKET.
*
* Return TRUE iff the packet is complete.
* (if so the status in pkt->irp is the final status).
*/
BOOLEAN RetryTransferPacket(PTRANSFER_PACKET Pkt)
{
BOOLEAN packetDone;
TracePrint((TRACE_LEVEL_INFORMATION, TRACE_FLAG_GENERAL, "retrying failed transfer (pkt=%ph, op=%s)", Pkt, DBGGETSCSIOPSTR(Pkt->Srb)));
NT_ASSERT(Pkt->NumRetries > 0 || Pkt->RetryHistory);
Pkt->NumRetries--;
//
// If this is the last retry, then turn off disconnect, sync transfer,
// and tagged queuing. On all other retries, leave the original settings.
//
if ( 0 == Pkt->NumRetries ) {
/*
* Tone down performance on the retry.
* This increases the chance for success on the retry.
* We've seen instances of drives that fail consistently but then start working
* once this scale-down is applied.
*/
SrbSetSrbFlags(Pkt->Srb, SRB_FLAGS_DISABLE_DISCONNECT);
SrbSetSrbFlags(Pkt->Srb, SRB_FLAGS_DISABLE_SYNCH_TRANSFER);
SrbClearSrbFlags(Pkt->Srb, SRB_FLAGS_QUEUE_ACTION_ENABLE);
SrbSetRequestTag(Pkt->Srb, SP_UNTAGGED);
}
if (Pkt->Irp->IoStatus.Status == STATUS_INSUFFICIENT_RESOURCES) {
PCDB pCdb = SrbGetCdb(Pkt->Srb);
UCHAR cdbOpcode = 0;
BOOLEAN isReadWrite = FALSE;
PFUNCTIONAL_DEVICE_EXTENSION fdoExtension = Pkt->Fdo->DeviceExtension;
PCLASS_PRIVATE_FDO_DATA fdoData = fdoExtension->PrivateFdoData;
if (pCdb) {
cdbOpcode = pCdb->CDB10.OperationCode;
isReadWrite = IS_SCSIOP_READWRITE(cdbOpcode);
}
if ((Pkt->DriverUsesStartIO) &&
( (cdbOpcode == SCSIOP_WRITE6 ) ||
(cdbOpcode == SCSIOP_WRITE ) ||
(cdbOpcode == SCSIOP_WRITE12) ||
(cdbOpcode == SCSIOP_WRITE16) )) {
/* don't retry writes in super-low-memory conditions if the
* driver must serialize against StartIO. This is because
* some write methods used in such drivers cannot accept
* random-sized writes. (i.e CD-RW in packet writing mode)
* Reads, however, are always safe to split up.
*/
SET_FLAG(fdoData->TrackingFlags, TRACKING_FORWARD_PROGRESS_PATH1);
packetDone = TRUE;
}
else if (Pkt->InLowMemRetry || !isReadWrite){
/*
* This should never happen under normal circumstances.
* The memory manager guarantees that at least four pages will
* be available to allow forward progress in the port driver.
* So a one-page transfer should never fail with insufficient resources.
*
* However, it is possible to get in here with virtual storage
* or thin provisioned storage for example.
* A single sector write can trigger an allocation request and
* presently a forward progress guarantee is not provided.
* VHD also may have some limitations in forward progress guarantee.
* And USB too might also fall into this category.
*/
SET_FLAG(fdoData->TrackingFlags, TRACKING_FORWARD_PROGRESS_PATH2);
packetDone = TRUE;
}
else {
/*
* We are in low-memory stress.
* Start the low-memory retry state machine, which tries to
* resend the packet in little one-page chunks.
*/
SET_FLAG(fdoData->TrackingFlags, TRACKING_FORWARD_PROGRESS_PATH3);
InitLowMemRetry(Pkt,
Pkt->BufPtrCopy,
Pkt->BufLenCopy,
Pkt->TargetLocationCopy);
StepLowMemRetry(Pkt);
packetDone = FALSE;
}
}
else {
/*
* Retry the packet by simply resending it after a delay.
* Put the packet back in the pending queue and
* schedule a timer to retry the transfer.
*
* Do not call SetupReadWriteTransferPacket again because:
* (1) The minidriver may have set some bits
* in the SRB that it needs again and
* (2) doing so would reset numRetries.
*
* BECAUSE we do not call SetupReadWriteTransferPacket again,
* we have to reset a couple fields in the SRB that
* some miniports overwrite when they fail an SRB.
*/
SrbSetDataBuffer(Pkt->Srb, Pkt->BufPtrCopy);
SrbSetDataTransferLength(Pkt->Srb, Pkt->BufLenCopy);
TransferPacketQueueRetryDpc(Pkt);
packetDone = FALSE;
}
return packetDone;
}
NTSTATUS TransferPktComplete(IN PDEVICE_OBJECT NullFdo, IN PIRP Irp, IN PVOID Context)
{
PTRANSFER_PACKET pkt = (PTRANSFER_PACKET)Context;
PFUNCTIONAL_DEVICE_EXTENSION fdoExt = pkt->Fdo->DeviceExtension;
PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
BOOLEAN packetDone = FALSE;
BOOLEAN idleRequest = FALSE;
/*
* Put all the assertions and spew in here so we don't have to look at them.
*/
DBGLOGRETURNPACKET(pkt);
DBGCHECKRETURNEDPKT(pkt);
HISTORYLOGRETURNEDPACKET(pkt);
if (fdoData->IdlePrioritySupported == TRUE) {
idleRequest = ClasspIsIdleRequest(pkt->OriginalIrp);
if (idleRequest) {
InterlockedDecrement(&fdoData->ActiveIdleIoCount);
ASSERT(fdoData->ActiveIdleIoCount >= 0);
} else {
InterlockedDecrement(&fdoData->ActiveIoCount);
ASSERT(fdoData->ActiveIoCount >= 0);
KeQuerySystemTime(&fdoData->LastIoTime);
fdoData->IdleTicks = 0;
}
}
//
// If partial MDL was used, unmap the pages. When the packet is retried, the
// MDL will be recreated. If the packet is done, the MDL will be ready to be reused.
//
if (pkt->UsePartialMdl) {
MmPrepareMdlForReuse(pkt->PartialMdl);
}
if (SRB_STATUS(pkt->Srb.SrbStatus) == SRB_STATUS_SUCCESS) {
fdoData->LoggedTURFailureSinceLastIO = FALSE;
/*
* The port driver should not have allocated a sense buffer
* if the SRB succeeded.
*/
ASSERT(!PORT_ALLOCATED_SENSE(fdoExt, &pkt->Srb));
/*
* Add this packet's transferred length to the original IRP's.
*/
InterlockedExchangeAdd((PLONG)&pkt->OriginalIrp->IoStatus.Information,
(LONG)pkt->Srb.DataTransferLength);
if ((pkt->InLowMemRetry) ||
(pkt->DriverUsesStartIO && pkt->LowMemRetry_remainingBufLen > 0)) {
packetDone = StepLowMemRetry(pkt);
}
else {
packetDone = TRUE;
}
}
else {
/*
* The packet failed. We may retry it if possible.
*/
BOOLEAN shouldRetry;
/*
* Make sure IRP status matches SRB error status (since we propagate it).
*/
if (NT_SUCCESS(Irp->IoStatus.Status)){
Irp->IoStatus.Status = STATUS_UNSUCCESSFUL;
}
/*
* The packet failed.
* So when sending the packet down we either saw either an error or STATUS_PENDING,
* and so we returned STATUS_PENDING for the original IRP.
* So now we must mark the original irp pending to match that, _regardless_ of
* whether we actually switch threads here by retrying.
* (We also have to mark the irp pending if the lower driver marked the irp pending;
* that is dealt with farther down).
*/
if (pkt->CompleteOriginalIrpWhenLastPacketCompletes){
IoMarkIrpPending(pkt->OriginalIrp);
}
/*
* Interpret the SRB error (to a meaningful IRP status)
* and determine if we should retry this packet.
* This call looks at the returned SENSE info to figure out what to do.
*/
shouldRetry = InterpretTransferPacketError(pkt);
/*
* Sometimes the port driver can allocates a new 'sense' buffer
* to report transfer errors, e.g. when the default sense buffer
* is too small. If so, it is up to us to free it.
* Now that we're done interpreting the sense info, free it if appropriate.
* Then clear the sense buffer so it doesn't pollute future errors returned in this packet.
*/
if (PORT_ALLOCATED_SENSE(fdoExt, &pkt->Srb)) {
TracePrint((TRACE_LEVEL_INFORMATION, TRACE_FLAG_RW, "Freeing port-allocated sense buffer for pkt %ph.", pkt));
FREE_PORT_ALLOCATED_SENSE_BUFFER(fdoExt, &pkt->Srb);
pkt->Srb.SenseInfoBuffer = &pkt->SrbErrorSenseData;
pkt->Srb.SenseInfoBufferLength = sizeof(SENSE_DATA);
}
else {
ASSERT(pkt->Srb.SenseInfoBuffer == &pkt->SrbErrorSenseData);
ASSERT(pkt->Srb.SenseInfoBufferLength <= sizeof(SENSE_DATA));
}
RtlZeroMemory(&pkt->SrbErrorSenseData, sizeof(SENSE_DATA));
/*
* If the SRB queue is locked-up, release it.
* Do this after calling the error handler.
*/
if (pkt->Srb.SrbStatus & SRB_STATUS_QUEUE_FROZEN){
ClassReleaseQueue(pkt->Fdo);
}
if (NT_SUCCESS(Irp->IoStatus.Status)){
/*
* The error was recovered above in the InterpretTransferPacketError() call.
*/
ASSERT(!shouldRetry);
/*
* In the case of a recovered error,
* add the transfer length to the original Irp as we would in the success case.
*/
InterlockedExchangeAdd((PLONG)&pkt->OriginalIrp->IoStatus.Information,
(LONG)pkt->Srb.DataTransferLength);
if ((pkt->InLowMemRetry) ||
(pkt->DriverUsesStartIO && pkt->LowMemRetry_remainingBufLen > 0)) {
packetDone = StepLowMemRetry(pkt);
}
else {
packetDone = TRUE;
}
}
else {
if (shouldRetry && (pkt->NumRetries > 0)){
packetDone = RetryTransferPacket(pkt);
}
else if (shouldRetry && (pkt->RetryHistory != NULL)){
// don't limit retries if class driver has custom interpretation routines
packetDone = RetryTransferPacket(pkt);
}
else {
packetDone = TRUE;
}
}
}
/*
* If the packet is completed, put it back in the free list.
* If it is the last packet servicing the original request, complete the original irp.
*/
if (packetDone){
LONG numPacketsRemaining;
PIRP deferredIrp;
PDEVICE_OBJECT Fdo = pkt->Fdo;
UCHAR uniqueAddr;
/*
* In case a remove is pending, bump the lock count so we don't get freed
* right after we complete the original irp.
*/
ClassAcquireRemoveLock(Fdo, (PIRP)&uniqueAddr);
/*
* The original IRP should get an error code
* if any one of the packets failed.
*/
if (!NT_SUCCESS(Irp->IoStatus.Status)){
pkt->OriginalIrp->IoStatus.Status = Irp->IoStatus.Status;
/*
* If the original I/O originated in user space (i.e. it is thread-queued),
* and the error is user-correctable (e.g. media is missing, for removable media),
* alert the user.
* Since this is only one of possibly several packets completing for the original IRP,
* we may do this more than once for a single request. That's ok; this allows
* us to test each returned status with IoIsErrorUserInduced().
*/
if (IoIsErrorUserInduced(Irp->IoStatus.Status) &&
pkt->CompleteOriginalIrpWhenLastPacketCompletes &&
pkt->OriginalIrp->Tail.Overlay.Thread){
IoSetHardErrorOrVerifyDevice(pkt->OriginalIrp, Fdo);
}
}
/*
* We use a field in the original IRP to count
* down the transfer pieces as they complete.
*/
numPacketsRemaining = InterlockedDecrement(
(PLONG)&pkt->OriginalIrp->Tail.Overlay.DriverContext[0]);
if (numPacketsRemaining > 0){
/*
* More transfer pieces remain for the original request.
* Wait for them to complete before completing the original irp.
*/
}
else {
/*
* All the transfer pieces are done.
* Complete the original irp if appropriate.
*/
ASSERT(numPacketsRemaining == 0);
if (pkt->CompleteOriginalIrpWhenLastPacketCompletes){
IO_PAGING_PRIORITY priority = (TEST_FLAG(pkt->OriginalIrp->Flags, IRP_PAGING_IO)) ? IoGetPagingIoPriority(pkt->OriginalIrp) : IoPagingPriorityInvalid;
KIRQL oldIrql;
if (NT_SUCCESS(pkt->OriginalIrp->IoStatus.Status)){
ASSERT((ULONG)pkt->OriginalIrp->IoStatus.Information == IoGetCurrentIrpStackLocation(pkt->OriginalIrp)->Parameters.Read.Length);
ClasspPerfIncrementSuccessfulIo(fdoExt);
}
ClassReleaseRemoveLock(Fdo, pkt->OriginalIrp);
/*
* We submitted all the downward irps, including this last one, on the thread
* that the OriginalIrp came in on. So the OriginalIrp is completing on a
* different thread iff this last downward irp is completing on a different thread.
* If BlkCache is loaded, for example, it will often complete
* requests out of the cache on the same thread, therefore not marking the downward
* irp pending and not requiring us to do so here. If the downward request is completing
* on the same thread, then by not marking the OriginalIrp pending we can save an APC
* and get extra perf benefit out of BlkCache.
* Note that if the packet ever cycled due to retry or LowMemRetry,
* we set the pending bit in those codepaths.
*/
if (pkt->Irp->PendingReturned){
IoMarkIrpPending(pkt->OriginalIrp);
}
ClassCompleteRequest(Fdo, pkt->OriginalIrp, IO_DISK_INCREMENT);
//
// Drop the count only after completing the request, to give
// Mm some amount of time to issue its next critical request
//
if (priority == IoPagingPriorityHigh)
{
KeAcquireSpinLock(&fdoData->SpinLock, &oldIrql);
if (fdoData->MaxInterleavedNormalIo < ClassMaxInterleavePerCriticalIo)
{
fdoData->MaxInterleavedNormalIo = 0;
}
else
{
fdoData->MaxInterleavedNormalIo -= ClassMaxInterleavePerCriticalIo;
}
fdoData->NumHighPriorityPagingIo--;
if (fdoData->NumHighPriorityPagingIo == 0)
{
LARGE_INTEGER period;
//
// Exiting throttle mode
//
KeQuerySystemTime(&fdoData->ThrottleStopTime);
period.QuadPart = fdoData->ThrottleStopTime.QuadPart - fdoData->ThrottleStartTime.QuadPart;
fdoData->LongestThrottlePeriod.QuadPart = max(fdoData->LongestThrottlePeriod.QuadPart, period.QuadPart);
}
KeReleaseSpinLock(&fdoData->SpinLock, oldIrql);
}
if (idleRequest) {
ClasspCompleteIdleRequest(fdoExt);
}
/*
* We may have been called by one of the class drivers (e.g. cdrom)
* via the legacy API ClassSplitRequest.
* This is the only case for which the packet engine is called for an FDO
* with a StartIo routine; in that case, we have to call IoStartNextPacket
* now that the original irp has been completed.
*/
if (fdoExt->CommonExtension.DriverExtension->InitData.ClassStartIo) {
if (TEST_FLAG(pkt->Srb.SrbFlags, SRB_FLAGS_DONT_START_NEXT_PACKET)){
TracePrint((TRACE_LEVEL_INFORMATION, TRACE_FLAG_RW, "SRB_FLAGS_DONT_START_NEXT_PACKET should never be set here (??)"));
}
else {
KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
IoStartNextPacket(Fdo, TRUE); // yes, some IO is now cancellable
KeLowerIrql(oldIrql);
}
}
}
}
/*
* If the packet was synchronous, write the final result back to the issuer's status buffer
* and signal his event.
*/
if (pkt->SyncEventPtr){
KeSetEvent(pkt->SyncEventPtr, 0, FALSE);
pkt->SyncEventPtr = NULL;
}
/*
* Free the completed packet.
*/
pkt->UsePartialMdl = FALSE;
// pkt->OriginalIrp = NULL;
pkt->InLowMemRetry = FALSE;
EnqueueFreeTransferPacket(Fdo, pkt);
/*
* Now that we have freed some resources,
* try again to send one of the previously deferred irps.
*/
deferredIrp = DequeueDeferredClientIrp(fdoData);
if (deferredIrp){
ServiceTransferRequest(Fdo, deferredIrp);
}
ClassReleaseRemoveLock(Fdo, (PIRP)&uniqueAddr);
}
return STATUS_MORE_PROCESSING_REQUIRED;
}
