NTSTATUS TransferPktComplete(IN PDEVICE_OBJECT NullFdo, IN PIRP Irp, IN PVOID Context)
{
PTRANSFER_PACKET pkt = (PTRANSFER_PACKET)Context;
PFUNCTIONAL_DEVICE_EXTENSION fdoExt = pkt->Fdo->DeviceExtension;
PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
PIO_STACK_LOCATION origCurrentSp = IoGetCurrentIrpStackLocation(pkt->OriginalIrp);
BOOLEAN packetDone = FALSE;
/*
* Put all the assertions and spew in here so we don't have to look at them.
*/
DBGCHECKRETURNEDPKT(pkt);
if (SRB_STATUS(pkt->Srb.SrbStatus) == SRB_STATUS_SUCCESS){
fdoData->LoggedTURFailureSinceLastIO = FALSE;
/*
* The port driver should not have allocated a sense buffer
* if the SRB succeeded.
*/
ASSERT(!PORT_ALLOCATED_SENSE(fdoExt, &pkt->Srb));
/*
* Add this packet's transferred length to the original IRP's.
*/
InterlockedExchangeAdd((PLONG)&pkt->OriginalIrp->IoStatus.Information,
(LONG)pkt->Srb.DataTransferLength);
if (pkt->InLowMemRetry){
packetDone = StepLowMemRetry(pkt);
}
else {
packetDone = TRUE;
}
}
else {
/*
* The packet failed. We may retry it if possible.
*/
BOOLEAN shouldRetry;
/*
* Make sure IRP status matches SRB error status (since we propagate it).
*/
if (NT_SUCCESS(Irp->IoStatus.Status)){
Irp->IoStatus.Status = STATUS_UNSUCCESSFUL;
}
/*
* Interpret the SRB error (to a meaningful IRP status)
* and determine if we should retry this packet.
* This call looks at the returned SENSE info to figure out what to do.
*/
shouldRetry = InterpretTransferPacketError(pkt);
/*
* Sometimes the port driver can allocates a new 'sense' buffer
* to report transfer errors, e.g. when the default sense buffer
* is too small. If so, it is up to us to free it.
* Now that we're done interpreting the sense info, free it if appropriate.
*/
if (PORT_ALLOCATED_SENSE(fdoExt, &pkt->Srb)) {
DBGTRACE(ClassDebugSenseInfo, ("Freeing port-allocated sense buffer for pkt %ph.", pkt));
FREE_PORT_ALLOCATED_SENSE_BUFFER(fdoExt, &pkt->Srb);
pkt->Srb.SenseInfoBuffer = &pkt->SrbErrorSenseData;
pkt->Srb.SenseInfoBufferLength = sizeof(SENSE_DATA);
}
/*
* If the SRB queue is locked-up, release it.
* Do this after calling the error handler.
*/
if (pkt->Srb.SrbStatus & SRB_STATUS_QUEUE_FROZEN){
ClassReleaseQueue(pkt->Fdo);
}
if (shouldRetry && (pkt->NumRetries > 0)){
packetDone = RetryTransferPacket(pkt);
}
else {
packetDone = TRUE;
}
}
/*
* If the packet is completed, put it back in the free list.
* If it is the last packet servicing the original request, complete the original irp.
*/
if (packetDone){
LONG numPacketsRemaining;
PIRP deferredIrp;
PDEVICE_OBJECT Fdo = pkt->Fdo;
UCHAR uniqueAddr;
/*
* In case a remove is pending, bump the lock count so we don't get freed
* right after we complete the original irp.
*/
ClassAcquireRemoveLock(Fdo, (PIRP)&uniqueAddr);
/*
* The original IRP should get an error code
* if any one of the packets failed.
*/
if (!NT_SUCCESS(Irp->IoStatus.Status)){
pkt->OriginalIrp->IoStatus.Status = Irp->IoStatus.Status;
/*
* If the original I/O originated in user space (i.e. it is thread-queued),
* and the error is user-correctable (e.g. media is missing, for removable media),
* alert the user.
* Since this is only one of possibly several packets completing for the original IRP,
* we may do this more than once for a single request. That's ok; this allows
* us to test each returned status with IoIsErrorUserInduced().
*/
if (IoIsErrorUserInduced(Irp->IoStatus.Status) &&
pkt->CompleteOriginalIrpWhenLastPacketCompletes &&
pkt->OriginalIrp->Tail.Overlay.Thread){
IoSetHardErrorOrVerifyDevice(pkt->OriginalIrp, pkt->Fdo);
}
}
/*
* We use a field in the original IRP to count
* down the transfer pieces as they complete.
*/
numPacketsRemaining = InterlockedDecrement(
(PLONG)&pkt->OriginalIrp->Tail.Overlay.DriverContext[0]);
if (numPacketsRemaining > 0){
/*
* More transfer pieces remain for the original request.
* Wait for them to complete before completing the original irp.
*/
}
else {
/*
* All the transfer pieces are done.
* Complete the original irp if appropriate.
*/
ASSERT(numPacketsRemaining == 0);
if (pkt->CompleteOriginalIrpWhenLastPacketCompletes){
if (NT_SUCCESS(pkt->OriginalIrp->IoStatus.Status)){
ASSERT((ULONG)pkt->OriginalIrp->IoStatus.Information == origCurrentSp->Parameters.Read.Length);
ClasspPerfIncrementSuccessfulIo(fdoExt);
}
ClassReleaseRemoveLock(pkt->Fdo, pkt->OriginalIrp);
ClassCompleteRequest(pkt->Fdo, pkt->OriginalIrp, IO_DISK_INCREMENT);
/*
* We may have been called by one of the class drivers (e.g. cdrom)
* via the legacy API ClassSplitRequest.
* This is the only case for which the packet engine is called for an FDO
* with a StartIo routine; in that case, we have to call IoStartNextPacket
* now that the original irp has been completed.
*/
if (fdoExt->CommonExtension.DriverExtension->InitData.ClassStartIo) {
if (TEST_FLAG(pkt->Srb.SrbFlags, SRB_FLAGS_DONT_START_NEXT_PACKET)){
DBGTRAP(("SRB_FLAGS_DONT_START_NEXT_PACKET should never be set here (?)"));
}
else {
KIRQL oldIrql;
KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
IoStartNextPacket(pkt->Fdo, FALSE);
KeLowerIrql(oldIrql);
}
}
}
}
/*
* If the packet was synchronous, write the final
* result back to the issuer's status buffer and
* signal his event.
*/
if (pkt->SyncEventPtr){
KeSetEvent(pkt->SyncEventPtr, 0, FALSE);
pkt->SyncEventPtr = NULL;
}
/*
* Free the completed packet.
*/
pkt->OriginalIrp = NULL;
pkt->InLowMemRetry = FALSE;
EnqueueFreeTransferPacket(pkt->Fdo, pkt);
/*
* Now that we have freed some resources,
* try again to send one of the previously deferred irps.
*/
deferredIrp = DequeueDeferredClientIrp(fdoData);
if (deferredIrp){
DBGWARN(("... retrying deferred irp %xh.", deferredIrp));
ServiceTransferRequest(pkt->Fdo, deferredIrp);
}
ClassReleaseRemoveLock(Fdo, (PIRP)&uniqueAddr);
}
return STATUS_MORE_PROCESSING_REQUIRED;
}
NTSTATUS TransferPktComplete(IN PDEVICE_OBJECT NullFdo, IN PIRP Irp, IN PVOID Context)
{
PTRANSFER_PACKET pkt = (PTRANSFER_PACKET)Context;
PFUNCTIONAL_DEVICE_EXTENSION fdoExt = pkt->Fdo->DeviceExtension;
PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
BOOLEAN packetDone = FALSE;
BOOLEAN idleRequest = FALSE;
/*
* Put all the assertions and spew in here so we don't have to look at them.
*/
DBGLOGRETURNPACKET(pkt);
DBGCHECKRETURNEDPKT(pkt);
HISTORYLOGRETURNEDPACKET(pkt);
if (fdoData->IdlePrioritySupported == TRUE) {
idleRequest = ClasspIsIdleRequest(pkt->OriginalIrp);
if (idleRequest) {
InterlockedDecrement(&fdoData->ActiveIdleIoCount);
ASSERT(fdoData->ActiveIdleIoCount >= 0);
} else {
InterlockedDecrement(&fdoData->ActiveIoCount);
ASSERT(fdoData->ActiveIoCount >= 0);
KeQuerySystemTime(&fdoData->LastIoTime);
fdoData->IdleTicks = 0;
}
}
//
// If partial MDL was used, unmap the pages. When the packet is retried, the
// MDL will be recreated. If the packet is done, the MDL will be ready to be reused.
//
if (pkt->UsePartialMdl) {
MmPrepareMdlForReuse(pkt->PartialMdl);
}
if (SRB_STATUS(pkt->Srb.SrbStatus) == SRB_STATUS_SUCCESS) {
fdoData->LoggedTURFailureSinceLastIO = FALSE;
/*
* The port driver should not have allocated a sense buffer
* if the SRB succeeded.
*/
ASSERT(!PORT_ALLOCATED_SENSE(fdoExt, &pkt->Srb));
/*
* Add this packet's transferred length to the original IRP's.
*/
InterlockedExchangeAdd((PLONG)&pkt->OriginalIrp->IoStatus.Information,
(LONG)pkt->Srb.DataTransferLength);
if ((pkt->InLowMemRetry) ||
(pkt->DriverUsesStartIO && pkt->LowMemRetry_remainingBufLen > 0)) {
packetDone = StepLowMemRetry(pkt);
}
else {
packetDone = TRUE;
}
}
else {
/*
* The packet failed. We may retry it if possible.
*/
BOOLEAN shouldRetry;
/*
* Make sure IRP status matches SRB error status (since we propagate it).
*/
if (NT_SUCCESS(Irp->IoStatus.Status)){
Irp->IoStatus.Status = STATUS_UNSUCCESSFUL;
}
/*
* The packet failed.
* So when sending the packet down we either saw either an error or STATUS_PENDING,
* and so we returned STATUS_PENDING for the original IRP.
* So now we must mark the original irp pending to match that, _regardless_ of
* whether we actually switch threads here by retrying.
* (We also have to mark the irp pending if the lower driver marked the irp pending;
* that is dealt with farther down).
*/
if (pkt->CompleteOriginalIrpWhenLastPacketCompletes){
IoMarkIrpPending(pkt->OriginalIrp);
}
/*
* Interpret the SRB error (to a meaningful IRP status)
* and determine if we should retry this packet.
* This call looks at the returned SENSE info to figure out what to do.
*/
shouldRetry = InterpretTransferPacketError(pkt);
/*
* Sometimes the port driver can allocates a new 'sense' buffer
* to report transfer errors, e.g. when the default sense buffer
* is too small. If so, it is up to us to free it.
* Now that we're done interpreting the sense info, free it if appropriate.
* Then clear the sense buffer so it doesn't pollute future errors returned in this packet.
*/
if (PORT_ALLOCATED_SENSE(fdoExt, &pkt->Srb)) {
TracePrint((TRACE_LEVEL_INFORMATION, TRACE_FLAG_RW, "Freeing port-allocated sense buffer for pkt %ph.", pkt));
FREE_PORT_ALLOCATED_SENSE_BUFFER(fdoExt, &pkt->Srb);
pkt->Srb.SenseInfoBuffer = &pkt->SrbErrorSenseData;
pkt->Srb.SenseInfoBufferLength = sizeof(SENSE_DATA);
}
else {
ASSERT(pkt->Srb.SenseInfoBuffer == &pkt->SrbErrorSenseData);
ASSERT(pkt->Srb.SenseInfoBufferLength <= sizeof(SENSE_DATA));
}
RtlZeroMemory(&pkt->SrbErrorSenseData, sizeof(SENSE_DATA));
/*
* If the SRB queue is locked-up, release it.
* Do this after calling the error handler.
*/
if (pkt->Srb.SrbStatus & SRB_STATUS_QUEUE_FROZEN){
ClassReleaseQueue(pkt->Fdo);
}
if (NT_SUCCESS(Irp->IoStatus.Status)){
/*
* The error was recovered above in the InterpretTransferPacketError() call.
*/
ASSERT(!shouldRetry);
/*
* In the case of a recovered error,
* add the transfer length to the original Irp as we would in the success case.
*/
InterlockedExchangeAdd((PLONG)&pkt->OriginalIrp->IoStatus.Information,
(LONG)pkt->Srb.DataTransferLength);
if ((pkt->InLowMemRetry) ||
(pkt->DriverUsesStartIO && pkt->LowMemRetry_remainingBufLen > 0)) {
packetDone = StepLowMemRetry(pkt);
}
else {
packetDone = TRUE;
}
}
else {
if (shouldRetry && (pkt->NumRetries > 0)){
packetDone = RetryTransferPacket(pkt);
}
else if (shouldRetry && (pkt->RetryHistory != NULL)){
// don't limit retries if class driver has custom interpretation routines
packetDone = RetryTransferPacket(pkt);
}
else {
packetDone = TRUE;
}
}
}
/*
* If the packet is completed, put it back in the free list.
* If it is the last packet servicing the original request, complete the original irp.
*/
if (packetDone){
LONG numPacketsRemaining;
PIRP deferredIrp;
PDEVICE_OBJECT Fdo = pkt->Fdo;
UCHAR uniqueAddr;
/*
* In case a remove is pending, bump the lock count so we don't get freed
* right after we complete the original irp.
*/
ClassAcquireRemoveLock(Fdo, (PIRP)&uniqueAddr);
/*
* The original IRP should get an error code
* if any one of the packets failed.
*/
if (!NT_SUCCESS(Irp->IoStatus.Status)){
pkt->OriginalIrp->IoStatus.Status = Irp->IoStatus.Status;
/*
* If the original I/O originated in user space (i.e. it is thread-queued),
* and the error is user-correctable (e.g. media is missing, for removable media),
* alert the user.
* Since this is only one of possibly several packets completing for the original IRP,
* we may do this more than once for a single request. That's ok; this allows
* us to test each returned status with IoIsErrorUserInduced().
*/
if (IoIsErrorUserInduced(Irp->IoStatus.Status) &&
pkt->CompleteOriginalIrpWhenLastPacketCompletes &&
pkt->OriginalIrp->Tail.Overlay.Thread){
IoSetHardErrorOrVerifyDevice(pkt->OriginalIrp, Fdo);
}
}
/*
* We use a field in the original IRP to count
* down the transfer pieces as they complete.
*/
numPacketsRemaining = InterlockedDecrement(
(PLONG)&pkt->OriginalIrp->Tail.Overlay.DriverContext[0]);
if (numPacketsRemaining > 0){
/*
* More transfer pieces remain for the original request.
* Wait for them to complete before completing the original irp.
*/
}
else {
/*
* All the transfer pieces are done.
* Complete the original irp if appropriate.
*/
ASSERT(numPacketsRemaining == 0);
if (pkt->CompleteOriginalIrpWhenLastPacketCompletes){
IO_PAGING_PRIORITY priority = (TEST_FLAG(pkt->OriginalIrp->Flags, IRP_PAGING_IO)) ? IoGetPagingIoPriority(pkt->OriginalIrp) : IoPagingPriorityInvalid;
KIRQL oldIrql;
if (NT_SUCCESS(pkt->OriginalIrp->IoStatus.Status)){
ASSERT((ULONG)pkt->OriginalIrp->IoStatus.Information == IoGetCurrentIrpStackLocation(pkt->OriginalIrp)->Parameters.Read.Length);
ClasspPerfIncrementSuccessfulIo(fdoExt);
}
ClassReleaseRemoveLock(Fdo, pkt->OriginalIrp);
/*
* We submitted all the downward irps, including this last one, on the thread
* that the OriginalIrp came in on. So the OriginalIrp is completing on a
* different thread iff this last downward irp is completing on a different thread.
* If BlkCache is loaded, for example, it will often complete
* requests out of the cache on the same thread, therefore not marking the downward
* irp pending and not requiring us to do so here. If the downward request is completing
* on the same thread, then by not marking the OriginalIrp pending we can save an APC
* and get extra perf benefit out of BlkCache.
* Note that if the packet ever cycled due to retry or LowMemRetry,
* we set the pending bit in those codepaths.
*/
if (pkt->Irp->PendingReturned){
IoMarkIrpPending(pkt->OriginalIrp);
}
ClassCompleteRequest(Fdo, pkt->OriginalIrp, IO_DISK_INCREMENT);
//
// Drop the count only after completing the request, to give
// Mm some amount of time to issue its next critical request
//
if (priority == IoPagingPriorityHigh)
{
KeAcquireSpinLock(&fdoData->SpinLock, &oldIrql);
if (fdoData->MaxInterleavedNormalIo < ClassMaxInterleavePerCriticalIo)
{
fdoData->MaxInterleavedNormalIo = 0;
}
else
{
fdoData->MaxInterleavedNormalIo -= ClassMaxInterleavePerCriticalIo;
}
fdoData->NumHighPriorityPagingIo--;
if (fdoData->NumHighPriorityPagingIo == 0)
{
LARGE_INTEGER period;
//
// Exiting throttle mode
//
KeQuerySystemTime(&fdoData->ThrottleStopTime);
period.QuadPart = fdoData->ThrottleStopTime.QuadPart - fdoData->ThrottleStartTime.QuadPart;
fdoData->LongestThrottlePeriod.QuadPart = max(fdoData->LongestThrottlePeriod.QuadPart, period.QuadPart);
}
KeReleaseSpinLock(&fdoData->SpinLock, oldIrql);
}
if (idleRequest) {
ClasspCompleteIdleRequest(fdoExt);
}
/*
* We may have been called by one of the class drivers (e.g. cdrom)
* via the legacy API ClassSplitRequest.
* This is the only case for which the packet engine is called for an FDO
* with a StartIo routine; in that case, we have to call IoStartNextPacket
* now that the original irp has been completed.
*/
if (fdoExt->CommonExtension.DriverExtension->InitData.ClassStartIo) {
if (TEST_FLAG(pkt->Srb.SrbFlags, SRB_FLAGS_DONT_START_NEXT_PACKET)){
TracePrint((TRACE_LEVEL_INFORMATION, TRACE_FLAG_RW, "SRB_FLAGS_DONT_START_NEXT_PACKET should never be set here (??)"));
}
else {
KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
IoStartNextPacket(Fdo, TRUE); // yes, some IO is now cancellable
KeLowerIrql(oldIrql);
}
}
}
}
/*
* If the packet was synchronous, write the final result back to the issuer's status buffer
* and signal his event.
*/
if (pkt->SyncEventPtr){
KeSetEvent(pkt->SyncEventPtr, 0, FALSE);
pkt->SyncEventPtr = NULL;
}
/*
* Free the completed packet.
*/
pkt->UsePartialMdl = FALSE;
// pkt->OriginalIrp = NULL;
pkt->InLowMemRetry = FALSE;
EnqueueFreeTransferPacket(Fdo, pkt);
/*
* Now that we have freed some resources,
* try again to send one of the previously deferred irps.
*/
deferredIrp = DequeueDeferredClientIrp(fdoData);
if (deferredIrp){
ServiceTransferRequest(Fdo, deferredIrp);
}
ClassReleaseRemoveLock(Fdo, (PIRP)&uniqueAddr);
}
return STATUS_MORE_PROCESSING_REQUIRED;
}