status_t DMAResource::TranslateNext(IORequest* request, IOOperation* operation, generic_size_t maxOperationLength) { IOBuffer* buffer = request->Buffer(); off_t originalOffset = request->Offset() + request->Length() - request->RemainingBytes(); off_t offset = originalOffset; generic_size_t partialBegin = offset & (fBlockSize - 1); // current iteration state uint32 vecIndex = request->VecIndex(); uint32 vecOffset = request->VecOffset(); generic_size_t totalLength = min_c(request->RemainingBytes(), fRestrictions.max_transfer_size); if (maxOperationLength > 0 && maxOperationLength < totalLength + partialBegin) { totalLength = maxOperationLength - partialBegin; } MutexLocker locker(fLock); DMABuffer* dmaBuffer = fDMABuffers.RemoveHead(); if (dmaBuffer == NULL) return B_BUSY; dmaBuffer->SetVecCount(0); generic_io_vec* vecs = NULL; uint32 segmentCount = 0; TRACE(" offset %Ld, remaining size: %lu, block size %lu -> partial: %lu\n", offset, request->RemainingBytes(), fBlockSize, partialBegin); if (buffer->IsVirtual()) { // Unless we need the bounce buffer anyway, we have to translate the // virtual addresses to physical addresses, so we can check the DMA // restrictions. TRACE(" buffer is virtual %s\n", buffer->IsUser() ? "user" : "kernel"); // TODO: !partialOperation || totalLength >= fBlockSize // TODO: Maybe enforce fBounceBufferSize >= 2 * fBlockSize. if (true) { generic_size_t transferLeft = totalLength; vecs = fScratchVecs; TRACE(" create physical map (for %ld vecs)\n", buffer->VecCount()); for (uint32 i = vecIndex; i < buffer->VecCount(); i++) { generic_io_vec& vec = buffer->VecAt(i); generic_addr_t base = vec.base + vecOffset; generic_size_t size = vec.length - vecOffset; vecOffset = 0; if (size > transferLeft) size = transferLeft; while (size > 0 && segmentCount < fRestrictions.max_segment_count) { physical_entry entry; uint32 count = 1; get_memory_map_etc(request->TeamID(), (void*)base, size, &entry, &count); vecs[segmentCount].base = entry.address; vecs[segmentCount].length = entry.size; transferLeft -= entry.size; base += entry.size; size -= entry.size; segmentCount++; } if (transferLeft == 0) break; } totalLength -= transferLeft; } vecIndex = 0; vecOffset = 0; } else { // We do already have physical addresses. locker.Unlock(); vecs = buffer->Vecs(); segmentCount = min_c(buffer->VecCount() - vecIndex, fRestrictions.max_segment_count); } #ifdef TRACE_DMA_RESOURCE TRACE(" physical count %lu\n", segmentCount); for (uint32 i = 0; i < segmentCount; i++) { TRACE(" [%" B_PRIu32 "] %#" B_PRIxGENADDR ", %" B_PRIxGENADDR "\n", i, vecs[vecIndex + i].base, vecs[vecIndex + i].length); } #endif // check alignment, boundaries, etc. and set vecs in DMA buffer // Fetch a bounce buffer we can use for the DMABuffer. // TODO: We should do that lazily when needed! DMABounceBuffer* bounceBuffer = NULL; if (_NeedsBoundsBuffers()) { bounceBuffer = fBounceBuffers.Head(); if (bounceBuffer == NULL) return B_BUSY; } dmaBuffer->SetBounceBuffer(bounceBuffer); generic_size_t dmaLength = 0; phys_addr_t physicalBounceBuffer = dmaBuffer->PhysicalBounceBufferAddress(); phys_size_t bounceLeft = fBounceBufferSize; generic_size_t transferLeft = totalLength; // If the offset isn't block-aligned, use the bounce buffer to bridge the // gap to the start of the vec. if (partialBegin > 0) { generic_size_t length; if (request->IsWrite()) { // we always need to read in a whole block for the partial write length = fBlockSize; } else { length = (partialBegin + fRestrictions.alignment - 1) & ~(fRestrictions.alignment - 1); } if (_AddBounceBuffer(*dmaBuffer, physicalBounceBuffer, bounceLeft, length, true) == 0) { TRACE(" adding partial begin failed, length %lu!\n", length); return B_BAD_VALUE; } dmaLength += length; generic_size_t transferred = length - partialBegin; vecOffset += transferred; offset -= partialBegin; if (transferLeft > transferred) transferLeft -= transferred; else transferLeft = 0; TRACE(" partial begin, using bounce buffer: offset: %lld, length: " "%lu\n", offset, length); } for (uint32 i = vecIndex; i < vecIndex + segmentCount && transferLeft > 0;) { if (dmaBuffer->VecCount() >= fRestrictions.max_segment_count) break; const generic_io_vec& vec = vecs[i]; if (vec.length <= vecOffset) { vecOffset -= vec.length; i++; continue; } generic_addr_t base = vec.base + vecOffset; generic_size_t maxLength = vec.length - vecOffset; if (maxLength > transferLeft) maxLength = transferLeft; generic_size_t length = maxLength; // Cut the vec according to transfer size, segment size, and boundary. if (dmaLength + length > fRestrictions.max_transfer_size) { length = fRestrictions.max_transfer_size - dmaLength; TRACE(" vec %lu: restricting length to %lu due to transfer size " "limit\n", i, length); } _RestrictBoundaryAndSegmentSize(base, length); phys_size_t useBounceBufferSize = 0; // Check low address: use bounce buffer for range to low address. // Check alignment: if not aligned, use bounce buffer for complete vec. if (base < fRestrictions.low_address) { useBounceBufferSize = fRestrictions.low_address - base; TRACE(" vec %lu: below low address, using bounce buffer: %lu\n", i, useBounceBufferSize); } else if (base & (fRestrictions.alignment - 1)) { useBounceBufferSize = length; TRACE(" vec %lu: misalignment, using bounce buffer: %lu\n", i, useBounceBufferSize); } // Enforce high address restriction if (base > fRestrictions.high_address) useBounceBufferSize = length; else if (base + length > fRestrictions.high_address) length = fRestrictions.high_address - base; // Align length as well if (useBounceBufferSize == 0) length &= ~(fRestrictions.alignment - 1); // If length is 0, use bounce buffer for complete vec. if (length == 0) { length = maxLength; useBounceBufferSize = length; TRACE(" vec %lu: 0 length, using bounce buffer: %lu\n", i, useBounceBufferSize); } if (useBounceBufferSize > 0) { // alignment could still be wrong (we round up here) useBounceBufferSize = (useBounceBufferSize + fRestrictions.alignment - 1) & ~(fRestrictions.alignment - 1); length = _AddBounceBuffer(*dmaBuffer, physicalBounceBuffer, bounceLeft, useBounceBufferSize, false); if (length == 0) { TRACE(" vec %lu: out of bounce buffer space\n", i); // We don't have any bounce buffer space left, we need to move // this request to the next I/O operation. break; } TRACE(" vec %lu: final bounce length: %lu\n", i, length); } else { TRACE(" vec %lu: final length restriction: %lu\n", i, length); dmaBuffer->AddVec(base, length); } dmaLength += length; vecOffset += length; transferLeft -= min_c(length, transferLeft); } // If we're writing partially, we always need to have a block sized bounce // buffer (or else we would overwrite memory to be written on the read in // the first phase). off_t requestEnd = request->Offset() + request->Length(); if (request->IsWrite()) { generic_size_t diff = dmaLength & (fBlockSize - 1); // If the transfer length is block aligned and we're writing past the // end of the given data, we still have to check the whether the last // vec is a bounce buffer segment shorter than the block size. If so, we // have to cut back the complete block and use a bounce buffer for it // entirely. if (diff == 0 && offset + (off_t)dmaLength > requestEnd) { const generic_io_vec& dmaVec = dmaBuffer->VecAt(dmaBuffer->VecCount() - 1); ASSERT(dmaVec.base >= dmaBuffer->PhysicalBounceBufferAddress() && dmaVec.base < dmaBuffer->PhysicalBounceBufferAddress() + fBounceBufferSize); // We can be certain that the last vec is a bounce buffer vec, // since otherwise the DMA buffer couldn't exceed the end of the // request data. if (dmaVec.length < fBlockSize) diff = fBlockSize; } if (diff != 0) { // Not yet block aligned -- cut back to the previous block and add // a block-sized bounce buffer segment. TRACE(" partial end write: %lu, diff %lu\n", dmaLength, diff); _CutBuffer(*dmaBuffer, physicalBounceBuffer, bounceLeft, diff); dmaLength -= diff; if (_AddBounceBuffer(*dmaBuffer, physicalBounceBuffer, bounceLeft, fBlockSize, true) == 0) { // If we cannot write anything, we can't process the request at // all. TRACE(" adding bounce buffer failed!!!\n"); if (dmaLength == 0) return B_BAD_VALUE; } else dmaLength += fBlockSize; } } // If total length not block aligned, use bounce buffer for padding (read // case only). while ((dmaLength & (fBlockSize - 1)) != 0) { TRACE(" dmaLength not block aligned: %lu\n", dmaLength); generic_size_t length = (dmaLength + fBlockSize - 1) & ~(fBlockSize - 1); // If total length > max transfer size, segment count > max segment // count, truncate. // TODO: sometimes we can replace the last vec with the bounce buffer // to let it match the restrictions. if (length > fRestrictions.max_transfer_size || dmaBuffer->VecCount() == fRestrictions.max_segment_count || bounceLeft < length - dmaLength) { // cut the part of dma length TRACE(" can't align length due to max transfer size, segment " "count restrictions, or lacking bounce buffer space\n"); generic_size_t toCut = dmaLength & (max_c(fBlockSize, fRestrictions.alignment) - 1); dmaLength -= toCut; if (dmaLength == 0) { // This can only happen, when we have too many small segments // and hit the max segment count. In this case we just use the // bounce buffer for as much as possible of the total length. dmaBuffer->SetVecCount(0); generic_addr_t base = dmaBuffer->PhysicalBounceBufferAddress(); dmaLength = min_c(totalLength, fBounceBufferSize) & ~(max_c(fBlockSize, fRestrictions.alignment) - 1); _RestrictBoundaryAndSegmentSize(base, dmaLength); dmaBuffer->AddVec(base, dmaLength); physicalBounceBuffer = base + dmaLength; bounceLeft = fBounceBufferSize - dmaLength; } else { _CutBuffer(*dmaBuffer, physicalBounceBuffer, bounceLeft, toCut); } } else { TRACE(" adding %lu bytes final bounce buffer\n", length - dmaLength); length -= dmaLength; length = _AddBounceBuffer(*dmaBuffer, physicalBounceBuffer, bounceLeft, length, true); if (length == 0) panic("don't do this to me!"); dmaLength += length; } } operation->SetBuffer(dmaBuffer); operation->SetBlockSize(fBlockSize); operation->SetOriginalRange(originalOffset, min_c(offset + (off_t)dmaLength, requestEnd) - originalOffset); operation->SetRange(offset, dmaLength); operation->SetPartial(partialBegin != 0, offset + (off_t)dmaLength > requestEnd); // If we don't need the bounce buffer, we put it back, otherwise operation->SetUsesBounceBuffer(bounceLeft < fBounceBufferSize); if (operation->UsesBounceBuffer()) fBounceBuffers.RemoveHead(); else dmaBuffer->SetBounceBuffer(NULL); status_t error = operation->Prepare(request); if (error != B_OK) return error; request->Advance(operation->OriginalLength()); return B_OK; }