/* static */ status_t
IORequest::_CopyUser(void* _bounceBuffer, void* _external, size_t size,
team_id team, bool copyIn)
{
uint8* bounceBuffer = (uint8*)_bounceBuffer;
uint8* external = (uint8*)_external;
while (size > 0) {
static const int32 kEntryCount = 8;
physical_entry entries[kEntryCount];
uint32 count = kEntryCount;
status_t error = get_memory_map_etc(team, external, size, entries,
&count);
if (error != B_OK && error != B_BUFFER_OVERFLOW) {
panic("IORequest::_CopyUser(): Failed to get physical memory for "
"user memory %p\n", external);
return B_BAD_ADDRESS;
}
for (uint32 i = 0; i < count; i++) {
const physical_entry& entry = entries[i];
error = _CopyPhysical(bounceBuffer, entry.address,
entry.size, team, copyIn);
if (error != B_OK)
return error;
size -= entry.size;
bounceBuffer += entry.size;
external += entry.size;
}
}
return B_OK;
}
status_t
AHCIPort::FillPrdTable(volatile prd* prdTable, int* prdCount, int prdMax,
const void* data, size_t dataSize)
{
int maxEntries = prdMax + 1;
physical_entry entries[maxEntries];
uint32 entriesUsed = maxEntries;
status_t status = get_memory_map_etc(B_CURRENT_TEAM, data, dataSize,
entries, &entriesUsed);
if (status != B_OK) {
TRACE("AHCIPort::FillPrdTable get_memory_map() failed: %s\n",
strerror(status));
return B_ERROR;
}
return FillPrdTable(prdTable, prdCount, prdMax, entries, entriesUsed,
dataSize);
}
status_t
DMAResource::TranslateNext(IORequest* request, IOOperation* operation,
generic_size_t maxOperationLength)
{
IOBuffer* buffer = request->Buffer();
off_t originalOffset = request->Offset() + request->Length()
- request->RemainingBytes();
off_t offset = originalOffset;
generic_size_t partialBegin = offset & (fBlockSize - 1);
// current iteration state
uint32 vecIndex = request->VecIndex();
uint32 vecOffset = request->VecOffset();
generic_size_t totalLength = min_c(request->RemainingBytes(),
fRestrictions.max_transfer_size);
if (maxOperationLength > 0
&& maxOperationLength < totalLength + partialBegin) {
totalLength = maxOperationLength - partialBegin;
}
MutexLocker locker(fLock);
DMABuffer* dmaBuffer = fDMABuffers.RemoveHead();
if (dmaBuffer == NULL)
return B_BUSY;
dmaBuffer->SetVecCount(0);
generic_io_vec* vecs = NULL;
uint32 segmentCount = 0;
TRACE(" offset %Ld, remaining size: %lu, block size %lu -> partial: %lu\n",
offset, request->RemainingBytes(), fBlockSize, partialBegin);
if (buffer->IsVirtual()) {
// Unless we need the bounce buffer anyway, we have to translate the
// virtual addresses to physical addresses, so we can check the DMA
// restrictions.
TRACE(" buffer is virtual %s\n", buffer->IsUser() ? "user" : "kernel");
// TODO: !partialOperation || totalLength >= fBlockSize
// TODO: Maybe enforce fBounceBufferSize >= 2 * fBlockSize.
if (true) {
generic_size_t transferLeft = totalLength;
vecs = fScratchVecs;
TRACE(" create physical map (for %ld vecs)\n", buffer->VecCount());
for (uint32 i = vecIndex; i < buffer->VecCount(); i++) {
generic_io_vec& vec = buffer->VecAt(i);
generic_addr_t base = vec.base + vecOffset;
generic_size_t size = vec.length - vecOffset;
vecOffset = 0;
if (size > transferLeft)
size = transferLeft;
while (size > 0 && segmentCount
< fRestrictions.max_segment_count) {
physical_entry entry;
uint32 count = 1;
get_memory_map_etc(request->TeamID(), (void*)base, size,
&entry, &count);
vecs[segmentCount].base = entry.address;
vecs[segmentCount].length = entry.size;
transferLeft -= entry.size;
base += entry.size;
size -= entry.size;
segmentCount++;
}
if (transferLeft == 0)
break;
}
totalLength -= transferLeft;
}
vecIndex = 0;
vecOffset = 0;
} else {
// We do already have physical addresses.
locker.Unlock();
vecs = buffer->Vecs();
segmentCount = min_c(buffer->VecCount() - vecIndex,
fRestrictions.max_segment_count);
}
#ifdef TRACE_DMA_RESOURCE
TRACE(" physical count %lu\n", segmentCount);
for (uint32 i = 0; i < segmentCount; i++) {
TRACE(" [%" B_PRIu32 "] %#" B_PRIxGENADDR ", %" B_PRIxGENADDR "\n",
i, vecs[vecIndex + i].base, vecs[vecIndex + i].length);
}
#endif
// check alignment, boundaries, etc. and set vecs in DMA buffer
// Fetch a bounce buffer we can use for the DMABuffer.
// TODO: We should do that lazily when needed!
DMABounceBuffer* bounceBuffer = NULL;
if (_NeedsBoundsBuffers()) {
bounceBuffer = fBounceBuffers.Head();
if (bounceBuffer == NULL)
return B_BUSY;
}
dmaBuffer->SetBounceBuffer(bounceBuffer);
generic_size_t dmaLength = 0;
phys_addr_t physicalBounceBuffer = dmaBuffer->PhysicalBounceBufferAddress();
phys_size_t bounceLeft = fBounceBufferSize;
generic_size_t transferLeft = totalLength;
// If the offset isn't block-aligned, use the bounce buffer to bridge the
// gap to the start of the vec.
if (partialBegin > 0) {
generic_size_t length;
if (request->IsWrite()) {
// we always need to read in a whole block for the partial write
length = fBlockSize;
} else {
length = (partialBegin + fRestrictions.alignment - 1)
& ~(fRestrictions.alignment - 1);
}
if (_AddBounceBuffer(*dmaBuffer, physicalBounceBuffer, bounceLeft,
length, true) == 0) {
TRACE(" adding partial begin failed, length %lu!\n", length);
return B_BAD_VALUE;
}
dmaLength += length;
generic_size_t transferred = length - partialBegin;
vecOffset += transferred;
offset -= partialBegin;
if (transferLeft > transferred)
transferLeft -= transferred;
else
transferLeft = 0;
TRACE(" partial begin, using bounce buffer: offset: %lld, length: "
"%lu\n", offset, length);
}
for (uint32 i = vecIndex;
i < vecIndex + segmentCount && transferLeft > 0;) {
if (dmaBuffer->VecCount() >= fRestrictions.max_segment_count)
break;
const generic_io_vec& vec = vecs[i];
if (vec.length <= vecOffset) {
vecOffset -= vec.length;
i++;
continue;
}
generic_addr_t base = vec.base + vecOffset;
generic_size_t maxLength = vec.length - vecOffset;
if (maxLength > transferLeft)
maxLength = transferLeft;
generic_size_t length = maxLength;
// Cut the vec according to transfer size, segment size, and boundary.
if (dmaLength + length > fRestrictions.max_transfer_size) {
length = fRestrictions.max_transfer_size - dmaLength;
TRACE(" vec %lu: restricting length to %lu due to transfer size "
"limit\n", i, length);
}
_RestrictBoundaryAndSegmentSize(base, length);
phys_size_t useBounceBufferSize = 0;
// Check low address: use bounce buffer for range to low address.
// Check alignment: if not aligned, use bounce buffer for complete vec.
if (base < fRestrictions.low_address) {
useBounceBufferSize = fRestrictions.low_address - base;
TRACE(" vec %lu: below low address, using bounce buffer: %lu\n", i,
useBounceBufferSize);
} else if (base & (fRestrictions.alignment - 1)) {
useBounceBufferSize = length;
TRACE(" vec %lu: misalignment, using bounce buffer: %lu\n", i,
useBounceBufferSize);
}
// Enforce high address restriction
if (base > fRestrictions.high_address)
useBounceBufferSize = length;
else if (base + length > fRestrictions.high_address)
length = fRestrictions.high_address - base;
// Align length as well
if (useBounceBufferSize == 0)
length &= ~(fRestrictions.alignment - 1);
// If length is 0, use bounce buffer for complete vec.
if (length == 0) {
length = maxLength;
useBounceBufferSize = length;
TRACE(" vec %lu: 0 length, using bounce buffer: %lu\n", i,
useBounceBufferSize);
}
if (useBounceBufferSize > 0) {
// alignment could still be wrong (we round up here)
useBounceBufferSize = (useBounceBufferSize
+ fRestrictions.alignment - 1) & ~(fRestrictions.alignment - 1);
length = _AddBounceBuffer(*dmaBuffer, physicalBounceBuffer,
bounceLeft, useBounceBufferSize, false);
if (length == 0) {
TRACE(" vec %lu: out of bounce buffer space\n", i);
// We don't have any bounce buffer space left, we need to move
// this request to the next I/O operation.
break;
}
TRACE(" vec %lu: final bounce length: %lu\n", i, length);
} else {
TRACE(" vec %lu: final length restriction: %lu\n", i, length);
dmaBuffer->AddVec(base, length);
}
dmaLength += length;
vecOffset += length;
transferLeft -= min_c(length, transferLeft);
}
// If we're writing partially, we always need to have a block sized bounce
// buffer (or else we would overwrite memory to be written on the read in
// the first phase).
off_t requestEnd = request->Offset() + request->Length();
if (request->IsWrite()) {
generic_size_t diff = dmaLength & (fBlockSize - 1);
// If the transfer length is block aligned and we're writing past the
// end of the given data, we still have to check the whether the last
// vec is a bounce buffer segment shorter than the block size. If so, we
// have to cut back the complete block and use a bounce buffer for it
// entirely.
if (diff == 0 && offset + (off_t)dmaLength > requestEnd) {
const generic_io_vec& dmaVec
= dmaBuffer->VecAt(dmaBuffer->VecCount() - 1);
ASSERT(dmaVec.base >= dmaBuffer->PhysicalBounceBufferAddress()
&& dmaVec.base
< dmaBuffer->PhysicalBounceBufferAddress()
+ fBounceBufferSize);
// We can be certain that the last vec is a bounce buffer vec,
// since otherwise the DMA buffer couldn't exceed the end of the
// request data.
if (dmaVec.length < fBlockSize)
diff = fBlockSize;
}
if (diff != 0) {
// Not yet block aligned -- cut back to the previous block and add
// a block-sized bounce buffer segment.
TRACE(" partial end write: %lu, diff %lu\n", dmaLength, diff);
_CutBuffer(*dmaBuffer, physicalBounceBuffer, bounceLeft, diff);
dmaLength -= diff;
if (_AddBounceBuffer(*dmaBuffer, physicalBounceBuffer,
bounceLeft, fBlockSize, true) == 0) {
// If we cannot write anything, we can't process the request at
// all.
TRACE(" adding bounce buffer failed!!!\n");
if (dmaLength == 0)
return B_BAD_VALUE;
} else
dmaLength += fBlockSize;
}
}
// If total length not block aligned, use bounce buffer for padding (read
// case only).
while ((dmaLength & (fBlockSize - 1)) != 0) {
TRACE(" dmaLength not block aligned: %lu\n", dmaLength);
generic_size_t length
= (dmaLength + fBlockSize - 1) & ~(fBlockSize - 1);
// If total length > max transfer size, segment count > max segment
// count, truncate.
// TODO: sometimes we can replace the last vec with the bounce buffer
// to let it match the restrictions.
if (length > fRestrictions.max_transfer_size
|| dmaBuffer->VecCount() == fRestrictions.max_segment_count
|| bounceLeft < length - dmaLength) {
// cut the part of dma length
TRACE(" can't align length due to max transfer size, segment "
"count restrictions, or lacking bounce buffer space\n");
generic_size_t toCut = dmaLength
& (max_c(fBlockSize, fRestrictions.alignment) - 1);
dmaLength -= toCut;
if (dmaLength == 0) {
// This can only happen, when we have too many small segments
// and hit the max segment count. In this case we just use the
// bounce buffer for as much as possible of the total length.
dmaBuffer->SetVecCount(0);
generic_addr_t base = dmaBuffer->PhysicalBounceBufferAddress();
dmaLength = min_c(totalLength, fBounceBufferSize)
& ~(max_c(fBlockSize, fRestrictions.alignment) - 1);
_RestrictBoundaryAndSegmentSize(base, dmaLength);
dmaBuffer->AddVec(base, dmaLength);
physicalBounceBuffer = base + dmaLength;
bounceLeft = fBounceBufferSize - dmaLength;
} else {
_CutBuffer(*dmaBuffer, physicalBounceBuffer, bounceLeft, toCut);
}
} else {
TRACE(" adding %lu bytes final bounce buffer\n",
length - dmaLength);
length -= dmaLength;
length = _AddBounceBuffer(*dmaBuffer, physicalBounceBuffer,
bounceLeft, length, true);
if (length == 0)
panic("don't do this to me!");
dmaLength += length;
}
}
operation->SetBuffer(dmaBuffer);
operation->SetBlockSize(fBlockSize);
operation->SetOriginalRange(originalOffset,
min_c(offset + (off_t)dmaLength, requestEnd) - originalOffset);
operation->SetRange(offset, dmaLength);
operation->SetPartial(partialBegin != 0,
offset + (off_t)dmaLength > requestEnd);
// If we don't need the bounce buffer, we put it back, otherwise
operation->SetUsesBounceBuffer(bounceLeft < fBounceBufferSize);
if (operation->UsesBounceBuffer())
fBounceBuffers.RemoveHead();
else
dmaBuffer->SetBounceBuffer(NULL);
status_t error = operation->Prepare(request);
if (error != B_OK)
return error;
request->Advance(operation->OriginalLength());
return B_OK;
}