char* Get()
{
if (mFreeCnt <= 0) {
QCASSERT(*mFreeListPtr == 0 && mFreeCnt == 0);
return 0;
}
const BufferIndex theIdx = *mFreeListPtr;
QCASSERT(theIdx > 0);
mFreeCnt--;
*mFreeListPtr = mFreeListPtr[theIdx];
return (mStartPtr + ((theIdx - 1) << mBufSizeShift));
}
void
QCDiskQueue::Queue::StopSelf()
{
QCASSERT(mMutex.IsOwned());
mRunFlag = false;
mWorkCond.NotifyAll();
for (int i = 0; i < mThreadCount; i++) {
QCThread& theThread = mThreadsPtr[i];
QCStMutexUnlocker theUnlock(mMutex);
theThread.Join();
}
QCASSERT(mCompletionRunningCount == 0);
if (mRequestsPtr) {
Request* theReqPtr;
while ((theReqPtr = Dequeue())) {
Cancel(*theReqPtr);
}
QCASSERT(mPendingCount == 0);
}
while (mFdCount > 0) {
if (mFdPtr[--mFdCount] >= 0) {
close(mFdPtr[mFdCount]);
}
}
delete [] mFdPtr;
mFdPtr = 0;
delete [] mFilePendingReqCountPtr;
mFilePendingReqCountPtr = 0;
delete [] mFileInfoPtr;
mFileInfoPtr = 0;
delete [] mThreadsPtr;
mThreadsPtr = 0;
delete [] mBuffersPtr;
mBuffersPtr = 0;
mRequestBufferCount = 0;
delete [] mRequestsPtr;
mRequestsPtr = 0;
mFreeCount = 0;
mTotalCount = 0;
mPendingCount = 0;
mBufferPoolPtr = 0;
delete [] mIoVecPtr;
mIoVecPtr = 0;
mIoVecPerThreadCount = 0;
mThreadCount = 0;
mFreeFdHead = kFreeFdEnd;
mFileCount = 0;
mReqWaitersCount = 0;
}
void RequestComplete(
Request& inReq,
Error inError,
int inSysError,
int64_t inIoByteCount,
bool inFreeBuffersIfNoIoCompletion = false)
{
QCASSERT(mMutex.IsOwned());
QCRTASSERT(
mPendingCount > 0 &&
// inReq.mFileIdx >= 0 && always true: unsigned
int(inReq.mFileIdx) < mFileCount &&
mFilePendingReqCountPtr[inReq.mFileIdx] > 0
);
mPendingCount--;
mFilePendingReqCountPtr[inReq.mFileIdx]--;
if (inReq.mReqType == kReqTypeRead) {
mPendingReadBlockCount -= inReq.mBufferCount;
} else if (inReq.mReqType == kReqTypeWrite) {
mPendingWriteBlockCount -= inReq.mBufferCount;
}
BuffersIterator theItr(*this, inReq, inReq.mBufferCount);
inReq.mReqType = kReqTypeNone;
if (! inReq.mIoCompletionPtr) {
if (inFreeBuffersIfNoIoCompletion) {
QCStMutexUnlocker theUnlock(mMutex);
mBufferPoolPtr->Put(theItr, inReq.mBufferCount);
}
return;
}
mCompletionRunningCount++;
IoCompletion* const theIoCompletionPtr = inReq.mIoCompletionPtr;
inReq.mIoCompletionPtr = 0;
{
QCStMutexUnlocker theUnlock(mMutex);
if (! theIoCompletionPtr->Done(
GetRequestId(inReq),
inReq.mFileIdx,
inReq.mBlockIdx,
theItr,
inReq.mBufferCount,
inError,
inSysError,
inIoByteCount)) {
// Free buffers.
BuffersIterator theItr(*this, inReq, inReq.mBufferCount);
mBufferPoolPtr->Put(theItr, inReq.mBufferCount);
}
}
mCompletionRunningCount--;
Put(inReq);
}
/* virtual */ void
QCDiskQueue::Queue::Run(
int inThreadIndex)
{
QCStMutexLocker theLock(mMutex);
QCASSERT(inThreadIndex >= 0 && inThreadIndex < mThreadCount);
int* const theFdPtr = mFdPtr +
mFdCount / mThreadCount * inThreadIndex;
struct iovec* const theIoVecPtr = mIoVecPtr +
mIoVecPerThreadCount * inThreadIndex;
while (mRunFlag) {
Request* theReqPtr;
while (! (theReqPtr = Dequeue()) && mRunFlag) {
mWorkCond.Wait(mMutex);
}
if (mRunFlag) {
QCASSERT(theReqPtr);
Process(*theReqPtr, theFdPtr, theIoVecPtr);
} else if (theReqPtr) {
Cancel(*theReqPtr);
}
}
}
void Put(
Request& inReq)
{
mFreeCount += GetReqListSize(inReq);
inReq.mReqType = kReqTypeNone;
inReq.mInFlightFlag = false;
inReq.mIoCompletionPtr = 0;
inReq.mBufferCount = 0;
Insert(mRequestsPtr[kFreeQueueIdx], inReq);
if (mReqWaitersCount > 0) {
QCASSERT(mFreeCount > 0);
mFreeReqCond.NotifyAll(); // Give them all a chance to retry.
}
}
Request* Dequeue()
{
Request* const theReqPtr = PopFront(kIoQueueIdx);
if (! theReqPtr) {
return 0;
}
// If there are more than one "sub request" then the list head has
// buffer count larger than request max buffers per request.
int theBufCount = theReqPtr->mBufferCount;
while ((theBufCount -= mRequestBufferCount) > 0) {
Request* const thePtr = PopFront(kIoQueueIdx);
QCASSERT(thePtr);
Insert(*theReqPtr, *thePtr);
}
return theReqPtr;
}
Request* Get(
int inBufferCount)
{
int theReqCount = GetReqListSize(inBufferCount);
if (mFreeCount < theReqCount) {
return 0;
}
Request* const theRetPtr = PopFront(kFreeQueueIdx);
if (! theRetPtr) {
return theRetPtr;
}
mFreeCount--;
while (--theReqCount > 0) {
Request* const thePtr = PopFront(kFreeQueueIdx);
QCASSERT(thePtr);
Insert(*theRetPtr, *thePtr);
mFreeCount--;
}
return theRetPtr;
}
QCDiskQueue::EnqueueStatus
QCDiskQueue::Queue::Sync(
QCDiskQueue::FileIdx inFileIdx,
QCDiskQueue::IoCompletion* inIoCompletionPtr,
QCDiskQueue::Time inTimeWaitNanoSec)
{
QCStMutexLocker theLock(mMutex);
if (! mRunFlag) {
return EnqueueStatus(kRequestIdNone, kErrorQueueStopped);
}
if (inFileIdx < 0 || inFileIdx >= mFileCount || mFdPtr[inFileIdx] < 0) {
return EnqueueStatus(kRequestIdNone, kErrorFileIdxOutOfRange);
}
Request* theReqPtr;
if (! (theReqPtr = Get(0))) {
QCStValueIncrementor<int> theIncr(mReqWaitersCount, 1);
if (inTimeWaitNanoSec < 0) {
mFreeReqCond.Wait(mMutex);
} else if (inTimeWaitNanoSec == 0 ||
! mFreeReqCond.Wait(mMutex, inTimeWaitNanoSec)) {
if (inTimeWaitNanoSec != 0) {
theLock.Detach();
}
return EnqueueStatus(kRequestIdNone, kErrorOutOfRequests);
}
}
QCASSERT(theReqPtr);
// FIXME: implement real io barrier, for now just queue empty read request.
Request& theReq = *theReqPtr;
theReq.mReqType = kReqTypeRead;
theReq.mBufferCount = 0;
theReq.mFileIdx = inFileIdx;
theReq.mBlockIdx = 0;
theReq.mIoCompletionPtr = inIoCompletionPtr;
char** const theBufsPtr = GetBuffersPtr(theReq);
theBufsPtr[0] = 0;
return EnqueueStatus(kErrorNone);
}
void
QCDiskQueue::Queue::Process(
Request& inReq,
int* inFdPtr,
struct iovec* inIoVecPtr)
{
QCASSERT(mMutex.IsOwned());
QCASSERT(mIoVecPerThreadCount > 0 && mBufferPoolPtr);
QCRTASSERT(//mFileInfoPtr[inReq.mFileIdx].mLastBlockIdx >= 0 &&
inReq.mBlockIdx + inReq.mBufferCount <=
uint64_t(mFileInfoPtr[inReq.mFileIdx].mLastBlockIdx));
const int theFd = inFdPtr[inReq.mFileIdx];
char** const theBufPtr = GetBuffersPtr(inReq);
const off_t theOffset = (off_t)inReq.mBlockIdx * mBlockSize;
const bool theReadFlag = inReq.mReqType == kReqTypeRead;
const int64_t theAllocSize = (inReq.mReqType == kReqTypeWrite &&
mFileInfoPtr[inReq.mFileIdx].mSpaceAllocPendingFlag) ?
mFileInfoPtr[inReq.mFileIdx].mLastBlockIdx * mBlockSize : 0;
QCASSERT((theReadFlag || inReq.mReqType == kReqTypeWrite) && theFd >= 0);
inReq.mInFlightFlag = true;
QCStMutexUnlocker theUnlock(mMutex);
Error theError = kErrorNone;
int theSysError = 0;
if (theAllocSize > 0) {
// Theoretically space allocation can be simultaneously invoked from
// more than one io thread. This is to ensure that allocation always
// happen before the first write.
// OS can deal with concurrent allocations.
const int64_t theResv = QCUtils::ReserveFileSpace(theFd, theAllocSize);
if (theResv < 0) {
theError = kErrorSpaceAlloc;
theSysError = int(-theResv);
}
if (theResv > 0 && ftruncate(theFd, theAllocSize)) {
theError = kErrorSpaceAlloc;
theSysError = errno;
}
if (theError == kErrorNone) {
QCStMutexLocker theLock(mMutex);
mFileInfoPtr[inReq.mFileIdx].mSpaceAllocPendingFlag = false;
}
}
const bool theGetBufFlag = ! theBufPtr[0];
if (theError == kErrorNone && theGetBufFlag) {
QCASSERT(theReadFlag);
BuffersIterator theIt(*this, inReq, inReq.mBufferCount);
// Allocate buffers for read request.
if (! mBufferPoolPtr->Get(theIt, inReq.mBufferCount,
QCIoBufferPool::kRefillReqIdRead)) {
theError = kErrorOutOfBuffers;
}
}
if (theError == kErrorNone &&
lseek(theFd, theOffset, SEEK_SET) != theOffset) {
theError = kErrorSeek;
theSysError = errno;
}
BuffersIterator theItr(*this, inReq, inReq.mBufferCount);
int theBufCnt = inReq.mBufferCount;
int64_t theIoByteCnt = 0;
while (theBufCnt > 0 && theError == kErrorNone) {
ssize_t theIoBytes = 0;
int theIoVecCnt = 0;
char* thePtr;
while (theIoVecCnt < mIoVecPerThreadCount && (thePtr = theItr.Get())) {
inIoVecPtr[theIoVecCnt ].iov_base = thePtr;
inIoVecPtr[theIoVecCnt++].iov_len = mBlockSize;
theIoBytes += mBlockSize;
theBufCnt--;
}
QCRTASSERT(theIoVecCnt > 0);
if (theReadFlag) {
const ssize_t theNRd = readv(theFd, inIoVecPtr, theIoVecCnt);
if (theNRd < 0) {
theError = kErrorRead;
theSysError = theNRd < 0 ? errno : 0;
break;
}
theIoByteCnt += theNRd;
if (theNRd < theIoBytes) {
if (theGetBufFlag) {
// Short read -- release extra buffers.
mBufferPoolPtr->Put(theItr, theBufCnt);
inReq.mBufferCount -= theBufCnt;
int i = (theNRd + mBlockSize - 1) / mBlockSize;
inReq.mBufferCount -= theIoVecCnt - i;
while (i < theIoVecCnt) {
mBufferPoolPtr->Put((char*)inIoVecPtr[i++].iov_base);
}
}
break;
}
} else {
const ssize_t theNWr = writev(theFd, inIoVecPtr, theIoVecCnt);
if (theNWr > 0) {
theIoByteCnt += theNWr;
}
if (theNWr != theIoBytes) {
theError = kErrorWrite;
theSysError = errno;
break;
}
}
}
if (theGetBufFlag && theError != kErrorNone && theBufPtr[0]) {
BuffersIterator theIt(*this, inReq, inReq.mBufferCount);
mBufferPoolPtr->Put(theIt, inReq.mBufferCount);
theBufPtr[0] = 0;
}
theUnlock.Lock();
RequestComplete(inReq, theError, theSysError, theIoByteCnt, theGetBufFlag);
}
QCDiskQueue::EnqueueStatus
QCDiskQueue::Queue::Enqueue(
QCDiskQueue::ReqType inReqType,
QCDiskQueue::FileIdx inFileIdx,
QCDiskQueue::BlockIdx inBlockIdx,
QCDiskQueue::InputIterator* inBufferIteratorPtr,
int inBufferCount,
QCDiskQueue::IoCompletion* inIoCompletionPtr,
QCDiskQueue::Time inTimeWaitNanoSec)
{
if ((inReqType != kReqTypeRead && inReqType != kReqTypeWrite) ||
inBufferCount <= 0 ||
inBufferCount > (mRequestBufferCount *
(mTotalCount - kRequestQueueCount)) ||
(! inBufferIteratorPtr && inReqType == kReqTypeWrite)) {
return EnqueueStatus(kRequestIdNone, kErrorParameter);
}
QCStMutexLocker theLock(mMutex);
if (! mRunFlag) {
return EnqueueStatus(kRequestIdNone, kErrorQueueStopped);
}
if (inFileIdx < 0 || inFileIdx >= mFileCount || mFdPtr[inFileIdx] < 0) {
return EnqueueStatus(kRequestIdNone, kErrorFileIdxOutOfRange);
}
if (inBlockIdx < 0 ||
inBlockIdx + (inBufferIteratorPtr ? 0 : inBufferCount) >
int64_t(mFileInfoPtr[inFileIdx].mLastBlockIdx)) {
return EnqueueStatus(kRequestIdNone, kErrorBlockIdxOutOfRange);
}
Request* theReqPtr;
while (! (theReqPtr = Get(inBufferCount))) {
QCStValueIncrementor<int> theIncr(mReqWaitersCount, 1);
if (inTimeWaitNanoSec < 0) {
mFreeReqCond.Wait(mMutex);
} else if (inTimeWaitNanoSec == 0 ||
! mFreeReqCond.Wait(mMutex, inTimeWaitNanoSec)) {
if (inTimeWaitNanoSec != 0) {
theLock.Detach();
}
return EnqueueStatus(kRequestIdNone, kErrorOutOfRequests);
}
}
QCASSERT(theReqPtr);
Request& theReq = *theReqPtr;
theReq.mReqType = inReqType;
theReq.mBufferCount = 0;
theReq.mFileIdx = inFileIdx;
theReq.mBlockIdx = inBlockIdx;
theReq.mIoCompletionPtr = inIoCompletionPtr;
if (inBufferIteratorPtr) {
BuffersIterator theItr(*this, theReq, inBufferCount);
for (int i = 0; i < inBufferCount; i++) {
char* const thePtr = inBufferIteratorPtr->Get();
if (! thePtr) {
break;
}
theItr.Put(thePtr);
theReq.mBufferCount++;
}
if (theReq.mBufferCount < inBufferCount) {
// Free unused requests if any.
TrimRequestList(theReq, theReq.mBufferCount);
}
} else if (inReqType == kReqTypeRead) {
// Defer buffer allocation.
GetBuffersPtr(theReq)[0] = 0;
theReq.mBufferCount = inBufferCount;
}
if (theReq.mBlockIdx + theReq.mBufferCount >
uint64_t(mFileInfoPtr[theReq.mFileIdx].mLastBlockIdx)) {
Put(theReq);
return EnqueueStatus(kRequestIdNone, kErrorBlockIdxOutOfRange);
}
if (theReq.mBufferCount <= 0) {
Put(theReq);
return EnqueueStatus(kRequestIdNone, kErrorBlockCountOutOfRange);
}
Enqueue(theReq);
mWorkCond.Notify();
return GetRequestId(theReq);
}
virtual void Run()
{
QCASSERT(mQueuePtr);
mQueuePtr->Run(mThreadIndex);
}
~QCStartedThreadList()
{ QCASSERT(mCount == 0); }
