char* Get() { if (mFreeCnt <= 0) { QCASSERT(*mFreeListPtr == 0 && mFreeCnt == 0); return 0; } const BufferIndex theIdx = *mFreeListPtr; QCASSERT(theIdx > 0); mFreeCnt--; *mFreeListPtr = mFreeListPtr[theIdx]; return (mStartPtr + ((theIdx - 1) << mBufSizeShift)); }
void QCDiskQueue::Queue::StopSelf() { QCASSERT(mMutex.IsOwned()); mRunFlag = false; mWorkCond.NotifyAll(); for (int i = 0; i < mThreadCount; i++) { QCThread& theThread = mThreadsPtr[i]; QCStMutexUnlocker theUnlock(mMutex); theThread.Join(); } QCASSERT(mCompletionRunningCount == 0); if (mRequestsPtr) { Request* theReqPtr; while ((theReqPtr = Dequeue())) { Cancel(*theReqPtr); } QCASSERT(mPendingCount == 0); } while (mFdCount > 0) { if (mFdPtr[--mFdCount] >= 0) { close(mFdPtr[mFdCount]); } } delete [] mFdPtr; mFdPtr = 0; delete [] mFilePendingReqCountPtr; mFilePendingReqCountPtr = 0; delete [] mFileInfoPtr; mFileInfoPtr = 0; delete [] mThreadsPtr; mThreadsPtr = 0; delete [] mBuffersPtr; mBuffersPtr = 0; mRequestBufferCount = 0; delete [] mRequestsPtr; mRequestsPtr = 0; mFreeCount = 0; mTotalCount = 0; mPendingCount = 0; mBufferPoolPtr = 0; delete [] mIoVecPtr; mIoVecPtr = 0; mIoVecPerThreadCount = 0; mThreadCount = 0; mFreeFdHead = kFreeFdEnd; mFileCount = 0; mReqWaitersCount = 0; }
void RequestComplete( Request& inReq, Error inError, int inSysError, int64_t inIoByteCount, bool inFreeBuffersIfNoIoCompletion = false) { QCASSERT(mMutex.IsOwned()); QCRTASSERT( mPendingCount > 0 && // inReq.mFileIdx >= 0 && always true: unsigned int(inReq.mFileIdx) < mFileCount && mFilePendingReqCountPtr[inReq.mFileIdx] > 0 ); mPendingCount--; mFilePendingReqCountPtr[inReq.mFileIdx]--; if (inReq.mReqType == kReqTypeRead) { mPendingReadBlockCount -= inReq.mBufferCount; } else if (inReq.mReqType == kReqTypeWrite) { mPendingWriteBlockCount -= inReq.mBufferCount; } BuffersIterator theItr(*this, inReq, inReq.mBufferCount); inReq.mReqType = kReqTypeNone; if (! inReq.mIoCompletionPtr) { if (inFreeBuffersIfNoIoCompletion) { QCStMutexUnlocker theUnlock(mMutex); mBufferPoolPtr->Put(theItr, inReq.mBufferCount); } return; } mCompletionRunningCount++; IoCompletion* const theIoCompletionPtr = inReq.mIoCompletionPtr; inReq.mIoCompletionPtr = 0; { QCStMutexUnlocker theUnlock(mMutex); if (! theIoCompletionPtr->Done( GetRequestId(inReq), inReq.mFileIdx, inReq.mBlockIdx, theItr, inReq.mBufferCount, inError, inSysError, inIoByteCount)) { // Free buffers. BuffersIterator theItr(*this, inReq, inReq.mBufferCount); mBufferPoolPtr->Put(theItr, inReq.mBufferCount); } } mCompletionRunningCount--; Put(inReq); }
/* virtual */ void QCDiskQueue::Queue::Run( int inThreadIndex) { QCStMutexLocker theLock(mMutex); QCASSERT(inThreadIndex >= 0 && inThreadIndex < mThreadCount); int* const theFdPtr = mFdPtr + mFdCount / mThreadCount * inThreadIndex; struct iovec* const theIoVecPtr = mIoVecPtr + mIoVecPerThreadCount * inThreadIndex; while (mRunFlag) { Request* theReqPtr; while (! (theReqPtr = Dequeue()) && mRunFlag) { mWorkCond.Wait(mMutex); } if (mRunFlag) { QCASSERT(theReqPtr); Process(*theReqPtr, theFdPtr, theIoVecPtr); } else if (theReqPtr) { Cancel(*theReqPtr); } } }
void Put( Request& inReq) { mFreeCount += GetReqListSize(inReq); inReq.mReqType = kReqTypeNone; inReq.mInFlightFlag = false; inReq.mIoCompletionPtr = 0; inReq.mBufferCount = 0; Insert(mRequestsPtr[kFreeQueueIdx], inReq); if (mReqWaitersCount > 0) { QCASSERT(mFreeCount > 0); mFreeReqCond.NotifyAll(); // Give them all a chance to retry. } }
Request* Dequeue() { Request* const theReqPtr = PopFront(kIoQueueIdx); if (! theReqPtr) { return 0; } // If there are more than one "sub request" then the list head has // buffer count larger than request max buffers per request. int theBufCount = theReqPtr->mBufferCount; while ((theBufCount -= mRequestBufferCount) > 0) { Request* const thePtr = PopFront(kIoQueueIdx); QCASSERT(thePtr); Insert(*theReqPtr, *thePtr); } return theReqPtr; }
Request* Get( int inBufferCount) { int theReqCount = GetReqListSize(inBufferCount); if (mFreeCount < theReqCount) { return 0; } Request* const theRetPtr = PopFront(kFreeQueueIdx); if (! theRetPtr) { return theRetPtr; } mFreeCount--; while (--theReqCount > 0) { Request* const thePtr = PopFront(kFreeQueueIdx); QCASSERT(thePtr); Insert(*theRetPtr, *thePtr); mFreeCount--; } return theRetPtr; }
QCDiskQueue::EnqueueStatus QCDiskQueue::Queue::Sync( QCDiskQueue::FileIdx inFileIdx, QCDiskQueue::IoCompletion* inIoCompletionPtr, QCDiskQueue::Time inTimeWaitNanoSec) { QCStMutexLocker theLock(mMutex); if (! mRunFlag) { return EnqueueStatus(kRequestIdNone, kErrorQueueStopped); } if (inFileIdx < 0 || inFileIdx >= mFileCount || mFdPtr[inFileIdx] < 0) { return EnqueueStatus(kRequestIdNone, kErrorFileIdxOutOfRange); } Request* theReqPtr; if (! (theReqPtr = Get(0))) { QCStValueIncrementor<int> theIncr(mReqWaitersCount, 1); if (inTimeWaitNanoSec < 0) { mFreeReqCond.Wait(mMutex); } else if (inTimeWaitNanoSec == 0 || ! mFreeReqCond.Wait(mMutex, inTimeWaitNanoSec)) { if (inTimeWaitNanoSec != 0) { theLock.Detach(); } return EnqueueStatus(kRequestIdNone, kErrorOutOfRequests); } } QCASSERT(theReqPtr); // FIXME: implement real io barrier, for now just queue empty read request. Request& theReq = *theReqPtr; theReq.mReqType = kReqTypeRead; theReq.mBufferCount = 0; theReq.mFileIdx = inFileIdx; theReq.mBlockIdx = 0; theReq.mIoCompletionPtr = inIoCompletionPtr; char** const theBufsPtr = GetBuffersPtr(theReq); theBufsPtr[0] = 0; return EnqueueStatus(kErrorNone); }
void QCDiskQueue::Queue::Process( Request& inReq, int* inFdPtr, struct iovec* inIoVecPtr) { QCASSERT(mMutex.IsOwned()); QCASSERT(mIoVecPerThreadCount > 0 && mBufferPoolPtr); QCRTASSERT(//mFileInfoPtr[inReq.mFileIdx].mLastBlockIdx >= 0 && inReq.mBlockIdx + inReq.mBufferCount <= uint64_t(mFileInfoPtr[inReq.mFileIdx].mLastBlockIdx)); const int theFd = inFdPtr[inReq.mFileIdx]; char** const theBufPtr = GetBuffersPtr(inReq); const off_t theOffset = (off_t)inReq.mBlockIdx * mBlockSize; const bool theReadFlag = inReq.mReqType == kReqTypeRead; const int64_t theAllocSize = (inReq.mReqType == kReqTypeWrite && mFileInfoPtr[inReq.mFileIdx].mSpaceAllocPendingFlag) ? mFileInfoPtr[inReq.mFileIdx].mLastBlockIdx * mBlockSize : 0; QCASSERT((theReadFlag || inReq.mReqType == kReqTypeWrite) && theFd >= 0); inReq.mInFlightFlag = true; QCStMutexUnlocker theUnlock(mMutex); Error theError = kErrorNone; int theSysError = 0; if (theAllocSize > 0) { // Theoretically space allocation can be simultaneously invoked from // more than one io thread. This is to ensure that allocation always // happen before the first write. // OS can deal with concurrent allocations. const int64_t theResv = QCUtils::ReserveFileSpace(theFd, theAllocSize); if (theResv < 0) { theError = kErrorSpaceAlloc; theSysError = int(-theResv); } if (theResv > 0 && ftruncate(theFd, theAllocSize)) { theError = kErrorSpaceAlloc; theSysError = errno; } if (theError == kErrorNone) { QCStMutexLocker theLock(mMutex); mFileInfoPtr[inReq.mFileIdx].mSpaceAllocPendingFlag = false; } } const bool theGetBufFlag = ! theBufPtr[0]; if (theError == kErrorNone && theGetBufFlag) { QCASSERT(theReadFlag); BuffersIterator theIt(*this, inReq, inReq.mBufferCount); // Allocate buffers for read request. if (! mBufferPoolPtr->Get(theIt, inReq.mBufferCount, QCIoBufferPool::kRefillReqIdRead)) { theError = kErrorOutOfBuffers; } } if (theError == kErrorNone && lseek(theFd, theOffset, SEEK_SET) != theOffset) { theError = kErrorSeek; theSysError = errno; } BuffersIterator theItr(*this, inReq, inReq.mBufferCount); int theBufCnt = inReq.mBufferCount; int64_t theIoByteCnt = 0; while (theBufCnt > 0 && theError == kErrorNone) { ssize_t theIoBytes = 0; int theIoVecCnt = 0; char* thePtr; while (theIoVecCnt < mIoVecPerThreadCount && (thePtr = theItr.Get())) { inIoVecPtr[theIoVecCnt ].iov_base = thePtr; inIoVecPtr[theIoVecCnt++].iov_len = mBlockSize; theIoBytes += mBlockSize; theBufCnt--; } QCRTASSERT(theIoVecCnt > 0); if (theReadFlag) { const ssize_t theNRd = readv(theFd, inIoVecPtr, theIoVecCnt); if (theNRd < 0) { theError = kErrorRead; theSysError = theNRd < 0 ? errno : 0; break; } theIoByteCnt += theNRd; if (theNRd < theIoBytes) { if (theGetBufFlag) { // Short read -- release extra buffers. mBufferPoolPtr->Put(theItr, theBufCnt); inReq.mBufferCount -= theBufCnt; int i = (theNRd + mBlockSize - 1) / mBlockSize; inReq.mBufferCount -= theIoVecCnt - i; while (i < theIoVecCnt) { mBufferPoolPtr->Put((char*)inIoVecPtr[i++].iov_base); } } break; } } else { const ssize_t theNWr = writev(theFd, inIoVecPtr, theIoVecCnt); if (theNWr > 0) { theIoByteCnt += theNWr; } if (theNWr != theIoBytes) { theError = kErrorWrite; theSysError = errno; break; } } } if (theGetBufFlag && theError != kErrorNone && theBufPtr[0]) { BuffersIterator theIt(*this, inReq, inReq.mBufferCount); mBufferPoolPtr->Put(theIt, inReq.mBufferCount); theBufPtr[0] = 0; } theUnlock.Lock(); RequestComplete(inReq, theError, theSysError, theIoByteCnt, theGetBufFlag); }
QCDiskQueue::EnqueueStatus QCDiskQueue::Queue::Enqueue( QCDiskQueue::ReqType inReqType, QCDiskQueue::FileIdx inFileIdx, QCDiskQueue::BlockIdx inBlockIdx, QCDiskQueue::InputIterator* inBufferIteratorPtr, int inBufferCount, QCDiskQueue::IoCompletion* inIoCompletionPtr, QCDiskQueue::Time inTimeWaitNanoSec) { if ((inReqType != kReqTypeRead && inReqType != kReqTypeWrite) || inBufferCount <= 0 || inBufferCount > (mRequestBufferCount * (mTotalCount - kRequestQueueCount)) || (! inBufferIteratorPtr && inReqType == kReqTypeWrite)) { return EnqueueStatus(kRequestIdNone, kErrorParameter); } QCStMutexLocker theLock(mMutex); if (! mRunFlag) { return EnqueueStatus(kRequestIdNone, kErrorQueueStopped); } if (inFileIdx < 0 || inFileIdx >= mFileCount || mFdPtr[inFileIdx] < 0) { return EnqueueStatus(kRequestIdNone, kErrorFileIdxOutOfRange); } if (inBlockIdx < 0 || inBlockIdx + (inBufferIteratorPtr ? 0 : inBufferCount) > int64_t(mFileInfoPtr[inFileIdx].mLastBlockIdx)) { return EnqueueStatus(kRequestIdNone, kErrorBlockIdxOutOfRange); } Request* theReqPtr; while (! (theReqPtr = Get(inBufferCount))) { QCStValueIncrementor<int> theIncr(mReqWaitersCount, 1); if (inTimeWaitNanoSec < 0) { mFreeReqCond.Wait(mMutex); } else if (inTimeWaitNanoSec == 0 || ! mFreeReqCond.Wait(mMutex, inTimeWaitNanoSec)) { if (inTimeWaitNanoSec != 0) { theLock.Detach(); } return EnqueueStatus(kRequestIdNone, kErrorOutOfRequests); } } QCASSERT(theReqPtr); Request& theReq = *theReqPtr; theReq.mReqType = inReqType; theReq.mBufferCount = 0; theReq.mFileIdx = inFileIdx; theReq.mBlockIdx = inBlockIdx; theReq.mIoCompletionPtr = inIoCompletionPtr; if (inBufferIteratorPtr) { BuffersIterator theItr(*this, theReq, inBufferCount); for (int i = 0; i < inBufferCount; i++) { char* const thePtr = inBufferIteratorPtr->Get(); if (! thePtr) { break; } theItr.Put(thePtr); theReq.mBufferCount++; } if (theReq.mBufferCount < inBufferCount) { // Free unused requests if any. TrimRequestList(theReq, theReq.mBufferCount); } } else if (inReqType == kReqTypeRead) { // Defer buffer allocation. GetBuffersPtr(theReq)[0] = 0; theReq.mBufferCount = inBufferCount; } if (theReq.mBlockIdx + theReq.mBufferCount > uint64_t(mFileInfoPtr[theReq.mFileIdx].mLastBlockIdx)) { Put(theReq); return EnqueueStatus(kRequestIdNone, kErrorBlockIdxOutOfRange); } if (theReq.mBufferCount <= 0) { Put(theReq); return EnqueueStatus(kRequestIdNone, kErrorBlockCountOutOfRange); } Enqueue(theReq); mWorkCond.Notify(); return GetRequestId(theReq); }
virtual void Run() { QCASSERT(mQueuePtr); mQueuePtr->Run(mThreadIndex); }
~QCStartedThreadList() { QCASSERT(mCount == 0); }