int basic_async_logger<traits>::commit(const struct timespec* tsp)
{
ASYNC_TRACE(("Committing head: %p\n", m_head));
int l_old_val = m_event.value();
while (!m_head) {
l_old_val = m_event.wait(tsp, &l_old_val);
if (m_cancel && !m_head)
return 0;
}
static const log_msg_type* l_new_head = NULL;
const log_msg_type* l_cur_head;
do {
l_cur_head = const_cast<const log_msg_type*>( m_head );
} while( !atomic::cas(&m_head, l_cur_head, l_new_head) );
ASYNC_TRACE(( " --> cur head: %p, new head: %p\n", l_cur_head, m_head));
BOOST_ASSERT(l_cur_head);
int count = 0;
const log_msg_type* l_last = NULL;
const log_msg_type* l_next = l_cur_head;
// Reverse the section of this list
for(const log_msg_type* p = l_cur_head; l_next; count++, p = l_next) {
ASYNC_TRACE(("Set last[%p] -> next[%p]\n", l_next, l_last));
l_next = p->next();
p->next(l_last);
l_last = p;
}
BOOST_ASSERT(l_last);
ASYNC_TRACE(("Total (%d). Sublist's head: %p (%s)\n", count, l_last, l_last->c_str()));
if (m_max_queue_size < count)
m_max_queue_size = count;
l_next = l_last;
for(const log_msg_type* p = l_last; l_next; p = l_next) {
if (::fwrite(p->c_str(), p->size(), 1, m_file) < 0)
return -1;
l_next = p->next();
p->next(NULL);
ASYNC_TRACE(("Wrote (%d bytes): %p (next: %p): %s\n",
p->size(), p, l_next, p->c_str()));
delete p;
}
if (::fflush(m_file) < 0)
return -2;
return 0;
}
/*
** Add an entry to the end of the global write-op list. pWrite should point
** to an AsyncWrite structure allocated using sqlite3OsMalloc(). The writer
** thread will call sqlite3OsFree() to free the structure after the specified
** operation has been completed.
**
** Once an AsyncWrite structure has been added to the list, it becomes the
** property of the writer thread and must not be read or modified by the
** caller.
*/
static void addAsyncWrite(AsyncWrite *pWrite){
/* We must hold the queue mutex in order to modify the queue pointers */
pthread_mutex_lock(&async.queueMutex);
/* Add the record to the end of the write-op queue */
assert( !pWrite->pNext );
if( async.pQueueLast ){
assert( async.pQueueFirst );
async.pQueueLast->pNext = pWrite;
}else{
async.pQueueFirst = pWrite;
}
async.pQueueLast = pWrite;
ASYNC_TRACE(("PUSH %p (%s %s %d)\n", pWrite, azOpcodeName[pWrite->op],
pWrite->pFile ? pWrite->pFile->zName : "-", pWrite->iOffset));
if( pWrite->op==ASYNC_CLOSE ){
async.nFile--;
if( async.nFile==0 ){
async.ioError = SQLITE_OK;
}
}
/* Drop the queue mutex */
pthread_mutex_unlock(&async.queueMutex);
/* The writer thread might have been idle because there was nothing
** on the write-op queue for it to do. So wake it up. */
pthread_cond_signal(&async.queueSignal);
}
/*
** No disk locking is performed. We keep track of locks locally in
** the async.aLock hash table. Locking should appear to work the same
** as with standard (unmodified) SQLite as long as all connections
** come from this one process. Connections from external processes
** cannot see our internal hash table (obviously) and will thus not
** honor our locks.
*/
static int asyncLock(OsFile *id, int lockType){
AsyncFile *pFile = (AsyncFile*)id;
ASYNC_TRACE(("LOCK %d (%s)\n", lockType, pFile->zName));
pthread_mutex_lock(&async.lockMutex);
sqlite3HashInsert(&async.aLock, pFile->zName, pFile->nName, (void*)lockType);
pthread_mutex_unlock(&async.lockMutex);
return SQLITE_OK;
}
/*
** This function is called when the pager layer first opens a database file
** and is checking for a hot-journal.
*/
static int asyncCheckReservedLock(OsFile *id){
AsyncFile *pFile = (AsyncFile*)id;
int rc;
pthread_mutex_lock(&async.lockMutex);
rc = (int)sqlite3HashFind(&async.aLock, pFile->zName, pFile->nName);
pthread_mutex_unlock(&async.lockMutex);
ASYNC_TRACE(("CHECK-LOCK %d (%s)\n", rc, pFile->zName));
return rc>SHARED_LOCK;
}
void basic_async_logger<traits>::stop()
{
if (!m_file)
return;
m_cancel = true;
ASYNC_TRACE(("Stopping async logger (head %p)\n", m_head));
m_event.signal();
if (m_thread)
m_thread->join();
}
void basic_async_logger<traits>::run(boost::barrier* a_barrier)
{
// Wait until the caller is ready
a_barrier->wait();
ASYNC_TRACE(("Started async logging thread (cancel=%s)\n",
m_cancel ? "true" : "false"));
static const timespec ts =
{traits::commit_timeout / 1000, (traits::commit_timeout % 1000) * 1000000 };
while (1) {
int n = commit(&ts);
ASYNC_TRACE(( "Async thread result: %d (head: %p, cancel=%s)\n",
n, m_head, m_cancel ? "true" : "false" ));
if (n || (!m_head && m_cancel))
break;
}
::fclose(m_file);
m_file = NULL;
}
int basic_async_logger<traits>::internal_write(const log_msg_type* msg) {
if (!msg)
return -9;
const log_msg_type* l_last_head;
do {
l_last_head = const_cast<const log_msg_type*>(m_head);
msg->next( l_last_head );
} while( !atomic::cas(&m_head, l_last_head, msg) );
if (!l_last_head)
m_event.signal();
ASYNC_TRACE(("internal_write - cur head: %p, prev head: %p\n",
m_head, l_last_head));
return 0;
}
/*
** Implementation of sqlite3OsFileExists. Return true if file 'z' exists
** in the file system.
**
** This method holds the mutex from start to finish.
*/
static int asyncFileExists(const char *z){
int ret;
AsyncWrite *p;
pthread_mutex_lock(&async.queueMutex);
/* See if the real file system contains the specified file. */
ret = xOrigFileExists(z);
for(p=async.pQueueFirst; p; p = p->pNext){
if( p->op==ASYNC_DELETE && 0==strcmp(p->zBuf, z) ){
ret = 0;
}else if( p->op==ASYNC_OPENEXCLUSIVE && 0==strcmp(p->zBuf, z) ){
ret = 1;
}
}
ASYNC_TRACE(("EXISTS: %s = %d\n", z, ret));
pthread_mutex_unlock(&async.queueMutex);
return ret;
}
/*
** This procedure runs in a separate thread, reading messages off of the
** write queue and processing them one by one.
**
** If async.writerHaltNow is true, then this procedure exits
** after processing a single message.
**
** If async.writerHaltWhenIdle is true, then this procedure exits when
** the write queue is empty.
**
** If both of the above variables are false, this procedure runs
** indefinately, waiting for operations to be added to the write queue
** and processing them in the order in which they arrive.
**
** An artifical delay of async.ioDelay milliseconds is inserted before
** each write operation in order to simulate the effect of a slow disk.
**
** Only one instance of this procedure may be running at a time.
*/
static void *asyncWriterThread(void *NotUsed){
AsyncWrite *p = 0;
int rc = SQLITE_OK;
int holdingMutex = 0;
if( pthread_mutex_trylock(&async.writerMutex) ){
return 0;
}
while( async.writerHaltNow==0 ){
OsFile *pBase = 0;
if( !holdingMutex ){
pthread_mutex_lock(&async.queueMutex);
}
while( (p = async.pQueueFirst)==0 ){
pthread_cond_broadcast(&async.emptySignal);
if( async.writerHaltWhenIdle ){
pthread_mutex_unlock(&async.queueMutex);
break;
}else{
ASYNC_TRACE(("IDLE\n"));
pthread_cond_wait(&async.queueSignal, &async.queueMutex);
ASYNC_TRACE(("WAKEUP\n"));
}
}
if( p==0 ) break;
holdingMutex = 1;
/* Right now this thread is holding the mutex on the write-op queue.
** Variable 'p' points to the first entry in the write-op queue. In
** the general case, we hold on to the mutex for the entire body of
** the loop.
**
** However in the cases enumerated below, we relinquish the mutex,
** perform the IO, and then re-request the mutex before removing 'p' from
** the head of the write-op queue. The idea is to increase concurrency with
** sqlite threads.
**
** * An ASYNC_CLOSE operation.
** * An ASYNC_OPENEXCLUSIVE operation. For this one, we relinquish
** the mutex, call the underlying xOpenExclusive() function, then
** re-aquire the mutex before seting the AsyncFile.pBaseRead
** variable.
** * ASYNC_SYNC and ASYNC_WRITE operations, if
** SQLITE_ASYNC_TWO_FILEHANDLES was set at compile time and two
** file-handles are open for the particular file being "synced".
*/
if( async.ioError!=SQLITE_OK && p->op!=ASYNC_CLOSE ){
p->op = ASYNC_NOOP;
}
if( p->pFile ){
pBase = p->pFile->pBaseWrite;
if(
p->op==ASYNC_CLOSE ||
p->op==ASYNC_OPENEXCLUSIVE ||
(pBase && (p->op==ASYNC_SYNC || p->op==ASYNC_WRITE) )
){
pthread_mutex_unlock(&async.queueMutex);
holdingMutex = 0;
}
if( !pBase ){
pBase = p->pFile->pBaseRead;
}
}
switch( p->op ){
case ASYNC_NOOP:
break;
case ASYNC_WRITE:
assert( pBase );
ASYNC_TRACE(("WRITE %s %d bytes at %d\n",
p->pFile->zName, p->nByte, p->iOffset));
rc = sqlite3OsSeek(pBase, p->iOffset);
if( rc==SQLITE_OK ){
rc = sqlite3OsWrite(pBase, (const void *)(p->zBuf), p->nByte);
}
break;
case ASYNC_SYNC:
assert( pBase );
ASYNC_TRACE(("SYNC %s\n", p->pFile->zName));
rc = sqlite3OsSync(pBase, p->nByte);
break;
case ASYNC_TRUNCATE:
assert( pBase );
ASYNC_TRACE(("TRUNCATE %s to %d bytes\n", p->pFile->zName, p->iOffset));
rc = sqlite3OsTruncate(pBase, p->iOffset);
break;
case ASYNC_CLOSE:
ASYNC_TRACE(("CLOSE %s\n", p->pFile->zName));
sqlite3OsClose(&p->pFile->pBaseWrite);
sqlite3OsClose(&p->pFile->pBaseRead);
sqlite3OsFree(p->pFile);
break;
case ASYNC_OPENDIRECTORY:
assert( pBase );
ASYNC_TRACE(("OPENDIR %s\n", p->zBuf));
sqlite3OsOpenDirectory(pBase, p->zBuf);
break;
case ASYNC_SETFULLSYNC:
assert( pBase );
ASYNC_TRACE(("SETFULLSYNC %s %d\n", p->pFile->zName, p->nByte));
sqlite3OsSetFullSync(pBase, p->nByte);
break;
case ASYNC_DELETE:
ASYNC_TRACE(("DELETE %s\n", p->zBuf));
rc = xOrigDelete(p->zBuf);
break;
case ASYNC_SYNCDIRECTORY:
ASYNC_TRACE(("SYNCDIR %s\n", p->zBuf));
rc = xOrigSyncDirectory(p->zBuf);
break;
case ASYNC_OPENEXCLUSIVE: {
AsyncFile *pFile = p->pFile;
int delFlag = ((p->iOffset)?1:0);
OsFile *pBase = 0;
ASYNC_TRACE(("OPEN %s delFlag=%d\n", p->zBuf, delFlag));
assert(pFile->pBaseRead==0 && pFile->pBaseWrite==0);
rc = xOrigOpenExclusive(p->zBuf, &pBase, delFlag);
assert( holdingMutex==0 );
pthread_mutex_lock(&async.queueMutex);
holdingMutex = 1;
if( rc==SQLITE_OK ){
pFile->pBaseRead = pBase;
}
break;
}
default: assert(!"Illegal value for AsyncWrite.op");
}
/* If we didn't hang on to the mutex during the IO op, obtain it now
** so that the AsyncWrite structure can be safely removed from the
** global write-op queue.
*/
if( !holdingMutex ){
pthread_mutex_lock(&async.queueMutex);
holdingMutex = 1;
}
/* ASYNC_TRACE(("UNLINK %p\n", p)); */
if( p==async.pQueueLast ){
async.pQueueLast = 0;
}
async.pQueueFirst = p->pNext;
sqlite3OsFree(p);
assert( holdingMutex );
/* An IO error has occured. We cannot report the error back to the
** connection that requested the I/O since the error happened
** asynchronously. The connection has already moved on. There
** really is nobody to report the error to.
**
** The file for which the error occured may have been a database or
** journal file. Regardless, none of the currently queued operations
** associated with the same database should now be performed. Nor should
** any subsequently requested IO on either a database or journal file
** handle for the same database be accepted until the main database
** file handle has been closed and reopened.
**
** Furthermore, no further IO should be queued or performed on any file
** handle associated with a database that may have been part of a
** multi-file transaction that included the database associated with
** the IO error (i.e. a database ATTACHed to the same handle at some
** point in time).
*/
if( rc!=SQLITE_OK ){
async.ioError = rc;
}
/* Drop the queue mutex before continuing to the next write operation
** in order to give other threads a chance to work with the write queue.
*/
if( !async.pQueueFirst || !async.ioError ){
sqlite3ApiExit(0, 0);
pthread_mutex_unlock(&async.queueMutex);
holdingMutex = 0;
if( async.ioDelay>0 ){
sqlite3OsSleep(async.ioDelay);
}else{
sched_yield();
}
}
}
pthread_mutex_unlock(&async.writerMutex);
return 0;
}
/*
** Read data from the file. First we read from the filesystem, then adjust
** the contents of the buffer based on ASYNC_WRITE operations in the
** write-op queue.
**
** This method holds the mutex from start to finish.
*/
static int asyncRead(OsFile *id, void *obuf, int amt){
int rc = SQLITE_OK;
i64 filesize;
int nRead;
AsyncFile *pFile = (AsyncFile *)id;
OsFile *pBase = pFile->pBaseRead;
/* If an I/O error has previously occurred on this file, then all
** subsequent operations fail.
*/
if( async.ioError!=SQLITE_OK ){
return async.ioError;
}
/* Grab the write queue mutex for the duration of the call */
pthread_mutex_lock(&async.queueMutex);
if( pBase ){
rc = sqlite3OsFileSize(pBase, &filesize);
if( rc!=SQLITE_OK ){
goto asyncread_out;
}
rc = sqlite3OsSeek(pBase, pFile->iOffset);
if( rc!=SQLITE_OK ){
goto asyncread_out;
}
nRead = MIN(filesize - pFile->iOffset, amt);
if( nRead>0 ){
rc = sqlite3OsRead(pBase, obuf, nRead);
ASYNC_TRACE(("READ %s %d bytes at %d\n", pFile->zName, nRead, pFile->iOffset));
}
}
if( rc==SQLITE_OK ){
AsyncWrite *p;
i64 iOffset = pFile->iOffset; /* Current seek offset */
for(p=async.pQueueFirst; p; p = p->pNext){
if( p->pFile==pFile && p->op==ASYNC_WRITE ){
int iBeginOut = (p->iOffset - iOffset);
int iBeginIn = -iBeginOut;
int nCopy;
if( iBeginIn<0 ) iBeginIn = 0;
if( iBeginOut<0 ) iBeginOut = 0;
nCopy = MIN(p->nByte-iBeginIn, amt-iBeginOut);
if( nCopy>0 ){
memcpy(&((char *)obuf)[iBeginOut], &p->zBuf[iBeginIn], nCopy);
ASYNC_TRACE(("OVERREAD %d bytes at %d\n", nCopy, iBeginOut+iOffset));
}
}
}
pFile->iOffset += (i64)amt;
}
asyncread_out:
pthread_mutex_unlock(&async.queueMutex);
return rc;
}
