int basic_async_logger<traits>::commit(const struct timespec* tsp) { ASYNC_TRACE(("Committing head: %p\n", m_head)); int l_old_val = m_event.value(); while (!m_head) { l_old_val = m_event.wait(tsp, &l_old_val); if (m_cancel && !m_head) return 0; } static const log_msg_type* l_new_head = NULL; const log_msg_type* l_cur_head; do { l_cur_head = const_cast<const log_msg_type*>( m_head ); } while( !atomic::cas(&m_head, l_cur_head, l_new_head) ); ASYNC_TRACE(( " --> cur head: %p, new head: %p\n", l_cur_head, m_head)); BOOST_ASSERT(l_cur_head); int count = 0; const log_msg_type* l_last = NULL; const log_msg_type* l_next = l_cur_head; // Reverse the section of this list for(const log_msg_type* p = l_cur_head; l_next; count++, p = l_next) { ASYNC_TRACE(("Set last[%p] -> next[%p]\n", l_next, l_last)); l_next = p->next(); p->next(l_last); l_last = p; } BOOST_ASSERT(l_last); ASYNC_TRACE(("Total (%d). Sublist's head: %p (%s)\n", count, l_last, l_last->c_str())); if (m_max_queue_size < count) m_max_queue_size = count; l_next = l_last; for(const log_msg_type* p = l_last; l_next; p = l_next) { if (::fwrite(p->c_str(), p->size(), 1, m_file) < 0) return -1; l_next = p->next(); p->next(NULL); ASYNC_TRACE(("Wrote (%d bytes): %p (next: %p): %s\n", p->size(), p, l_next, p->c_str())); delete p; } if (::fflush(m_file) < 0) return -2; return 0; }
/* ** Add an entry to the end of the global write-op list. pWrite should point ** to an AsyncWrite structure allocated using sqlite3OsMalloc(). The writer ** thread will call sqlite3OsFree() to free the structure after the specified ** operation has been completed. ** ** Once an AsyncWrite structure has been added to the list, it becomes the ** property of the writer thread and must not be read or modified by the ** caller. */ static void addAsyncWrite(AsyncWrite *pWrite){ /* We must hold the queue mutex in order to modify the queue pointers */ pthread_mutex_lock(&async.queueMutex); /* Add the record to the end of the write-op queue */ assert( !pWrite->pNext ); if( async.pQueueLast ){ assert( async.pQueueFirst ); async.pQueueLast->pNext = pWrite; }else{ async.pQueueFirst = pWrite; } async.pQueueLast = pWrite; ASYNC_TRACE(("PUSH %p (%s %s %d)\n", pWrite, azOpcodeName[pWrite->op], pWrite->pFile ? pWrite->pFile->zName : "-", pWrite->iOffset)); if( pWrite->op==ASYNC_CLOSE ){ async.nFile--; if( async.nFile==0 ){ async.ioError = SQLITE_OK; } } /* Drop the queue mutex */ pthread_mutex_unlock(&async.queueMutex); /* The writer thread might have been idle because there was nothing ** on the write-op queue for it to do. So wake it up. */ pthread_cond_signal(&async.queueSignal); }
/* ** No disk locking is performed. We keep track of locks locally in ** the async.aLock hash table. Locking should appear to work the same ** as with standard (unmodified) SQLite as long as all connections ** come from this one process. Connections from external processes ** cannot see our internal hash table (obviously) and will thus not ** honor our locks. */ static int asyncLock(OsFile *id, int lockType){ AsyncFile *pFile = (AsyncFile*)id; ASYNC_TRACE(("LOCK %d (%s)\n", lockType, pFile->zName)); pthread_mutex_lock(&async.lockMutex); sqlite3HashInsert(&async.aLock, pFile->zName, pFile->nName, (void*)lockType); pthread_mutex_unlock(&async.lockMutex); return SQLITE_OK; }
/* ** This function is called when the pager layer first opens a database file ** and is checking for a hot-journal. */ static int asyncCheckReservedLock(OsFile *id){ AsyncFile *pFile = (AsyncFile*)id; int rc; pthread_mutex_lock(&async.lockMutex); rc = (int)sqlite3HashFind(&async.aLock, pFile->zName, pFile->nName); pthread_mutex_unlock(&async.lockMutex); ASYNC_TRACE(("CHECK-LOCK %d (%s)\n", rc, pFile->zName)); return rc>SHARED_LOCK; }
void basic_async_logger<traits>::stop() { if (!m_file) return; m_cancel = true; ASYNC_TRACE(("Stopping async logger (head %p)\n", m_head)); m_event.signal(); if (m_thread) m_thread->join(); }
void basic_async_logger<traits>::run(boost::barrier* a_barrier) { // Wait until the caller is ready a_barrier->wait(); ASYNC_TRACE(("Started async logging thread (cancel=%s)\n", m_cancel ? "true" : "false")); static const timespec ts = {traits::commit_timeout / 1000, (traits::commit_timeout % 1000) * 1000000 }; while (1) { int n = commit(&ts); ASYNC_TRACE(( "Async thread result: %d (head: %p, cancel=%s)\n", n, m_head, m_cancel ? "true" : "false" )); if (n || (!m_head && m_cancel)) break; } ::fclose(m_file); m_file = NULL; }
int basic_async_logger<traits>::internal_write(const log_msg_type* msg) { if (!msg) return -9; const log_msg_type* l_last_head; do { l_last_head = const_cast<const log_msg_type*>(m_head); msg->next( l_last_head ); } while( !atomic::cas(&m_head, l_last_head, msg) ); if (!l_last_head) m_event.signal(); ASYNC_TRACE(("internal_write - cur head: %p, prev head: %p\n", m_head, l_last_head)); return 0; }
/* ** Implementation of sqlite3OsFileExists. Return true if file 'z' exists ** in the file system. ** ** This method holds the mutex from start to finish. */ static int asyncFileExists(const char *z){ int ret; AsyncWrite *p; pthread_mutex_lock(&async.queueMutex); /* See if the real file system contains the specified file. */ ret = xOrigFileExists(z); for(p=async.pQueueFirst; p; p = p->pNext){ if( p->op==ASYNC_DELETE && 0==strcmp(p->zBuf, z) ){ ret = 0; }else if( p->op==ASYNC_OPENEXCLUSIVE && 0==strcmp(p->zBuf, z) ){ ret = 1; } } ASYNC_TRACE(("EXISTS: %s = %d\n", z, ret)); pthread_mutex_unlock(&async.queueMutex); return ret; }
/* ** This procedure runs in a separate thread, reading messages off of the ** write queue and processing them one by one. ** ** If async.writerHaltNow is true, then this procedure exits ** after processing a single message. ** ** If async.writerHaltWhenIdle is true, then this procedure exits when ** the write queue is empty. ** ** If both of the above variables are false, this procedure runs ** indefinately, waiting for operations to be added to the write queue ** and processing them in the order in which they arrive. ** ** An artifical delay of async.ioDelay milliseconds is inserted before ** each write operation in order to simulate the effect of a slow disk. ** ** Only one instance of this procedure may be running at a time. */ static void *asyncWriterThread(void *NotUsed){ AsyncWrite *p = 0; int rc = SQLITE_OK; int holdingMutex = 0; if( pthread_mutex_trylock(&async.writerMutex) ){ return 0; } while( async.writerHaltNow==0 ){ OsFile *pBase = 0; if( !holdingMutex ){ pthread_mutex_lock(&async.queueMutex); } while( (p = async.pQueueFirst)==0 ){ pthread_cond_broadcast(&async.emptySignal); if( async.writerHaltWhenIdle ){ pthread_mutex_unlock(&async.queueMutex); break; }else{ ASYNC_TRACE(("IDLE\n")); pthread_cond_wait(&async.queueSignal, &async.queueMutex); ASYNC_TRACE(("WAKEUP\n")); } } if( p==0 ) break; holdingMutex = 1; /* Right now this thread is holding the mutex on the write-op queue. ** Variable 'p' points to the first entry in the write-op queue. In ** the general case, we hold on to the mutex for the entire body of ** the loop. ** ** However in the cases enumerated below, we relinquish the mutex, ** perform the IO, and then re-request the mutex before removing 'p' from ** the head of the write-op queue. The idea is to increase concurrency with ** sqlite threads. ** ** * An ASYNC_CLOSE operation. ** * An ASYNC_OPENEXCLUSIVE operation. For this one, we relinquish ** the mutex, call the underlying xOpenExclusive() function, then ** re-aquire the mutex before seting the AsyncFile.pBaseRead ** variable. ** * ASYNC_SYNC and ASYNC_WRITE operations, if ** SQLITE_ASYNC_TWO_FILEHANDLES was set at compile time and two ** file-handles are open for the particular file being "synced". */ if( async.ioError!=SQLITE_OK && p->op!=ASYNC_CLOSE ){ p->op = ASYNC_NOOP; } if( p->pFile ){ pBase = p->pFile->pBaseWrite; if( p->op==ASYNC_CLOSE || p->op==ASYNC_OPENEXCLUSIVE || (pBase && (p->op==ASYNC_SYNC || p->op==ASYNC_WRITE) ) ){ pthread_mutex_unlock(&async.queueMutex); holdingMutex = 0; } if( !pBase ){ pBase = p->pFile->pBaseRead; } } switch( p->op ){ case ASYNC_NOOP: break; case ASYNC_WRITE: assert( pBase ); ASYNC_TRACE(("WRITE %s %d bytes at %d\n", p->pFile->zName, p->nByte, p->iOffset)); rc = sqlite3OsSeek(pBase, p->iOffset); if( rc==SQLITE_OK ){ rc = sqlite3OsWrite(pBase, (const void *)(p->zBuf), p->nByte); } break; case ASYNC_SYNC: assert( pBase ); ASYNC_TRACE(("SYNC %s\n", p->pFile->zName)); rc = sqlite3OsSync(pBase, p->nByte); break; case ASYNC_TRUNCATE: assert( pBase ); ASYNC_TRACE(("TRUNCATE %s to %d bytes\n", p->pFile->zName, p->iOffset)); rc = sqlite3OsTruncate(pBase, p->iOffset); break; case ASYNC_CLOSE: ASYNC_TRACE(("CLOSE %s\n", p->pFile->zName)); sqlite3OsClose(&p->pFile->pBaseWrite); sqlite3OsClose(&p->pFile->pBaseRead); sqlite3OsFree(p->pFile); break; case ASYNC_OPENDIRECTORY: assert( pBase ); ASYNC_TRACE(("OPENDIR %s\n", p->zBuf)); sqlite3OsOpenDirectory(pBase, p->zBuf); break; case ASYNC_SETFULLSYNC: assert( pBase ); ASYNC_TRACE(("SETFULLSYNC %s %d\n", p->pFile->zName, p->nByte)); sqlite3OsSetFullSync(pBase, p->nByte); break; case ASYNC_DELETE: ASYNC_TRACE(("DELETE %s\n", p->zBuf)); rc = xOrigDelete(p->zBuf); break; case ASYNC_SYNCDIRECTORY: ASYNC_TRACE(("SYNCDIR %s\n", p->zBuf)); rc = xOrigSyncDirectory(p->zBuf); break; case ASYNC_OPENEXCLUSIVE: { AsyncFile *pFile = p->pFile; int delFlag = ((p->iOffset)?1:0); OsFile *pBase = 0; ASYNC_TRACE(("OPEN %s delFlag=%d\n", p->zBuf, delFlag)); assert(pFile->pBaseRead==0 && pFile->pBaseWrite==0); rc = xOrigOpenExclusive(p->zBuf, &pBase, delFlag); assert( holdingMutex==0 ); pthread_mutex_lock(&async.queueMutex); holdingMutex = 1; if( rc==SQLITE_OK ){ pFile->pBaseRead = pBase; } break; } default: assert(!"Illegal value for AsyncWrite.op"); } /* If we didn't hang on to the mutex during the IO op, obtain it now ** so that the AsyncWrite structure can be safely removed from the ** global write-op queue. */ if( !holdingMutex ){ pthread_mutex_lock(&async.queueMutex); holdingMutex = 1; } /* ASYNC_TRACE(("UNLINK %p\n", p)); */ if( p==async.pQueueLast ){ async.pQueueLast = 0; } async.pQueueFirst = p->pNext; sqlite3OsFree(p); assert( holdingMutex ); /* An IO error has occured. We cannot report the error back to the ** connection that requested the I/O since the error happened ** asynchronously. The connection has already moved on. There ** really is nobody to report the error to. ** ** The file for which the error occured may have been a database or ** journal file. Regardless, none of the currently queued operations ** associated with the same database should now be performed. Nor should ** any subsequently requested IO on either a database or journal file ** handle for the same database be accepted until the main database ** file handle has been closed and reopened. ** ** Furthermore, no further IO should be queued or performed on any file ** handle associated with a database that may have been part of a ** multi-file transaction that included the database associated with ** the IO error (i.e. a database ATTACHed to the same handle at some ** point in time). */ if( rc!=SQLITE_OK ){ async.ioError = rc; } /* Drop the queue mutex before continuing to the next write operation ** in order to give other threads a chance to work with the write queue. */ if( !async.pQueueFirst || !async.ioError ){ sqlite3ApiExit(0, 0); pthread_mutex_unlock(&async.queueMutex); holdingMutex = 0; if( async.ioDelay>0 ){ sqlite3OsSleep(async.ioDelay); }else{ sched_yield(); } } } pthread_mutex_unlock(&async.writerMutex); return 0; }
/* ** Read data from the file. First we read from the filesystem, then adjust ** the contents of the buffer based on ASYNC_WRITE operations in the ** write-op queue. ** ** This method holds the mutex from start to finish. */ static int asyncRead(OsFile *id, void *obuf, int amt){ int rc = SQLITE_OK; i64 filesize; int nRead; AsyncFile *pFile = (AsyncFile *)id; OsFile *pBase = pFile->pBaseRead; /* If an I/O error has previously occurred on this file, then all ** subsequent operations fail. */ if( async.ioError!=SQLITE_OK ){ return async.ioError; } /* Grab the write queue mutex for the duration of the call */ pthread_mutex_lock(&async.queueMutex); if( pBase ){ rc = sqlite3OsFileSize(pBase, &filesize); if( rc!=SQLITE_OK ){ goto asyncread_out; } rc = sqlite3OsSeek(pBase, pFile->iOffset); if( rc!=SQLITE_OK ){ goto asyncread_out; } nRead = MIN(filesize - pFile->iOffset, amt); if( nRead>0 ){ rc = sqlite3OsRead(pBase, obuf, nRead); ASYNC_TRACE(("READ %s %d bytes at %d\n", pFile->zName, nRead, pFile->iOffset)); } } if( rc==SQLITE_OK ){ AsyncWrite *p; i64 iOffset = pFile->iOffset; /* Current seek offset */ for(p=async.pQueueFirst; p; p = p->pNext){ if( p->pFile==pFile && p->op==ASYNC_WRITE ){ int iBeginOut = (p->iOffset - iOffset); int iBeginIn = -iBeginOut; int nCopy; if( iBeginIn<0 ) iBeginIn = 0; if( iBeginOut<0 ) iBeginOut = 0; nCopy = MIN(p->nByte-iBeginIn, amt-iBeginOut); if( nCopy>0 ){ memcpy(&((char *)obuf)[iBeginOut], &p->zBuf[iBeginIn], nCopy); ASYNC_TRACE(("OVERREAD %d bytes at %d\n", nCopy, iBeginOut+iOffset)); } } } pFile->iOffset += (i64)amt; } asyncread_out: pthread_mutex_unlock(&async.queueMutex); return rc; }