void TRI_CompactorVocBase (void* data) { TRI_vocbase_t* vocbase; TRI_vector_pointer_t collections; vocbase = data; assert(vocbase->_state == 1); TRI_InitVectorPointer(&collections, TRI_UNKNOWN_MEM_ZONE); while (true) { int state; // keep initial _state value as vocbase->_state might change during compaction loop state = vocbase->_state; // check if compaction is currently disallowed if (CheckAndLockCompaction(vocbase)) { // compaction is currently allowed size_t i, n; // copy all collections TRI_READ_LOCK_COLLECTIONS_VOCBASE(vocbase); TRI_CopyDataVectorPointer(&collections, &vocbase->_collections); TRI_READ_UNLOCK_COLLECTIONS_VOCBASE(vocbase); n = collections._length; for (i = 0; i < n; ++i) { TRI_vocbase_col_t* collection; TRI_primary_collection_t* primary; TRI_col_type_e type; bool doCompact; bool worked; collection = collections._buffer[i]; if (! TRI_TRY_READ_LOCK_STATUS_VOCBASE_COL(collection)) { // if we can't acquire the read lock instantly, we continue directly // we don't want to stall here for too long continue; } primary = collection->_collection; if (primary == NULL) { TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); continue; } worked = false; doCompact = primary->base._info._doCompact; type = primary->base._info._type; // for document collection, compactify datafiles if (TRI_IS_DOCUMENT_COLLECTION(type)) { if (collection->_status == TRI_VOC_COL_STATUS_LOADED && doCompact) { TRI_barrier_t* ce; // check whether someone else holds a read-lock on the compaction lock if (! TRI_TryWriteLockReadWriteLock(&primary->_compactionLock)) { // someone else is holding the compactor lock, we'll not compact TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); continue; } ce = TRI_CreateBarrierCompaction(&primary->_barrierList); if (ce == NULL) { // out of memory LOG_WARNING("out of memory when trying to create a barrier element"); } else { worked = CompactifyDocumentCollection((TRI_document_collection_t*) primary); TRI_FreeBarrier(ce); } // read-unlock the compaction lock TRI_WriteUnlockReadWriteLock(&primary->_compactionLock); } } TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); if (worked) { // signal the cleanup thread that we worked and that it can now wake up TRI_LockCondition(&vocbase->_cleanupCondition); TRI_SignalCondition(&vocbase->_cleanupCondition); TRI_UnlockCondition(&vocbase->_cleanupCondition); } } UnlockCompaction(vocbase); } if (state != 2 && vocbase->_state == 1) { // only sleep while server is still running TRI_LockCondition(&vocbase->_compactorCondition); TRI_TimedWaitCondition(&vocbase->_compactorCondition, (uint64_t) COMPACTOR_INTERVAL); TRI_UnlockCondition(&vocbase->_compactorCondition); } if (state == 2) { // server shutdown break; } } TRI_DestroyVectorPointer(&collections); LOG_TRACE("shutting down compactor thread"); }
void TRI_CompactorVocBase (void* data) { TRI_vocbase_t* vocbase = data; TRI_vector_pointer_t collections; assert(vocbase->_active); TRI_InitVectorPointer(&collections, TRI_UNKNOWN_MEM_ZONE); while (true) { size_t n; size_t i; TRI_col_type_e type; // keep initial _active value as vocbase->_active might change during compaction loop int active = vocbase->_active; if (active == 2) { // shadows must be cleaned before collections are handled // otherwise the shadows might still hold barriers on collections // and collections cannot be closed properly CleanupShadows(vocbase, true); } // copy all collections TRI_READ_LOCK_COLLECTIONS_VOCBASE(vocbase); TRI_CopyDataVectorPointer(&collections, &vocbase->_collections); TRI_READ_UNLOCK_COLLECTIONS_VOCBASE(vocbase); n = collections._length; for (i = 0; i < n; ++i) { TRI_vocbase_col_t* collection; TRI_doc_collection_t* doc; collection = collections._buffer[i]; if (! TRI_TRY_READ_LOCK_STATUS_VOCBASE_COL(collection)) { continue; } doc = collection->_collection; if (doc == NULL) { TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); continue; } type = doc->base._type; // for simple document collection, compactify datafiles if (type == TRI_COL_TYPE_SIMPLE_DOCUMENT) { if (collection->_status == TRI_VOC_COL_STATUS_LOADED) { CompactifySimCollection((TRI_sim_collection_t*) doc); } } TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); // now release the lock and maybe unload the collection or some datafiles if (type == TRI_COL_TYPE_SIMPLE_DOCUMENT) { CleanupSimCollection((TRI_sim_collection_t*) doc); } } if (vocbase->_active == 1) { // clean up unused shadows CleanupShadows(vocbase, false); // only sleep while server is still running usleep(COMPACTOR_INTERVAL); } if (active == 2) { // server shutdown break; } } TRI_DestroyVectorPointer(&collections); }
void TRI_CleanupVocBase (void* data) { TRI_vocbase_t* vocbase = data; TRI_vector_pointer_t collections; assert(vocbase->_state == 1); TRI_InitVectorPointer(&collections, TRI_UNKNOWN_MEM_ZONE); while (true) { size_t n; size_t i; TRI_col_type_e type; // keep initial _state value as vocbase->_state might change during compaction loop int state = vocbase->_state; if (state == 2) { // shadows must be cleaned before collections are handled // otherwise the shadows might still hold barriers on collections // and collections cannot be closed properly CleanupShadows(vocbase, true); } // copy all collections TRI_READ_LOCK_COLLECTIONS_VOCBASE(vocbase); TRI_CopyDataVectorPointer(&collections, &vocbase->_collections); TRI_READ_UNLOCK_COLLECTIONS_VOCBASE(vocbase); n = collections._length; for (i = 0; i < n; ++i) { TRI_vocbase_col_t* collection; TRI_primary_collection_t* primary; collection = collections._buffer[i]; TRI_READ_LOCK_STATUS_VOCBASE_COL(collection); primary = collection->_collection; if (primary == NULL) { TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); continue; } type = primary->base._type; TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); // now release the lock and maybe unload the collection or some datafiles if (TRI_IS_DOCUMENT_COLLECTION(type)) { CleanupDocumentCollection((TRI_document_collection_t*) primary); } } if (vocbase->_state >= 1) { // server is still running, clean up unused shadows CleanupShadows(vocbase, false); TRI_LockCondition(&vocbase->_cleanupCondition); TRI_TimedWaitCondition(&vocbase->_cleanupCondition, CLEANUP_INTERVAL); TRI_UnlockCondition(&vocbase->_cleanupCondition); } if (state == 3) { // server shutdown break; } } TRI_DestroyVectorPointer(&collections); }
void TRI_SynchroniserVocBase (void* data) { TRI_col_type_e type; TRI_vocbase_t* vocbase = data; TRI_vector_pointer_t collections; assert(vocbase->_state == 1); TRI_InitVectorPointer(&collections, TRI_UNKNOWN_MEM_ZONE); while (true) { size_t n; size_t i; bool worked; // keep initial _state value as vocbase->_state might change during sync loop int state = vocbase->_state; worked = false; // copy all collections and release the lock TRI_READ_LOCK_COLLECTIONS_VOCBASE(vocbase); TRI_CopyDataVectorPointer(&collections, &vocbase->_collections); TRI_READ_UNLOCK_COLLECTIONS_VOCBASE(vocbase); // loop over all copied collections n = collections._length; for (i = 0; i < n; ++i) { TRI_vocbase_col_t* collection; TRI_primary_collection_t* primary; collection = collections._buffer[i]; // if we cannot acquire the read lock instantly, we will continue. // otherwise we'll risk a multi-thread deadlock between synchroniser, // compactor and data-modification threads (e.g. POST /_api/document) if (! TRI_TRY_READ_LOCK_STATUS_VOCBASE_COL(collection)) { continue; } if (collection->_status != TRI_VOC_COL_STATUS_LOADED) { TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); continue; } primary = collection->_collection; // for simple collection, first sync and then seal type = primary->base._info._type; if (TRI_IS_DOCUMENT_COLLECTION(type)) { bool result; result = CheckSyncDocumentCollection((TRI_document_collection_t*) primary); worked |= result; result = CheckJournalDocumentCollection((TRI_document_collection_t*) primary); worked |= result; } TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); } // only sleep while server is still running and no-one is waiting if (! worked && vocbase->_state == 1) { TRI_LOCK_SYNCHRONISER_WAITER_VOCBASE(vocbase); if (vocbase->_syncWaiters == 0) { TRI_WAIT_SYNCHRONISER_WAITER_VOCBASE(vocbase, (uint64_t) SYNCHRONISER_INTERVAL); } TRI_UNLOCK_SYNCHRONISER_WAITER_VOCBASE(vocbase); } // server shutdown if (state == 2) { break; } } TRI_DestroyVectorPointer(&collections); LOG_TRACE("shutting down synchroniser thread"); }
void TRI_CleanupVocBase (void* data) { TRI_vocbase_t* vocbase; TRI_vector_pointer_t collections; uint64_t iterations = 0; vocbase = data; assert(vocbase); assert(vocbase->_state == 1); TRI_InitVectorPointer(&collections, TRI_UNKNOWN_MEM_ZONE); while (true) { int state; // keep initial _state value as vocbase->_state might change during cleanup loop state = vocbase->_state; ++iterations; if (state == 2) { // shadows must be cleaned before collections are handled // otherwise the shadows might still hold barriers on collections // and collections cannot be closed properly CleanupCursors(vocbase, true); } // check if we can get the compactor lock exclusively if (TRI_CheckAndLockCompactorVocBase(vocbase)) { size_t i, n; TRI_col_type_e type; // copy all collections TRI_READ_LOCK_COLLECTIONS_VOCBASE(vocbase); TRI_CopyDataVectorPointer(&collections, &vocbase->_collections); TRI_READ_UNLOCK_COLLECTIONS_VOCBASE(vocbase); n = collections._length; for (i = 0; i < n; ++i) { TRI_vocbase_col_t* collection; TRI_primary_collection_t* primary; collection = collections._buffer[i]; TRI_READ_LOCK_STATUS_VOCBASE_COL(collection); primary = collection->_collection; if (primary == NULL) { TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); continue; } type = primary->base._info._type; TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); // we're the only ones that can unload the collection, so using // the collection pointer outside the lock is ok // maybe cleanup indexes, unload the collection or some datafiles if (TRI_IS_DOCUMENT_COLLECTION(type)) { TRI_document_collection_t* document = (TRI_document_collection_t*) primary; // clean indexes? if (iterations % (uint64_t) CLEANUP_INDEX_ITERATIONS == 0) { document->cleanupIndexes(document); } CleanupDocumentCollection(document); } } TRI_UnlockCompactorVocBase(vocbase); } if (vocbase->_state >= 1) { // server is still running, clean up unused shadows if (iterations % CLEANUP_SHADOW_ITERATIONS == 0) { CleanupCursors(vocbase, false); } // clean up expired compactor locks TRI_CleanupCompactorVocBase(vocbase); if (state == 1) { TRI_LockCondition(&vocbase->_cleanupCondition); TRI_TimedWaitCondition(&vocbase->_cleanupCondition, (uint64_t) CLEANUP_INTERVAL); TRI_UnlockCondition(&vocbase->_cleanupCondition); } } if (state == 3) { // server shutdown break; } } TRI_DestroyVectorPointer(&collections); LOG_TRACE("shutting down cleanup thread"); }