int TRI_UpdateCollectionInfo (TRI_vocbase_t* vocbase,
TRI_collection_t* collection,
TRI_col_info_t const* parameter) {
if (TRI_IS_DOCUMENT_COLLECTION(collection->_info._type)) {
TRI_LOCK_JOURNAL_ENTRIES_DOC_COLLECTION((TRI_document_collection_t*) collection);
}
if (parameter != 0) {
collection->_info._maximalSize = parameter->_maximalSize;
collection->_info._waitForSync = parameter->_waitForSync;
if (collection->_info._maximalSize < collection->_maximumMarkerSize + TRI_JOURNAL_OVERHEAD) {
collection->_info._maximalSize = collection->_maximumMarkerSize + TRI_JOURNAL_OVERHEAD;
}
// the following collection properties are intentionally not updated as updating
// them would be very complicated:
// - _cid
// - _name
// - _type
// - _isSystem
// - _isVolatile
// ... probably a few others missing here ...
}
if (TRI_IS_DOCUMENT_COLLECTION(collection->_info._type)) {
TRI_document_collection_t* docCollection = (TRI_document_collection_t*) collection;
if (docCollection->base._shaper != NULL) {
TRI_shape_collection_t* shapeCollection = TRI_CollectionVocShaper(((TRI_document_collection_t*) collection)->base._shaper);
if (shapeCollection != NULL) {
// adjust wait for sync value of underlying shape collection
shapeCollection->base._info._waitForSync = (vocbase->_forceSyncShapes || collection->_info._waitForSync);
}
}
TRI_UNLOCK_JOURNAL_ENTRIES_DOC_COLLECTION((TRI_document_collection_t*) collection);
}
return TRI_SaveCollectionInfo(collection->_directory, &collection->_info);
}
char* TRI_GetDirectoryCollection (char const* path,
const TRI_col_info_t* const parameter) {
char* filename;
assert(path);
assert(parameter);
// shape collections use just the name, e.g. path/SHAPES
if (parameter->_type == TRI_COL_TYPE_SHAPE) {
filename = TRI_Concatenate2File(path, parameter->_name);
}
// other collections use the collection identifier
else if (TRI_IS_DOCUMENT_COLLECTION(parameter->_type)) {
char* tmp1;
char* tmp2;
tmp1 = TRI_StringUInt64(parameter->_cid);
if (tmp1 == NULL) {
TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY);
return NULL;
}
tmp2 = TRI_Concatenate2String("collection-", tmp1);
if (tmp2 == NULL) {
TRI_FreeString(TRI_CORE_MEM_ZONE, tmp1);
TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY);
return NULL;
}
filename = TRI_Concatenate2File(path, tmp2);
TRI_FreeString(TRI_CORE_MEM_ZONE, tmp1);
TRI_FreeString(TRI_CORE_MEM_ZONE, tmp2);
}
// oops, unknown collection type
else {
TRI_set_errno(TRI_ERROR_ARANGO_UNKNOWN_COLLECTION_TYPE);
return NULL;
}
if (filename == NULL) {
TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY);
}
// might be NULL
return filename;
}
void TRI_CompactorVocBase (void* data) {
TRI_vocbase_t* vocbase;
TRI_vector_pointer_t collections;
vocbase = data;
assert(vocbase->_state == 1);
TRI_InitVectorPointer(&collections, TRI_UNKNOWN_MEM_ZONE);
while (true) {
int state;
// keep initial _state value as vocbase->_state might change during compaction loop
state = vocbase->_state;
// check if compaction is currently disallowed
if (CheckAndLockCompaction(vocbase)) {
// compaction is currently allowed
size_t i, n;
// copy all collections
TRI_READ_LOCK_COLLECTIONS_VOCBASE(vocbase);
TRI_CopyDataVectorPointer(&collections, &vocbase->_collections);
TRI_READ_UNLOCK_COLLECTIONS_VOCBASE(vocbase);
n = collections._length;
for (i = 0; i < n; ++i) {
TRI_vocbase_col_t* collection;
TRI_primary_collection_t* primary;
TRI_col_type_e type;
bool doCompact;
bool worked;
collection = collections._buffer[i];
if (! TRI_TRY_READ_LOCK_STATUS_VOCBASE_COL(collection)) {
// if we can't acquire the read lock instantly, we continue directly
// we don't want to stall here for too long
continue;
}
primary = collection->_collection;
if (primary == NULL) {
TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection);
continue;
}
worked = false;
doCompact = primary->base._info._doCompact;
type = primary->base._info._type;
// for document collection, compactify datafiles
if (TRI_IS_DOCUMENT_COLLECTION(type)) {
if (collection->_status == TRI_VOC_COL_STATUS_LOADED && doCompact) {
TRI_barrier_t* ce;
// check whether someone else holds a read-lock on the compaction lock
if (! TRI_TryWriteLockReadWriteLock(&primary->_compactionLock)) {
// someone else is holding the compactor lock, we'll not compact
TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection);
continue;
}
ce = TRI_CreateBarrierCompaction(&primary->_barrierList);
if (ce == NULL) {
// out of memory
LOG_WARNING("out of memory when trying to create a barrier element");
}
else {
worked = CompactifyDocumentCollection((TRI_document_collection_t*) primary);
TRI_FreeBarrier(ce);
}
// read-unlock the compaction lock
TRI_WriteUnlockReadWriteLock(&primary->_compactionLock);
}
}
TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection);
if (worked) {
// signal the cleanup thread that we worked and that it can now wake up
TRI_LockCondition(&vocbase->_cleanupCondition);
TRI_SignalCondition(&vocbase->_cleanupCondition);
TRI_UnlockCondition(&vocbase->_cleanupCondition);
}
}
UnlockCompaction(vocbase);
}
if (state != 2 && vocbase->_state == 1) {
// only sleep while server is still running
TRI_LockCondition(&vocbase->_compactorCondition);
TRI_TimedWaitCondition(&vocbase->_compactorCondition, (uint64_t) COMPACTOR_INTERVAL);
TRI_UnlockCondition(&vocbase->_compactorCondition);
}
if (state == 2) {
// server shutdown
break;
}
}
TRI_DestroyVectorPointer(&collections);
LOG_TRACE("shutting down compactor thread");
}
void TRI_SynchroniserVocBase (void* data) {
TRI_col_type_e type;
TRI_vocbase_t* vocbase = data;
TRI_vector_pointer_t collections;
assert(vocbase->_state == 1);
TRI_InitVectorPointer(&collections, TRI_UNKNOWN_MEM_ZONE);
while (true) {
size_t n;
size_t i;
bool worked;
// keep initial _state value as vocbase->_state might change during sync loop
int state = vocbase->_state;
worked = false;
// copy all collections and release the lock
TRI_READ_LOCK_COLLECTIONS_VOCBASE(vocbase);
TRI_CopyDataVectorPointer(&collections, &vocbase->_collections);
TRI_READ_UNLOCK_COLLECTIONS_VOCBASE(vocbase);
// loop over all copied collections
n = collections._length;
for (i = 0; i < n; ++i) {
TRI_vocbase_col_t* collection;
TRI_primary_collection_t* primary;
collection = collections._buffer[i];
// if we cannot acquire the read lock instantly, we will continue.
// otherwise we'll risk a multi-thread deadlock between synchroniser,
// compactor and data-modification threads (e.g. POST /_api/document)
if (! TRI_TRY_READ_LOCK_STATUS_VOCBASE_COL(collection)) {
continue;
}
if (collection->_status != TRI_VOC_COL_STATUS_LOADED) {
TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection);
continue;
}
primary = collection->_collection;
// for simple collection, first sync and then seal
type = primary->base._info._type;
if (TRI_IS_DOCUMENT_COLLECTION(type)) {
bool result;
result = CheckSyncDocumentCollection((TRI_document_collection_t*) primary);
worked |= result;
result = CheckJournalDocumentCollection((TRI_document_collection_t*) primary);
worked |= result;
}
TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection);
}
// only sleep while server is still running and no-one is waiting
if (! worked && vocbase->_state == 1) {
TRI_LOCK_SYNCHRONISER_WAITER_VOCBASE(vocbase);
if (vocbase->_syncWaiters == 0) {
TRI_WAIT_SYNCHRONISER_WAITER_VOCBASE(vocbase, (uint64_t) SYNCHRONISER_INTERVAL);
}
TRI_UNLOCK_SYNCHRONISER_WAITER_VOCBASE(vocbase);
}
// server shutdown
if (state == 2) {
break;
}
}
TRI_DestroyVectorPointer(&collections);
LOG_TRACE("shutting down synchroniser thread");
}
void TRI_CleanupVocBase (void* data) {
TRI_vocbase_t* vocbase = data;
TRI_vector_pointer_t collections;
assert(vocbase->_state == 1);
TRI_InitVectorPointer(&collections, TRI_UNKNOWN_MEM_ZONE);
while (true) {
size_t n;
size_t i;
TRI_col_type_e type;
// keep initial _state value as vocbase->_state might change during compaction loop
int state = vocbase->_state;
if (state == 2) {
// shadows must be cleaned before collections are handled
// otherwise the shadows might still hold barriers on collections
// and collections cannot be closed properly
CleanupShadows(vocbase, true);
}
// copy all collections
TRI_READ_LOCK_COLLECTIONS_VOCBASE(vocbase);
TRI_CopyDataVectorPointer(&collections, &vocbase->_collections);
TRI_READ_UNLOCK_COLLECTIONS_VOCBASE(vocbase);
n = collections._length;
for (i = 0; i < n; ++i) {
TRI_vocbase_col_t* collection;
TRI_primary_collection_t* primary;
collection = collections._buffer[i];
TRI_READ_LOCK_STATUS_VOCBASE_COL(collection);
primary = collection->_collection;
if (primary == NULL) {
TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection);
continue;
}
type = primary->base._type;
TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection);
// now release the lock and maybe unload the collection or some datafiles
if (TRI_IS_DOCUMENT_COLLECTION(type)) {
CleanupDocumentCollection((TRI_document_collection_t*) primary);
}
}
if (vocbase->_state >= 1) {
// server is still running, clean up unused shadows
CleanupShadows(vocbase, false);
TRI_LockCondition(&vocbase->_cleanupCondition);
TRI_TimedWaitCondition(&vocbase->_cleanupCondition, CLEANUP_INTERVAL);
TRI_UnlockCondition(&vocbase->_cleanupCondition);
}
if (state == 3) {
// server shutdown
break;
}
}
TRI_DestroyVectorPointer(&collections);
}
bool TRI_LoadAuthInfo (TRI_vocbase_t* vocbase) {
TRI_vocbase_col_t* collection;
TRI_primary_collection_t* primary;
void** beg;
void** end;
void** ptr;
LOG_DEBUG("starting to load authentication and authorisation information");
collection = TRI_LookupCollectionByNameVocBase(vocbase, "_users");
if (collection == NULL) {
LOG_INFO("collection '_users' does not exist, no authentication available");
return false;
}
TRI_UseCollectionVocBase(vocbase, collection);
primary = collection->_collection;
if (primary == NULL) {
LOG_FATAL_AND_EXIT("collection '_users' cannot be loaded");
}
if (! TRI_IS_DOCUMENT_COLLECTION(primary->base._info._type)) {
TRI_ReleaseCollectionVocBase(vocbase, collection);
LOG_FATAL_AND_EXIT("collection '_users' has an unknown collection type");
}
TRI_WriteLockReadWriteLock(&vocbase->_authInfoLock);
// .............................................................................
// inside a write transaction
// .............................................................................
collection->_collection->beginWrite(collection->_collection);
beg = primary->_primaryIndex._table;
end = beg + primary->_primaryIndex._nrAlloc;
ptr = beg;
for (; ptr < end; ++ptr) {
if (*ptr) {
TRI_doc_mptr_t const* d;
TRI_shaped_json_t shapedJson;
d = (TRI_doc_mptr_t const*) *ptr;
if (d->_validTo == 0) {
TRI_vocbase_auth_t* auth;
TRI_EXTRACT_SHAPED_JSON_MARKER(shapedJson, d->_data);
auth = ConvertAuthInfo(vocbase, primary, &shapedJson);
if (auth != NULL) {
TRI_vocbase_auth_t* old;
old = TRI_InsertKeyAssociativePointer(&vocbase->_authInfo, auth->_username, auth, true);
if (old != NULL) {
FreeAuthInfo(old);
}
}
}
}
}
collection->_collection->endWrite(collection->_collection);
// .............................................................................
// outside a write transaction
// .............................................................................
vocbase->_authInfoFlush = true;
TRI_WriteUnlockReadWriteLock(&vocbase->_authInfoLock);
TRI_ReleaseCollectionVocBase(vocbase, collection);
return true;
}
void TRI_CleanupVocBase (void* data) {
TRI_vocbase_t* vocbase;
TRI_vector_pointer_t collections;
uint64_t iterations = 0;
vocbase = data;
assert(vocbase);
assert(vocbase->_state == 1);
TRI_InitVectorPointer(&collections, TRI_UNKNOWN_MEM_ZONE);
while (true) {
int state;
// keep initial _state value as vocbase->_state might change during cleanup loop
state = vocbase->_state;
++iterations;
if (state == 2) {
// shadows must be cleaned before collections are handled
// otherwise the shadows might still hold barriers on collections
// and collections cannot be closed properly
CleanupCursors(vocbase, true);
}
// check if we can get the compactor lock exclusively
if (TRI_CheckAndLockCompactorVocBase(vocbase)) {
size_t i, n;
TRI_col_type_e type;
// copy all collections
TRI_READ_LOCK_COLLECTIONS_VOCBASE(vocbase);
TRI_CopyDataVectorPointer(&collections, &vocbase->_collections);
TRI_READ_UNLOCK_COLLECTIONS_VOCBASE(vocbase);
n = collections._length;
for (i = 0; i < n; ++i) {
TRI_vocbase_col_t* collection;
TRI_primary_collection_t* primary;
collection = collections._buffer[i];
TRI_READ_LOCK_STATUS_VOCBASE_COL(collection);
primary = collection->_collection;
if (primary == NULL) {
TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection);
continue;
}
type = primary->base._info._type;
TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection);
// we're the only ones that can unload the collection, so using
// the collection pointer outside the lock is ok
// maybe cleanup indexes, unload the collection or some datafiles
if (TRI_IS_DOCUMENT_COLLECTION(type)) {
TRI_document_collection_t* document = (TRI_document_collection_t*) primary;
// clean indexes?
if (iterations % (uint64_t) CLEANUP_INDEX_ITERATIONS == 0) {
document->cleanupIndexes(document);
}
CleanupDocumentCollection(document);
}
}
TRI_UnlockCompactorVocBase(vocbase);
}
if (vocbase->_state >= 1) {
// server is still running, clean up unused shadows
if (iterations % CLEANUP_SHADOW_ITERATIONS == 0) {
CleanupCursors(vocbase, false);
}
// clean up expired compactor locks
TRI_CleanupCompactorVocBase(vocbase);
if (state == 1) {
TRI_LockCondition(&vocbase->_cleanupCondition);
TRI_TimedWaitCondition(&vocbase->_cleanupCondition, (uint64_t) CLEANUP_INTERVAL);
TRI_UnlockCondition(&vocbase->_cleanupCondition);
}
}
if (state == 3) {
// server shutdown
break;
}
}
TRI_DestroyVectorPointer(&collections);
LOG_TRACE("shutting down cleanup thread");
}
