TRI_aql_node_t* TRI_CreateNodeReturnEmptyAql (void) { TRI_aql_node_t* node; TRI_aql_node_t* list; node = (TRI_aql_node_t*) TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_aql_node_t), false); if (node == NULL) { return NULL; } node->_type = TRI_AQL_NODE_RETURN_EMPTY; list = (TRI_aql_node_t*) TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_aql_node_t), false); if (list == NULL) { TRI_Free(TRI_UNKNOWN_MEM_ZONE, list); return NULL; } list->_type = TRI_AQL_NODE_LIST; TRI_InitVectorPointer(&list->_members, TRI_UNKNOWN_MEM_ZONE); TRI_InitVectorPointer(&node->_members, TRI_UNKNOWN_MEM_ZONE); TRI_PushBackVectorPointer(&node->_members, (void*) list); return node; }
bool TRI_InitScopesAql (TRI_aql_context_t* const context) { assert(context); TRI_InitVectorPointer(&context->_memory._scopes, TRI_UNKNOWN_MEM_ZONE); TRI_InitVectorPointer(&context->_currentScopes, TRI_UNKNOWN_MEM_ZONE); return true; }
TRI_shaper_t* TRI_CreateArrayShaper (TRI_memory_zone_t* zone) { array_shaper_t* shaper; bool ok; // create the shaper shaper = TRI_Allocate(zone, sizeof(array_shaper_t), false); if (shaper == NULL) { return NULL; } TRI_InitShaper(&shaper->base, zone); // create the attribute dictionary TRI_InitAssociativePointer(&shaper->_attributeNames, zone, HashKeyAttributeName, HashElementAttributeName, EqualKeyAttributeName, 0); // create the attributes vector TRI_InitVectorPointer(&shaper->_attributes, zone); // create the shape dictionary TRI_InitAssociativePointer(&shaper->_shapeDictionary, zone, 0, HashElementShape, 0, EqualElementShape); // create the shapes vector TRI_InitVectorPointer(&shaper->_shapes, zone); // set the find and lookup functions shaper->base.findAttributeName = FindAttributeNameArrayShaper; shaper->base.lookupAttributeId = LookupAttributeIdArrayShaper; shaper->base.findShape = FindShapeShape; shaper->base.lookupShapeId = LookupShapeId; shaper->base.lookupAttributeWeight = LookupAttributeWeight; // handle basics ok = TRI_InsertBasicTypesShaper(&shaper->base); if (! ok || zone->_failed) { TRI_FreeArrayShaper(zone, &shaper->base); return NULL; } // and return return &shaper->base; }
TRI_vector_pointer_t TRI_LookupByKeyMultiPointer (TRI_memory_zone_t* zone, TRI_multi_pointer_t* array, void const* key) { TRI_vector_pointer_t result; uint64_t hash; uint64_t i; // initialises the result vector TRI_InitVectorPointer(&result, zone); // compute the hash hash = array->hashKey(array, key); i = hash % array->_nrAlloc; #ifdef TRI_INTERNAL_STATS // update statistics array->_nrFinds++; #endif // search the table while (array->_table[i] != NULL) { if (array->isEqualKeyElement(array, key, array->_table[i])) { TRI_PushBackVectorPointer(&result, array->_table[i]); } #ifdef TRI_INTERNAL_STATS else { array->_nrProbesF++; } #endif i = (i + 1) % array->_nrAlloc; } // return whatever we found return result; }
static inline void InitNode (TRI_aql_context_t* const context, TRI_aql_node_t* const node, const TRI_aql_node_type_e type) { node->_type = type; TRI_InitVectorPointer(&node->_members, TRI_UNKNOWN_MEM_ZONE); TRI_RegisterNodeContextAql(context, node); }
TRI_aql_parser_t* TRI_CreateParserAql (const char* const query) { TRI_aql_parser_t* parser; assert(query); parser = (TRI_aql_parser_t*) TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_aql_parser_t), false); if (!parser) { return NULL; } TRI_InitVectorPointer(&parser->_scopes, TRI_UNKNOWN_MEM_ZONE); TRI_InitVectorPointer(&parser->_stack, TRI_UNKNOWN_MEM_ZONE); parser->_buffer = (char*) query; parser->_length = strlen(query); return parser; }
TRI_headers_t::TRI_headers_t () : _freelist(nullptr), _begin(nullptr), _end(nullptr), _nrAllocated(0), _nrLinked(0), _totalSize(0) { TRI_InitVectorPointer(&_blocks, TRI_UNKNOWN_MEM_ZONE, 16); }
TRI_vector_pointer_t TRI_LookupByElementHashArrayMulti (TRI_hash_array_t* array, TRI_hash_index_element_t* element) { TRI_vector_pointer_t result; uint64_t hash; uint64_t i; // ........................................................................... // initialise the vector which will hold the result if any // ........................................................................... TRI_InitVectorPointer(&result, TRI_UNKNOWN_MEM_ZONE); // ........................................................................... // compute the hash // ........................................................................... hash = HashElement(array, element); i = hash % array->_nrAlloc; // ........................................................................... // update statistics // ........................................................................... #ifdef TRI_INTERNAL_STATS array->_nrFinds++; #endif // ........................................................................... // search the table // ........................................................................... while (! IsEmptyElement(array, &array->_table[i])) { if (IsEqualElementElement(array, element, &array->_table[i])) { TRI_PushBackVectorPointer(&result, &array->_table[i]); } #ifdef TRI_INTERNAL_STATS else { array->_nrProbesF++; } #endif i = (i + 1) % array->_nrAlloc; } // ........................................................................... // return whatever we found -- which could be an empty vector list if nothing // matches. Note that we allow multiple elements (compare with pointer impl). // ........................................................................... return result; }
TRI_aql_node_t* TRI_CreateNodeNopAql (void) { TRI_aql_node_t* node = (TRI_aql_node_t*) TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_aql_node_t), false); if (node == NULL) { return NULL; } node->_type = TRI_AQL_NODE_NOP; TRI_InitVectorPointer(&node->_members, TRI_UNKNOWN_MEM_ZONE); return node; }
void TRI_InitializeProcess (int argc, char* argv[]) { TRI_PhysicalMemory = GetPhysicalMemory(); if (ProcessName != nullptr) { return; } ProcessName = TRI_DuplicateString(argv[0]); ARGC = argc; ARGV = argv; TRI_InitVectorPointer(&ExternalProcesses, TRI_CORE_MEM_ZONE); TRI_InitMutex(&ExternalProcessesLock); }
TRI_aql_statement_list_t* TRI_CreateStatementListAql (void) { TRI_aql_statement_list_t* list; list = (TRI_aql_statement_list_t*) TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_aql_statement_list_t), false); if (list == NULL) { return NULL; } TRI_InitVectorPointer(&list->_statements, TRI_UNKNOWN_MEM_ZONE); list->_currentLevel = 0; return list; }
static TRI_aql_scope_t* CreateScope (TRI_aql_context_t* const context, const TRI_aql_scope_e type) { TRI_aql_scope_t* scope; int res; assert(context); scope = (TRI_aql_scope_t*) TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_aql_scope_t), false); if (scope == NULL) { return NULL; } scope->_type = NextType(context, type); scope->_ranges = NULL; scope->_selfContained = true; scope->_empty = false; scope->_level = 0; // init with a dummy value scope->_limit._offset = 0; scope->_limit._limit = INT64_MAX; scope->_limit._status = TRI_AQL_LIMIT_UNDEFINED; scope->_limit._hasFilter = false; scope->_limit._found = 0; if (context->_fullCount) { // if option "fullCount" is specified, we must ignore all limit optimisations scope->_limit._status = TRI_AQL_LIMIT_IGNORE; } TRI_InitVectorPointer(&scope->_sorts, TRI_UNKNOWN_MEM_ZONE); res = TRI_InitAssociativePointer(&scope->_variables, TRI_UNKNOWN_MEM_ZONE, &TRI_HashStringKeyAssociativePointer, &TRI_HashVariableAql, &TRI_EqualVariableAql, 0); if (res != TRI_ERROR_NO_ERROR) { TRI_DestroyVectorPointer(&scope->_sorts); TRI_Free(TRI_UNKNOWN_MEM_ZONE, scope); return NULL; } return scope; }
int TRI_InitVectorPointer2 (TRI_vector_pointer_t* vector, TRI_memory_zone_t* zone, size_t initialCapacity) { TRI_InitVectorPointer(vector, zone); if (initialCapacity != 0) { vector->_buffer = (void*) TRI_Allocate(vector->_memoryZone, (initialCapacity * sizeof(void*)), true); if (vector->_buffer == NULL) { return TRI_ERROR_OUT_OF_MEMORY; } } vector->_capacity = initialCapacity; return TRI_ERROR_NO_ERROR; }
static void InitCollection (TRI_vocbase_t* vocbase, TRI_collection_t* collection, char* directory, const TRI_col_info_t* const info) { assert(collection); memset(collection, 0, sizeof(TRI_collection_t)); TRI_CopyCollectionInfo(&collection->_info, info); collection->_vocbase = vocbase; collection->_state = TRI_COL_STATE_WRITE; collection->_lastError = 0; collection->_maximumMarkerSize = 0; collection->_directory = directory; TRI_InitVectorPointer(&collection->_datafiles, TRI_UNKNOWN_MEM_ZONE); TRI_InitVectorPointer(&collection->_journals, TRI_UNKNOWN_MEM_ZONE); TRI_InitVectorPointer(&collection->_compactors, TRI_UNKNOWN_MEM_ZONE); TRI_InitVectorString(&collection->_indexFiles, TRI_CORE_MEM_ZONE); }
TRI_headers_t* TRI_CreateSimpleHeaders (size_t headerSize) { simple_headers_t* headers = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(simple_headers_t), false); if (headers == NULL) { TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY); return NULL; } headers->base.request = RequestSimpleHeaders; headers->base.verify = VerifySimpleHeaders; headers->base.release = ReleaseSimpleHeaders; headers->_freelist = NULL; headers->_headerSize = headerSize; TRI_InitVectorPointer(&headers->_blocks, TRI_UNKNOWN_MEM_ZONE); return &headers->base; }
TRI_vector_pointer_t TRI_LookupByElementMultiArray (TRI_memory_zone_t* zone, TRI_multi_array_t* array, void* element) { TRI_vector_pointer_t result; uint64_t hash; uint64_t i; // initialise the vector which will hold the result if any TRI_InitVectorPointer(&result, zone); // compute the hash hash = array->hashElement(array, element); i = hash % array->_nrAlloc; #ifdef TRI_INTERNAL_STATS // update statistics array->_nrFinds++; #endif // search the table while (! array->isEmptyElement(array, array->_table + i * array->_elementSize)) { if (array->isEqualElementElement(array, element, array->_table + i * array->_elementSize)) { TRI_PushBackVectorPointer(&result, array->_table + i * array->_elementSize); } #ifdef TRI_INTERNAL_STATS else { array->_nrProbesF++; } #endif i = (i + 1) % array->_nrAlloc; } // ........................................................................... // return whatever we found -- which could be an empty vector list if nothing // matches. Note that we allow multiple elements (compare with pointer // impl). If an out-of-memory occurred than the zone will have a suitable // marker. // ........................................................................... return result; }
TRI_vector_pointer_t TRI_LookupEdgesDocumentCollection (TRI_document_collection_t* document, TRI_edge_direction_e direction, TRI_voc_cid_t cid, TRI_voc_key_t key) { TRI_vector_pointer_t result; TRI_edge_header_t entry; TRI_multi_pointer_t* edgesIndex; // search criteria entry._mptr = NULL; entry._cid = cid; entry._searchKey._key = key; // initialise the result vector TRI_InitVectorPointer(&result, TRI_UNKNOWN_MEM_ZONE); edgesIndex = FindEdgesIndex(document); if (edgesIndex == NULL) { LOG_ERROR("collection does not have an edges index"); return result; } if (direction == TRI_EDGE_IN) { // get all edges with a matching IN vertex FindEdges(TRI_EDGE_IN, edgesIndex, &result, &entry, 1); } else if (direction == TRI_EDGE_OUT) { // get all edges with a matching OUT vertex FindEdges(TRI_EDGE_OUT, edgesIndex, &result, &entry, 1); } else if (direction == TRI_EDGE_ANY) { // get all edges with a matching IN vertex FindEdges(TRI_EDGE_IN, edgesIndex, &result, &entry, 1); // add all non-reflexive edges with a matching OUT vertex FindEdges(TRI_EDGE_OUT, edgesIndex, &result, &entry, 3); } return result; }
TRI_aql_statement_walker_t* TRI_CreateStatementWalkerAql (void* data, const bool canModify, TRI_aql_visit_f visitMember, TRI_aql_visit_f preVisitStatement, TRI_aql_visit_f postVisitStatement) { TRI_aql_statement_walker_t* walker; walker = (TRI_aql_statement_walker_t*) TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_aql_statement_walker_t), false); if (walker == NULL) { return NULL; } walker->_data = data; walker->_canModify = canModify; walker->visitMember = visitMember; walker->preVisitStatement = preVisitStatement; walker->postVisitStatement = postVisitStatement; TRI_InitVectorPointer(&walker->_currentScopes, TRI_UNKNOWN_MEM_ZONE); return walker; }
TRI_vector_pointer_t TRI_LookupByKeyHashArrayMulti (TRI_hash_array_multi_t const* array, TRI_index_search_value_t const* key) { TRI_ASSERT_EXPENSIVE(array->_nrUsed < array->_nrAlloc); // ........................................................................... // initialise the vector which will hold the result if any // ........................................................................... TRI_vector_pointer_t result; TRI_InitVectorPointer(&result, TRI_UNKNOWN_MEM_ZONE); uint64_t const n = array->_nrAlloc; uint64_t i, k; i = k = HashKey(array, key) % n; for (; i < n && array->_table[i]._document != nullptr && ! IsEqualKeyElement(array, key, &array->_table[i]); ++i); if (i == n) { for (i = 0; i < k && array->_table[i]._document != nullptr && ! IsEqualKeyElement(array, key, &array->_table[i]); ++i); } TRI_ASSERT_EXPENSIVE(i < n); if (array->_table[i]._document != nullptr) { // add the element itself TRI_PushBackVectorPointer(&result, array->_table[i]._document); // add the overflow elements auto current = array->_table[i]._next; while (current != nullptr) { TRI_PushBackVectorPointer(&result, current->_document); current = current->_next; } } return result; }
void TRI_SynchroniserVocBase (void* data) { TRI_col_type_e type; TRI_vocbase_t* vocbase = data; TRI_vector_pointer_t collections; assert(vocbase->_state == 1); TRI_InitVectorPointer(&collections, TRI_UNKNOWN_MEM_ZONE); while (true) { size_t n; size_t i; bool worked; // keep initial _state value as vocbase->_state might change during sync loop int state = vocbase->_state; worked = false; // copy all collections and release the lock TRI_READ_LOCK_COLLECTIONS_VOCBASE(vocbase); TRI_CopyDataVectorPointer(&collections, &vocbase->_collections); TRI_READ_UNLOCK_COLLECTIONS_VOCBASE(vocbase); // loop over all copied collections n = collections._length; for (i = 0; i < n; ++i) { TRI_vocbase_col_t* collection; TRI_primary_collection_t* primary; collection = collections._buffer[i]; // if we cannot acquire the read lock instantly, we will continue. // otherwise we'll risk a multi-thread deadlock between synchroniser, // compactor and data-modification threads (e.g. POST /_api/document) if (! TRI_TRY_READ_LOCK_STATUS_VOCBASE_COL(collection)) { continue; } if (collection->_status != TRI_VOC_COL_STATUS_LOADED) { TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); continue; } primary = collection->_collection; // for simple collection, first sync and then seal type = primary->base._info._type; if (TRI_IS_DOCUMENT_COLLECTION(type)) { bool result; result = CheckSyncDocumentCollection((TRI_document_collection_t*) primary); worked |= result; result = CheckJournalDocumentCollection((TRI_document_collection_t*) primary); worked |= result; } TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); } // only sleep while server is still running and no-one is waiting if (! worked && vocbase->_state == 1) { TRI_LOCK_SYNCHRONISER_WAITER_VOCBASE(vocbase); if (vocbase->_syncWaiters == 0) { TRI_WAIT_SYNCHRONISER_WAITER_VOCBASE(vocbase, (uint64_t) SYNCHRONISER_INTERVAL); } TRI_UNLOCK_SYNCHRONISER_WAITER_VOCBASE(vocbase); } // server shutdown if (state == 2) { break; } } TRI_DestroyVectorPointer(&collections); LOG_TRACE("shutting down synchroniser thread"); }
TRI_aql_context_t* TRI_CreateContextAql (TRI_vocbase_t* vocbase, const char* const query) { TRI_aql_context_t* context; TRI_ASSERT_MAINTAINER(vocbase != NULL); TRI_ASSERT_MAINTAINER(query != NULL); LOG_TRACE("creating context"); context = (TRI_aql_context_t*) TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_aql_context_t), false); if (context == NULL) { return NULL; } context->_vocbase = vocbase; context->_variableIndex = 0; context->_scopeIndex = 0; // actual bind parameter values TRI_InitAssociativePointer(&context->_parameters._values, TRI_UNKNOWN_MEM_ZONE, &TRI_HashStringKeyAssociativePointer, &TRI_HashBindParameterAql, &TRI_EqualBindParameterAql, 0); // bind parameter names used in the query TRI_InitAssociativePointer(&context->_parameters._names, TRI_UNKNOWN_MEM_ZONE, &TRI_HashStringKeyAssociativePointer, &TRI_HashStringKeyAssociativePointer, &TRI_EqualStringKeyAssociativePointer, 0); // collections TRI_InitAssociativePointer(&context->_collectionNames, TRI_UNKNOWN_MEM_ZONE, &TRI_HashStringKeyAssociativePointer, &TRI_HashStringKeyAssociativePointer, &TRI_EqualStringKeyAssociativePointer, 0); TRI_InitVectorPointer2(&context->_memory._nodes, TRI_UNKNOWN_MEM_ZONE, 16); TRI_InitVectorPointer2(&context->_memory._strings, TRI_UNKNOWN_MEM_ZONE, 16); TRI_InitVectorPointer(&context->_collections, TRI_UNKNOWN_MEM_ZONE); TRI_InitErrorAql(&context->_error); context->_parser = NULL; context->_statements = NULL; context->_query = query; context->_parser = TRI_CreateParserAql(context->_query); if (context->_parser == NULL) { // could not create the parser TRI_FreeContextAql(context); return NULL; } if (! TRI_InitParserAql(context)) { // could not initialise the lexer TRI_FreeContextAql(context); return NULL; } context->_statements = TRI_CreateStatementListAql(); if (context->_statements == NULL) { // could not create statement list TRI_FreeContextAql(context); return NULL; } TRI_InitScopesAql(context); return context; }
static bool BytecodeShapeAccessor (TRI_shaper_t* shaper, TRI_shape_access_t* accessor) { union { void const* c; void* v; } cv; TRI_shape_aid_t const* paids; TRI_shape_path_t const* path; TRI_shape_t const* shape; TRI_vector_pointer_t ops; size_t i; size_t j; // find the shape shape = shaper->lookupShapeId(shaper, accessor->_sid); if (shape == NULL) { LOG_ERROR("unknown shape id '%ld'", (unsigned long) accessor->_sid); return false; } // find the attribute path path = shaper->lookupAttributePathByPid(shaper, accessor->_pid); if (path == NULL) { LOG_ERROR("unknown attribute path '%ld'", (unsigned long) accessor->_pid); return false; } paids = (TRI_shape_aid_t*) (((char const*) path) + sizeof(TRI_shape_path_t)); // collect the bytecode TRI_InitVectorPointer(&ops); // start with the shape TRI_PushBackVectorPointer(&ops, (void*) TRI_SHAPE_AC_SHAPE_PTR); cv.c = shape; TRI_PushBackVectorPointer(&ops, cv.v); // and follow it for (i = 0; i < path->_aidLength; ++i, ++paids) { #ifdef DEBUG_SHAPE_ACCESSOR printf("%lu: aid: %lu, sid: %lu, type %lu\n", (unsigned long) i, (unsigned long) *paids, (unsigned long) shape->_sid, (unsigned long) shape->_type); #endif if (shape->_type == TRI_SHAPE_ARRAY) { TRI_array_shape_t* s; TRI_shape_aid_t const* aids; TRI_shape_sid_t const* sids; TRI_shape_sid_t sid; TRI_shape_size_t const* offsetsF; TRI_shape_size_t f; TRI_shape_size_t n; TRI_shape_size_t v; char const* qtr; s = (TRI_array_shape_t*) shape; f = s->_fixedEntries; v = s->_variableEntries; n = f + v; // find the aid within the shape qtr = (char const*) shape; qtr += sizeof(TRI_array_shape_t); sids = (TRI_shape_sid_t const*) qtr; qtr += n * sizeof(TRI_shape_sid_t); aids = (TRI_shape_aid_t const*) qtr; qtr += n * sizeof(TRI_shape_aid_t); offsetsF = (TRI_shape_size_t const*) qtr; // check for fixed size aid for (j = 0; j < f; ++j, ++sids, ++aids, ++offsetsF) { if (*paids == *aids) { sid = *sids; LOG_TRACE("found aid '%ld' as fixed entry with sid '%ld' and offset '%ld' - '%ld'", (unsigned long) *paids, (unsigned long) sid, (unsigned long) offsetsF[0], (unsigned long) offsetsF[1]); shape = shaper->lookupShapeId(shaper, sid); if (shape == NULL) { LOG_ERROR("unknown shape id '%ld' for attribute id '%ld'", (unsigned long) accessor->_sid, (unsigned long) *paids); TRI_DestroyVectorPointer(&ops); return false; } TRI_PushBackVectorPointer(&ops, (void*) TRI_SHAPE_AC_OFFSET_FIX); TRI_PushBackVectorPointer(&ops, (void*) (intptr_t) (offsetsF[0])); // offset is always smaller than 4 GByte TRI_PushBackVectorPointer(&ops, (void*) (intptr_t) (offsetsF[1])); // offset is always smaller than 4 GByte TRI_PushBackVectorPointer(&ops, (void*) TRI_SHAPE_AC_SHAPE_PTR); cv.c = shape; TRI_PushBackVectorPointer(&ops, cv.v); break; } } if (j < f) { continue; } // check for variable size aid for (j = 0; j < v; ++j, ++sids, ++aids) { if (*paids == *aids) { sid = *sids; LOG_TRACE("found aid '%ld' as variable entry with sid '%ld'", (unsigned long) *paids, (unsigned long) sid); shape = shaper->lookupShapeId(shaper, sid); if (shape == NULL) { LOG_ERROR("unknown shape id '%ld' for attribute id '%ld'", (unsigned long) accessor->_sid, (unsigned long) *paids); TRI_DestroyVectorPointer(&ops); return false; } TRI_PushBackVectorPointer(&ops, (void*) TRI_SHAPE_AC_OFFSET_VAR); TRI_PushBackVectorPointer(&ops, (void*) j); TRI_PushBackVectorPointer(&ops, (void*) TRI_SHAPE_AC_SHAPE_PTR); cv.c = shape; TRI_PushBackVectorPointer(&ops, cv.v); break; } } if (j < v) { continue; } LOG_TRACE("unknown attribute id '%ld'", (unsigned long) *paids); TRI_DestroyVectorPointer(&ops); accessor->_shape = NULL; accessor->_code = NULL; return true; } else { TRI_DestroyVectorPointer(&ops); accessor->_shape = NULL; accessor->_code = NULL; return true; } } // travel attribute path to the end TRI_PushBackVectorPointer(&ops, (void*) TRI_SHAPE_AC_DONE); accessor->_shape = shape; cv.c = accessor->_code = TRI_Allocate(ops._length * sizeof(void*)); memcpy(cv.v, ops._buffer, ops._length * sizeof(void*)); TRI_DestroyVectorPointer(&ops); return true; }
void TRI_CompactorVocBase (void* data) { TRI_vocbase_t* vocbase = data; TRI_vector_pointer_t collections; assert(vocbase->_active); TRI_InitVectorPointer(&collections, TRI_UNKNOWN_MEM_ZONE); while (true) { size_t n; size_t i; TRI_col_type_e type; // keep initial _active value as vocbase->_active might change during compaction loop int active = vocbase->_active; if (active == 2) { // shadows must be cleaned before collections are handled // otherwise the shadows might still hold barriers on collections // and collections cannot be closed properly CleanupShadows(vocbase, true); } // copy all collections TRI_READ_LOCK_COLLECTIONS_VOCBASE(vocbase); TRI_CopyDataVectorPointer(&collections, &vocbase->_collections); TRI_READ_UNLOCK_COLLECTIONS_VOCBASE(vocbase); n = collections._length; for (i = 0; i < n; ++i) { TRI_vocbase_col_t* collection; TRI_doc_collection_t* doc; collection = collections._buffer[i]; if (! TRI_TRY_READ_LOCK_STATUS_VOCBASE_COL(collection)) { continue; } doc = collection->_collection; if (doc == NULL) { TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); continue; } type = doc->base._type; // for simple document collection, compactify datafiles if (type == TRI_COL_TYPE_SIMPLE_DOCUMENT) { if (collection->_status == TRI_VOC_COL_STATUS_LOADED) { CompactifySimCollection((TRI_sim_collection_t*) doc); } } TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); // now release the lock and maybe unload the collection or some datafiles if (type == TRI_COL_TYPE_SIMPLE_DOCUMENT) { CleanupSimCollection((TRI_sim_collection_t*) doc); } } if (vocbase->_active == 1) { // clean up unused shadows CleanupShadows(vocbase, false); // only sleep while server is still running usleep(COMPACTOR_INTERVAL); } if (active == 2) { // server shutdown break; } } TRI_DestroyVectorPointer(&collections); }
static bool CheckCollection (TRI_collection_t* collection) { TRI_datafile_t* datafile; TRI_vector_pointer_t all; TRI_vector_pointer_t compactors; TRI_vector_pointer_t datafiles; TRI_vector_pointer_t journals; TRI_vector_pointer_t sealed; TRI_vector_string_t files; bool stop; regex_t re; size_t i; size_t n; stop = false; // check files within the directory files = TRI_FilesDirectory(collection->_directory); n = files._length; regcomp(&re, "^(journal|datafile|index|compactor)-([0-9][0-9]*)\\.(db|json)$", REG_EXTENDED); TRI_InitVectorPointer(&journals, TRI_UNKNOWN_MEM_ZONE); TRI_InitVectorPointer(&compactors, TRI_UNKNOWN_MEM_ZONE); TRI_InitVectorPointer(&datafiles, TRI_UNKNOWN_MEM_ZONE); TRI_InitVectorPointer(&sealed, TRI_UNKNOWN_MEM_ZONE); TRI_InitVectorPointer(&all, TRI_UNKNOWN_MEM_ZONE); for (i = 0; i < n; ++i) { char const* file = files._buffer[i]; regmatch_t matches[4]; if (regexec(&re, file, sizeof(matches) / sizeof(matches[0]), matches, 0) == 0) { char const* first = file + matches[1].rm_so; size_t firstLen = matches[1].rm_eo - matches[1].rm_so; char const* third = file + matches[3].rm_so; size_t thirdLen = matches[3].rm_eo - matches[3].rm_so; // ............................................................................. // file is an index, just store the filename // ............................................................................. if (TRI_EqualString2("index", first, firstLen) && TRI_EqualString2("json", third, thirdLen)) { char* filename; filename = TRI_Concatenate2File(collection->_directory, file); TRI_PushBackVectorString(&collection->_indexFiles, filename); } // ............................................................................. // file is a journal or datafile, open the datafile // ............................................................................. else if (TRI_EqualString2("db", third, thirdLen)) { char* filename; char* ptr; TRI_col_header_marker_t* cm; filename = TRI_Concatenate2File(collection->_directory, file); datafile = TRI_OpenDatafile(filename); if (datafile == NULL) { collection->_lastError = TRI_errno(); stop = true; LOG_ERROR("cannot open datafile '%s': %s", filename, TRI_last_error()); break; } TRI_PushBackVectorPointer(&all, datafile); // check the document header ptr = datafile->_data; ptr += TRI_DF_ALIGN_BLOCK(sizeof(TRI_df_header_marker_t)); cm = (TRI_col_header_marker_t*) ptr; if (cm->base._type != TRI_COL_MARKER_HEADER) { LOG_ERROR("collection header mismatch in file '%s', expected TRI_COL_MARKER_HEADER, found %lu", filename, (unsigned long) cm->base._type); TRI_FreeString(TRI_CORE_MEM_ZONE, filename); stop = true; break; } if (cm->_cid != collection->_info._cid) { LOG_ERROR("collection identifier mismatch, expected %llu, found %llu", (unsigned long long) collection->_info._cid, (unsigned long long) cm->_cid); TRI_FreeString(TRI_CORE_MEM_ZONE, filename); stop = true; break; } // file is a journal if (TRI_EqualString2("journal", first, firstLen)) { if (datafile->_isSealed) { LOG_WARNING("strange, journal '%s' is already sealed; must be a left over; will use it as datafile", filename); TRI_PushBackVectorPointer(&sealed, datafile); } else { TRI_PushBackVectorPointer(&journals, datafile); } } // file is a compactor file else if (TRI_EqualString2("compactor", first, firstLen)) { if (datafile->_isSealed) { LOG_WARNING("strange, compactor journal '%s' is already sealed; must be a left over; will use it as datafile", filename); TRI_PushBackVectorPointer(&sealed, datafile); } else { TRI_PushBackVectorPointer(&compactors, datafile); } } // file is a datafile else if (TRI_EqualString2("datafile", first, firstLen)) { if (! datafile->_isSealed) { LOG_ERROR("datafile '%s' is not sealed, this should never happen", filename); collection->_lastError = TRI_set_errno(TRI_ERROR_ARANGO_CORRUPTED_DATAFILE); stop = true; break; } else { TRI_PushBackVectorPointer(&datafiles, datafile); } } else { LOG_ERROR("unknown datafile '%s'", file); } TRI_FreeString(TRI_CORE_MEM_ZONE, filename); } else { LOG_ERROR("unknown datafile '%s'", file); } } } TRI_DestroyVectorString(&files); regfree(&re); // convert the sealed journals into datafiles if (! stop) { n = sealed._length; for (i = 0; i < n; ++i) { char* number; char* dname; char* filename; bool ok; datafile = sealed._buffer[i]; number = TRI_StringUInt64(datafile->_fid); dname = TRI_Concatenate3String("datafile-", number, ".db"); filename = TRI_Concatenate2File(collection->_directory, dname); TRI_FreeString(TRI_CORE_MEM_ZONE, dname); TRI_FreeString(TRI_CORE_MEM_ZONE, number); ok = TRI_RenameDatafile(datafile, filename); if (ok) { TRI_PushBackVectorPointer(&datafiles, datafile); LOG_DEBUG("renamed sealed journal to '%s'", filename); } else { collection->_lastError = datafile->_lastError; stop = true; LOG_ERROR("cannot rename sealed log-file to %s, this should not happen: %s", filename, TRI_last_error()); break; } TRI_FreeString(TRI_CORE_MEM_ZONE, filename); } } TRI_DestroyVectorPointer(&sealed); // stop if necessary if (stop) { n = all._length; for (i = 0; i < n; ++i) { datafile = all._buffer[i]; LOG_TRACE("closing datafile '%s'", datafile->_filename); TRI_CloseDatafile(datafile); TRI_FreeDatafile(datafile); } TRI_DestroyVectorPointer(&all); TRI_DestroyVectorPointer(&datafiles); return false; } TRI_DestroyVectorPointer(&all); // add the datafiles and journals collection->_datafiles = datafiles; collection->_journals = journals; collection->_compactors = compactors; return true; }
TRI_aql_index_t* TRI_DetermineIndexAql (TRI_aql_context_t* const context, const TRI_vector_pointer_t* const availableIndexes, const char* const collectionName, const TRI_vector_pointer_t* candidates) { TRI_aql_index_t* picked = NULL; TRI_vector_pointer_t matches; size_t i, n; TRI_InitVectorPointer(&matches, TRI_UNKNOWN_MEM_ZONE); assert(context); assert(collectionName); assert(candidates); n = availableIndexes->_length; for (i = 0; i < n; ++i) { TRI_index_t* idx = (TRI_index_t*) availableIndexes->_buffer[i]; size_t numIndexFields; bool lastTypeWasExact; size_t j; if (! CanUseIndex(idx)) { continue; } LogIndexString("checking", idx, collectionName); TRI_ClearVectorPointer(&matches); lastTypeWasExact = true; numIndexFields = idx->_fields._length; // now loop over all index fields, from left to right // index field order is important because skiplists can be used with leftmost prefixes as well, // but not with rightmost prefixes for (j = 0; j < numIndexFields; ++j) { char* indexedFieldName; char* fieldName; size_t k; indexedFieldName = idx->_fields._buffer[j]; if (indexedFieldName == NULL) { continue; } // now loop over all candidates for (k = 0; k < candidates->_length; ++k) { TRI_aql_field_access_t* candidate = (TRI_aql_field_access_t*) TRI_AtVectorPointer(candidates, k); if (candidate->_type == TRI_AQL_ACCESS_IMPOSSIBLE || candidate->_type == TRI_AQL_ACCESS_ALL) { // wrong index type, doesn't help us at all continue; } fieldName = candidate->_fullName + candidate->_variableNameLength + 1; if (idx->_type == TRI_IDX_TYPE_PRIMARY_INDEX) { // primary index key names must be treated differently. _id and _key are the same if (! TRI_EqualString("_id", fieldName) && ! TRI_EqualString(TRI_VOC_ATTRIBUTE_KEY, fieldName)) { continue; } } else if (idx->_type == TRI_IDX_TYPE_EDGE_INDEX) { // edge index key names must be treated differently. _from and _to can be used independently if (! TRI_EqualString(TRI_VOC_ATTRIBUTE_FROM, fieldName) && ! TRI_EqualString(TRI_VOC_ATTRIBUTE_TO, fieldName)) { continue; } } else if (! TRI_EqualString(indexedFieldName, fieldName)) { // different attribute, doesn't help continue; } // attribute is used in index if (idx->_type == TRI_IDX_TYPE_PRIMARY_INDEX || idx->_type == TRI_IDX_TYPE_EDGE_INDEX) { if (! IsExactCandidate(candidate)) { // wrong access type for primary index continue; } TRI_PushBackVectorPointer(&matches, candidate); } else if (idx->_type == TRI_IDX_TYPE_HASH_INDEX) { if (! IsExactCandidate(candidate)) { // wrong access type for hash index continue; } if (candidate->_type == TRI_AQL_ACCESS_LIST && numIndexFields != 1) { // we found a list, but the index covers multiple attributes. that means we cannot use list access continue; } TRI_PushBackVectorPointer(&matches, candidate); } else if (idx->_type == TRI_IDX_TYPE_BITARRAY_INDEX) { if (! IsExactCandidate(candidate)) { // wrong access type for hash index continue; } if (candidate->_type == TRI_AQL_ACCESS_LIST) { // we found a list, but the index covers multiple attributes. that means we cannot use list access continue; } TRI_PushBackVectorPointer(&matches, candidate); } else if (idx->_type == TRI_IDX_TYPE_SKIPLIST_INDEX) { bool candidateIsExact; if (candidate->_type != TRI_AQL_ACCESS_EXACT && candidate->_type != TRI_AQL_ACCESS_LIST && candidate->_type != TRI_AQL_ACCESS_RANGE_SINGLE && candidate->_type != TRI_AQL_ACCESS_RANGE_DOUBLE && candidate->_type != TRI_AQL_ACCESS_REFERENCE) { // wrong access type for skiplists continue; } if (candidate->_type == TRI_AQL_ACCESS_LIST && numIndexFields != 1) { // we found a list, but the index covers multiple attributes. that means we cannot use list access continue; } candidateIsExact = IsExactCandidate(candidate); if ((candidateIsExact && ! lastTypeWasExact) || (! candidateIsExact && ! lastTypeWasExact)) { // if we already had a range query, we cannot check for equality after that // if we already had a range query, we cannot check another range after that continue; } if (candidate->_type == TRI_AQL_ACCESS_RANGE_SINGLE) { // range type. check if the compare value is a list or an object TRI_json_t* value = candidate->_value._singleRange._value; if (TRI_IsListJson(value) || TRI_IsArrayJson(value)) { // list or object, we cannot use this for comparison in a skiplist continue; } } else if (candidate->_type == TRI_AQL_ACCESS_RANGE_DOUBLE) { // range type. check if the compare value is a list or an object TRI_json_t* value = candidate->_value._between._lower._value; if (TRI_IsListJson(value) || TRI_IsArrayJson(value)) { // list or object, we cannot use this for comparison in a skiplist continue; } value = candidate->_value._between._upper._value; if (TRI_IsListJson(value) || TRI_IsArrayJson(value)) { // list or object, we cannot use this for comparison in a skiplist continue; } } lastTypeWasExact = candidateIsExact; TRI_PushBackVectorPointer(&matches, candidate); } } // finished iterating over all candidates if (matches._length != j + 1) { // we already have picked less candidate fields than we should break; } } if (matches._length < 1) { // nothing found continue; } // we now do or don't have an index candidate in the matches vector if (matches._length < numIndexFields && TRI_NeedsFullCoverageIndex(idx->_type)) { // the matches vector does not fully cover the indexed fields, but the index requires it continue; } // if we can use the primary index, we'll use it picked = PickIndex(context, picked, idx, &matches); } TRI_DestroyVectorPointer(&matches); if (picked) { LogIndexString("using", picked->_idx, collectionName); } return picked; }
void TRI_CleanupVocBase (void* data) { TRI_vocbase_t* vocbase; TRI_vector_pointer_t collections; uint64_t iterations = 0; vocbase = data; assert(vocbase); assert(vocbase->_state == 1); TRI_InitVectorPointer(&collections, TRI_UNKNOWN_MEM_ZONE); while (true) { int state; // keep initial _state value as vocbase->_state might change during cleanup loop state = vocbase->_state; ++iterations; if (state == 2) { // shadows must be cleaned before collections are handled // otherwise the shadows might still hold barriers on collections // and collections cannot be closed properly CleanupCursors(vocbase, true); } // check if we can get the compactor lock exclusively if (TRI_CheckAndLockCompactorVocBase(vocbase)) { size_t i, n; TRI_col_type_e type; // copy all collections TRI_READ_LOCK_COLLECTIONS_VOCBASE(vocbase); TRI_CopyDataVectorPointer(&collections, &vocbase->_collections); TRI_READ_UNLOCK_COLLECTIONS_VOCBASE(vocbase); n = collections._length; for (i = 0; i < n; ++i) { TRI_vocbase_col_t* collection; TRI_primary_collection_t* primary; collection = collections._buffer[i]; TRI_READ_LOCK_STATUS_VOCBASE_COL(collection); primary = collection->_collection; if (primary == NULL) { TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); continue; } type = primary->base._info._type; TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); // we're the only ones that can unload the collection, so using // the collection pointer outside the lock is ok // maybe cleanup indexes, unload the collection or some datafiles if (TRI_IS_DOCUMENT_COLLECTION(type)) { TRI_document_collection_t* document = (TRI_document_collection_t*) primary; // clean indexes? if (iterations % (uint64_t) CLEANUP_INDEX_ITERATIONS == 0) { document->cleanupIndexes(document); } CleanupDocumentCollection(document); } } TRI_UnlockCompactorVocBase(vocbase); } if (vocbase->_state >= 1) { // server is still running, clean up unused shadows if (iterations % CLEANUP_SHADOW_ITERATIONS == 0) { CleanupCursors(vocbase, false); } // clean up expired compactor locks TRI_CleanupCompactorVocBase(vocbase); if (state == 1) { TRI_LockCondition(&vocbase->_cleanupCondition); TRI_TimedWaitCondition(&vocbase->_cleanupCondition, (uint64_t) CLEANUP_INTERVAL); TRI_UnlockCondition(&vocbase->_cleanupCondition); } } if (state == 3) { // server shutdown break; } } TRI_DestroyVectorPointer(&collections); LOG_TRACE("shutting down cleanup thread"); }
void TRI_CleanupVocBase (void* data) { TRI_vocbase_t* vocbase = data; TRI_vector_pointer_t collections; assert(vocbase->_state == 1); TRI_InitVectorPointer(&collections, TRI_UNKNOWN_MEM_ZONE); while (true) { size_t n; size_t i; TRI_col_type_e type; // keep initial _state value as vocbase->_state might change during compaction loop int state = vocbase->_state; if (state == 2) { // shadows must be cleaned before collections are handled // otherwise the shadows might still hold barriers on collections // and collections cannot be closed properly CleanupShadows(vocbase, true); } // copy all collections TRI_READ_LOCK_COLLECTIONS_VOCBASE(vocbase); TRI_CopyDataVectorPointer(&collections, &vocbase->_collections); TRI_READ_UNLOCK_COLLECTIONS_VOCBASE(vocbase); n = collections._length; for (i = 0; i < n; ++i) { TRI_vocbase_col_t* collection; TRI_primary_collection_t* primary; collection = collections._buffer[i]; TRI_READ_LOCK_STATUS_VOCBASE_COL(collection); primary = collection->_collection; if (primary == NULL) { TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); continue; } type = primary->base._type; TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); // now release the lock and maybe unload the collection or some datafiles if (TRI_IS_DOCUMENT_COLLECTION(type)) { CleanupDocumentCollection((TRI_document_collection_t*) primary); } } if (vocbase->_state >= 1) { // server is still running, clean up unused shadows CleanupShadows(vocbase, false); TRI_LockCondition(&vocbase->_cleanupCondition); TRI_TimedWaitCondition(&vocbase->_cleanupCondition, CLEANUP_INTERVAL); TRI_UnlockCondition(&vocbase->_cleanupCondition); } if (state == 3) { // server shutdown break; } } TRI_DestroyVectorPointer(&collections); }
void TRI_CompactorVocBase (void* data) { TRI_vocbase_t* vocbase; TRI_vector_pointer_t collections; vocbase = data; assert(vocbase->_state == 1); TRI_InitVectorPointer(&collections, TRI_UNKNOWN_MEM_ZONE); while (true) { int state; // keep initial _state value as vocbase->_state might change during compaction loop state = vocbase->_state; // check if compaction is currently disallowed if (CheckAndLockCompaction(vocbase)) { // compaction is currently allowed size_t i, n; // copy all collections TRI_READ_LOCK_COLLECTIONS_VOCBASE(vocbase); TRI_CopyDataVectorPointer(&collections, &vocbase->_collections); TRI_READ_UNLOCK_COLLECTIONS_VOCBASE(vocbase); n = collections._length; for (i = 0; i < n; ++i) { TRI_vocbase_col_t* collection; TRI_primary_collection_t* primary; TRI_col_type_e type; bool doCompact; bool worked; collection = collections._buffer[i]; if (! TRI_TRY_READ_LOCK_STATUS_VOCBASE_COL(collection)) { // if we can't acquire the read lock instantly, we continue directly // we don't want to stall here for too long continue; } primary = collection->_collection; if (primary == NULL) { TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); continue; } worked = false; doCompact = primary->base._info._doCompact; type = primary->base._info._type; // for document collection, compactify datafiles if (TRI_IS_DOCUMENT_COLLECTION(type)) { if (collection->_status == TRI_VOC_COL_STATUS_LOADED && doCompact) { TRI_barrier_t* ce; // check whether someone else holds a read-lock on the compaction lock if (! TRI_TryWriteLockReadWriteLock(&primary->_compactionLock)) { // someone else is holding the compactor lock, we'll not compact TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); continue; } ce = TRI_CreateBarrierCompaction(&primary->_barrierList); if (ce == NULL) { // out of memory LOG_WARNING("out of memory when trying to create a barrier element"); } else { worked = CompactifyDocumentCollection((TRI_document_collection_t*) primary); TRI_FreeBarrier(ce); } // read-unlock the compaction lock TRI_WriteUnlockReadWriteLock(&primary->_compactionLock); } } TRI_READ_UNLOCK_STATUS_VOCBASE_COL(collection); if (worked) { // signal the cleanup thread that we worked and that it can now wake up TRI_LockCondition(&vocbase->_cleanupCondition); TRI_SignalCondition(&vocbase->_cleanupCondition); TRI_UnlockCondition(&vocbase->_cleanupCondition); } } UnlockCompaction(vocbase); } if (state != 2 && vocbase->_state == 1) { // only sleep while server is still running TRI_LockCondition(&vocbase->_compactorCondition); TRI_TimedWaitCondition(&vocbase->_compactorCondition, (uint64_t) COMPACTOR_INTERVAL); TRI_UnlockCondition(&vocbase->_compactorCondition); } if (state == 2) { // server shutdown break; } } TRI_DestroyVectorPointer(&collections); LOG_TRACE("shutting down compactor thread"); }
TRI_aql_context_t* TRI_CreateContextAql (TRI_vocbase_t* vocbase, const char* const query, const size_t queryLength, bool isCoordinator, TRI_json_t* userOptions) { TRI_aql_context_t* context; int res; TRI_ASSERT(vocbase != NULL); TRI_ASSERT(query != NULL); LOG_TRACE("creating context"); context = (TRI_aql_context_t*) TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_aql_context_t), false); if (context == NULL) { return NULL; } context->_type = TRI_AQL_QUERY_READ; context->_vocbase = vocbase; context->_userOptions = userOptions; context->_writeOptions = NULL; context->_writeCollection = NULL; context->_variableIndex = 0; context->_scopeIndex = 0; context->_subQueries = 0; // actual bind parameter values res = TRI_InitAssociativePointer(&context->_parameters._values, TRI_UNKNOWN_MEM_ZONE, &TRI_HashStringKeyAssociativePointer, &TRI_HashBindParameterAql, &TRI_EqualBindParameterAql, 0); if (res != TRI_ERROR_NO_ERROR) { TRI_Free(TRI_UNKNOWN_MEM_ZONE, context); return NULL; } // bind parameter names used in the query res = TRI_InitAssociativePointer(&context->_parameters._names, TRI_UNKNOWN_MEM_ZONE, &TRI_HashStringKeyAssociativePointer, &TRI_HashStringKeyAssociativePointer, &TRI_EqualStringKeyAssociativePointer, 0); if (res != TRI_ERROR_NO_ERROR) { TRI_DestroyAssociativePointer(&context->_parameters._values); TRI_Free(TRI_UNKNOWN_MEM_ZONE, context); return NULL; } // collections res = TRI_InitAssociativePointer(&context->_collectionNames, TRI_UNKNOWN_MEM_ZONE, &TRI_HashStringKeyAssociativePointer, &TRI_HashStringKeyAssociativePointer, &TRI_EqualStringKeyAssociativePointer, 0); if (res != TRI_ERROR_NO_ERROR) { TRI_DestroyAssociativePointer(&context->_parameters._names); TRI_DestroyAssociativePointer(&context->_parameters._values); TRI_Free(TRI_UNKNOWN_MEM_ZONE, context); return NULL; } TRI_InitVectorPointer2(&context->_memory._nodes, TRI_UNKNOWN_MEM_ZONE, 16); TRI_InitVectorPointer2(&context->_memory._strings, TRI_UNKNOWN_MEM_ZONE, 16); TRI_InitVectorPointer(&context->_collections, TRI_UNKNOWN_MEM_ZONE); TRI_InitErrorAql(&context->_error); context->_parser = NULL; context->_statements = NULL; context->_query = query; TRI_InitScopesAql(context); context->_parser = TRI_CreateParserAql(context->_query, queryLength); if (context->_parser == NULL) { // could not create the parser TRI_FreeContextAql(context); return NULL; } if (! TRI_InitParserAql(context)) { // could not initialise the lexer TRI_FreeContextAql(context); return NULL; } context->_statements = TRI_CreateStatementListAql(); if (context->_statements == NULL) { // could not create statement list TRI_FreeContextAql(context); return NULL; } ProcessOptions(context); context->_isCoordinator = isCoordinator; return context; }