void TRI_headers_t::moveBack (TRI_doc_mptr_t* header, TRI_doc_mptr_t* old) { if (header == nullptr) { return; } TRI_ASSERT(_nrAllocated > 0); TRI_ASSERT(_nrLinked > 0); TRI_ASSERT(_totalSize > 0); // we have at least one element in the list TRI_ASSERT(_begin != nullptr); TRI_ASSERT(_end != nullptr); TRI_ASSERT(header->_prev != header); TRI_ASSERT(header->_next != header); TRI_ASSERT(old != nullptr); TRI_ASSERT(old->getDataPtr() != nullptr); // ONLY IN HEADERS, PROTECTED by RUNTIME int64_t newSize = (int64_t) (((TRI_df_marker_t*) header->getDataPtr())->_size); // ONLY IN HEADERS, PROTECTED by RUNTIME int64_t oldSize = (int64_t) (((TRI_df_marker_t*) old->getDataPtr())->_size); // ONLY IN HEADERS, PROTECTED by RUNTIME // we must adjust the size of the collection _totalSize += ( TRI_DF_ALIGN_BLOCK(newSize) - TRI_DF_ALIGN_BLOCK(oldSize)); if (_end == header) { // header is already at the end TRI_ASSERT(header->_next == nullptr); return; } TRI_ASSERT(_begin != _end); // unlink the element if (header->_prev != nullptr) { header->_prev->_next = header->_next; } if (header->_next != nullptr) { header->_next->_prev = header->_prev; } if (_begin == header) { TRI_ASSERT(header->_next != nullptr); _begin = header->_next; } header->_prev = _end; header->_next = nullptr; _end = header; header->_prev->_next = header; TRI_ASSERT(_begin != nullptr); TRI_ASSERT(_end != nullptr); TRI_ASSERT(header->_prev != header); TRI_ASSERT(header->_next != header); TRI_ASSERT(_totalSize > 0); }
void TRI_headers_t::adjustTotalSize (int64_t oldSize, int64_t newSize) { // oldSize = size of marker in WAL // newSize = size of marker in datafile _totalSize -= ( TRI_DF_ALIGN_BLOCK(oldSize) - TRI_DF_ALIGN_BLOCK(newSize)); }
static bool CalculateSize (TRI_df_marker_t const* marker, void* data, TRI_datafile_t* datafile, bool journal) { TRI_document_collection_t* document; TRI_primary_collection_t* primary; compaction_initial_context_t* context; TRI_voc_size_t alignedSize; context = data; document = context->_document; primary = &document->base; alignedSize = TRI_DF_ALIGN_BLOCK(marker->_size); // new or updated document if (marker->_type == TRI_DOC_MARKER_KEY_DOCUMENT || marker->_type == TRI_DOC_MARKER_KEY_EDGE) { TRI_doc_document_key_marker_t const* d; TRI_doc_mptr_t const* found; TRI_voc_key_t key; bool deleted; d = (TRI_doc_document_key_marker_t const*) marker; key = (char*) d + d->_offsetKey; // check if the document is still active TRI_READ_LOCK_DOCUMENTS_INDEXES_PRIMARY_COLLECTION(primary); found = TRI_LookupByKeyAssociativePointer(&primary->_primaryIndex, key); deleted = (found == NULL || found->_rid > d->_rid); TRI_READ_UNLOCK_DOCUMENTS_INDEXES_PRIMARY_COLLECTION(primary); if (deleted) { return true; } context->_keepDeletions = true; context->_targetSize += alignedSize; } else if (marker->_type == TRI_DOC_MARKER_KEY_DELETION && context->_keepDeletions) { context->_targetSize += alignedSize; } else if (marker->_type == TRI_DOC_MARKER_BEGIN_TRANSACTION || marker->_type == TRI_DOC_MARKER_COMMIT_TRANSACTION || marker->_type == TRI_DOC_MARKER_ABORT_TRANSACTION || marker->_type == TRI_DOC_MARKER_PREPARE_TRANSACTION) { context->_targetSize += alignedSize; } return true; }
void TRI_headers_t::unlink (TRI_doc_mptr_t* header) { int64_t size; TRI_ASSERT(header != nullptr); TRI_ASSERT(header->getDataPtr() != nullptr); // ONLY IN HEADERS, PROTECTED by RUNTIME TRI_ASSERT(header->_prev != header); TRI_ASSERT(header->_next != header); size = (int64_t) ((TRI_df_marker_t*) header->getDataPtr())->_size; // ONLY IN HEADERS, PROTECTED by RUNTIME TRI_ASSERT(size > 0); // unlink the header if (header->_prev != nullptr) { header->_prev->_next = header->_next; } if (header->_next != nullptr) { header->_next->_prev = header->_prev; } // adjust begin & end pointers if (_begin == header) { _begin = header->_next; } if (_end == header) { _end = header->_prev; } TRI_ASSERT(_begin != header); TRI_ASSERT(_end != header); TRI_ASSERT(_nrLinked > 0); _nrLinked--; _totalSize -= TRI_DF_ALIGN_BLOCK(size); if (_nrLinked == 0) { TRI_ASSERT(_begin == nullptr); TRI_ASSERT(_end == nullptr); TRI_ASSERT(_totalSize == 0); } else { TRI_ASSERT(_begin != nullptr); TRI_ASSERT(_end != nullptr); TRI_ASSERT(_totalSize > 0); } TRI_ASSERT(header->_prev != header); TRI_ASSERT(header->_next != header); }
void TRI_headers_t::relink (TRI_doc_mptr_t* header, TRI_doc_mptr_t* old) { if (header == nullptr) { return; } TRI_ASSERT(header->getDataPtr() != nullptr); // ONLY IN HEADERS, PROTECTED by RUNTIME int64_t size = (int64_t) ((TRI_df_marker_t*) header->getDataPtr())->_size; // ONLY IN HEADERS, PROTECTED by RUNTIME TRI_ASSERT(size > 0); TRI_ASSERT(_begin != header); TRI_ASSERT(_end != header); this->move(header, old); _nrLinked++; _totalSize += TRI_DF_ALIGN_BLOCK(size); TRI_ASSERT(_totalSize > 0); TRI_ASSERT(header->_prev != header); TRI_ASSERT(header->_next != header); }
static bool CheckCollection (TRI_collection_t* collection) { TRI_datafile_t* datafile; TRI_vector_pointer_t all; TRI_vector_pointer_t compactors; TRI_vector_pointer_t datafiles; TRI_vector_pointer_t journals; TRI_vector_pointer_t sealed; TRI_vector_string_t files; bool stop; regex_t re; size_t i; size_t n; stop = false; // check files within the directory files = TRI_FilesDirectory(collection->_directory); n = files._length; regcomp(&re, "^(journal|datafile|index|compactor)-([0-9][0-9]*)\\.(db|json)$", REG_EXTENDED); TRI_InitVectorPointer(&journals, TRI_UNKNOWN_MEM_ZONE); TRI_InitVectorPointer(&compactors, TRI_UNKNOWN_MEM_ZONE); TRI_InitVectorPointer(&datafiles, TRI_UNKNOWN_MEM_ZONE); TRI_InitVectorPointer(&sealed, TRI_UNKNOWN_MEM_ZONE); TRI_InitVectorPointer(&all, TRI_UNKNOWN_MEM_ZONE); for (i = 0; i < n; ++i) { char const* file = files._buffer[i]; regmatch_t matches[4]; if (regexec(&re, file, sizeof(matches) / sizeof(matches[0]), matches, 0) == 0) { char const* first = file + matches[1].rm_so; size_t firstLen = matches[1].rm_eo - matches[1].rm_so; char const* third = file + matches[3].rm_so; size_t thirdLen = matches[3].rm_eo - matches[3].rm_so; // ............................................................................. // file is an index, just store the filename // ............................................................................. if (TRI_EqualString2("index", first, firstLen) && TRI_EqualString2("json", third, thirdLen)) { char* filename; filename = TRI_Concatenate2File(collection->_directory, file); TRI_PushBackVectorString(&collection->_indexFiles, filename); } // ............................................................................. // file is a journal or datafile, open the datafile // ............................................................................. else if (TRI_EqualString2("db", third, thirdLen)) { char* filename; char* ptr; TRI_col_header_marker_t* cm; filename = TRI_Concatenate2File(collection->_directory, file); datafile = TRI_OpenDatafile(filename); if (datafile == NULL) { collection->_lastError = TRI_errno(); stop = true; LOG_ERROR("cannot open datafile '%s': %s", filename, TRI_last_error()); break; } TRI_PushBackVectorPointer(&all, datafile); // check the document header ptr = datafile->_data; ptr += TRI_DF_ALIGN_BLOCK(sizeof(TRI_df_header_marker_t)); cm = (TRI_col_header_marker_t*) ptr; if (cm->base._type != TRI_COL_MARKER_HEADER) { LOG_ERROR("collection header mismatch in file '%s', expected TRI_COL_MARKER_HEADER, found %lu", filename, (unsigned long) cm->base._type); TRI_FreeString(TRI_CORE_MEM_ZONE, filename); stop = true; break; } if (cm->_cid != collection->_info._cid) { LOG_ERROR("collection identifier mismatch, expected %llu, found %llu", (unsigned long long) collection->_info._cid, (unsigned long long) cm->_cid); TRI_FreeString(TRI_CORE_MEM_ZONE, filename); stop = true; break; } // file is a journal if (TRI_EqualString2("journal", first, firstLen)) { if (datafile->_isSealed) { LOG_WARNING("strange, journal '%s' is already sealed; must be a left over; will use it as datafile", filename); TRI_PushBackVectorPointer(&sealed, datafile); } else { TRI_PushBackVectorPointer(&journals, datafile); } } // file is a compactor file else if (TRI_EqualString2("compactor", first, firstLen)) { if (datafile->_isSealed) { LOG_WARNING("strange, compactor journal '%s' is already sealed; must be a left over; will use it as datafile", filename); TRI_PushBackVectorPointer(&sealed, datafile); } else { TRI_PushBackVectorPointer(&compactors, datafile); } } // file is a datafile else if (TRI_EqualString2("datafile", first, firstLen)) { if (! datafile->_isSealed) { LOG_ERROR("datafile '%s' is not sealed, this should never happen", filename); collection->_lastError = TRI_set_errno(TRI_ERROR_ARANGO_CORRUPTED_DATAFILE); stop = true; break; } else { TRI_PushBackVectorPointer(&datafiles, datafile); } } else { LOG_ERROR("unknown datafile '%s'", file); } TRI_FreeString(TRI_CORE_MEM_ZONE, filename); } else { LOG_ERROR("unknown datafile '%s'", file); } } } TRI_DestroyVectorString(&files); regfree(&re); // convert the sealed journals into datafiles if (! stop) { n = sealed._length; for (i = 0; i < n; ++i) { char* number; char* dname; char* filename; bool ok; datafile = sealed._buffer[i]; number = TRI_StringUInt64(datafile->_fid); dname = TRI_Concatenate3String("datafile-", number, ".db"); filename = TRI_Concatenate2File(collection->_directory, dname); TRI_FreeString(TRI_CORE_MEM_ZONE, dname); TRI_FreeString(TRI_CORE_MEM_ZONE, number); ok = TRI_RenameDatafile(datafile, filename); if (ok) { TRI_PushBackVectorPointer(&datafiles, datafile); LOG_DEBUG("renamed sealed journal to '%s'", filename); } else { collection->_lastError = datafile->_lastError; stop = true; LOG_ERROR("cannot rename sealed log-file to %s, this should not happen: %s", filename, TRI_last_error()); break; } TRI_FreeString(TRI_CORE_MEM_ZONE, filename); } } TRI_DestroyVectorPointer(&sealed); // stop if necessary if (stop) { n = all._length; for (i = 0; i < n; ++i) { datafile = all._buffer[i]; LOG_TRACE("closing datafile '%s'", datafile->_filename); TRI_CloseDatafile(datafile); TRI_FreeDatafile(datafile); } TRI_DestroyVectorPointer(&all); TRI_DestroyVectorPointer(&datafiles); return false; } TRI_DestroyVectorPointer(&all); // add the datafiles and journals collection->_datafiles = datafiles; collection->_journals = journals; collection->_compactors = compactors; return true; }
TRI_doc_mptr_t* TRI_headers_t::request (size_t size) { TRI_doc_mptr_t* header; TRI_ASSERT(size > 0); if (_freelist == nullptr) { size_t blockSize = GetBlockSize(_blocks._length); TRI_ASSERT(blockSize > 0); TRI_doc_mptr_t* begin; try { begin = new TRI_doc_mptr_t[blockSize]; } catch (std::exception&) { begin = nullptr; } // out of memory if (begin == nullptr) { TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY); return nullptr; } TRI_doc_mptr_t* ptr = begin + (blockSize - 1); header = nullptr; for (; begin <= ptr; ptr--) { ptr->setDataPtr(header); // ONLY IN HEADERS header = ptr; } _freelist = header; TRI_PushBackVectorPointer(&_blocks, begin); } TRI_ASSERT(_freelist != nullptr); TRI_doc_mptr_t* result = const_cast<TRI_doc_mptr_t*>(_freelist); TRI_ASSERT(result != nullptr); _freelist = static_cast<TRI_doc_mptr_t const*>(result->getDataPtr()); // ONLY IN HEADERS, PROTECTED by RUNTIME result->setDataPtr(nullptr); // ONLY IN HEADERS // put new header at the end of the list if (_begin == nullptr) { // list of headers is empty TRI_ASSERT(_nrLinked == 0); TRI_ASSERT(_totalSize == 0); _begin = result; _end = result; result->_prev = nullptr; result->_next = nullptr; } else { // list is not empty TRI_ASSERT(_nrLinked > 0); TRI_ASSERT(_totalSize > 0); TRI_ASSERT(_nrAllocated > 0); TRI_ASSERT(_begin != nullptr); TRI_ASSERT(_end != nullptr); _end->_next = result; result->_prev = _end; result->_next = nullptr; _end = result; } _nrAllocated++; _nrLinked++; _totalSize += (int64_t) TRI_DF_ALIGN_BLOCK(size); return result; }
void TRI_headers_t::move (TRI_doc_mptr_t* header, TRI_doc_mptr_t* old) { if (header == nullptr) { return; } TRI_ASSERT(_nrAllocated > 0); TRI_ASSERT(header->_prev != header); TRI_ASSERT(header->_next != header); TRI_ASSERT(header->getDataPtr() != nullptr); // ONLY IN HEADERS, PROTECTED by RUNTIME TRI_ASSERT(((TRI_df_marker_t*) header->getDataPtr())->_size > 0); // ONLY IN HEADERS, PROTECTED by RUNTIME TRI_ASSERT(old != nullptr); TRI_ASSERT(old->getDataPtr() != nullptr); // ONLY IN HEADERS, PROTECTED by RUNTIME int64_t newSize = (int64_t) (((TRI_df_marker_t*) header->getDataPtr())->_size); // ONLY IN HEADERS, PROTECTED by RUNTIME int64_t oldSize = (int64_t) (((TRI_df_marker_t*) old->getDataPtr())->_size); // ONLY IN HEADERS, PROTECTED by RUNTIME // Please note the following: This operation is only used to revert an // update operation. The "new" document is removed again and the "old" // one is used once more. Therefore, the signs in the following statement // are actually OK: _totalSize -= ( TRI_DF_ALIGN_BLOCK(newSize) - TRI_DF_ALIGN_BLOCK(oldSize)); // adjust list start and end pointers if (old->_prev == nullptr) { _begin = header; } else if (_begin == header) { if (old->_prev != nullptr) { _begin = old->_prev; } } if (old->_next == nullptr) { _end = header; } else if (_end == header) { if (old->_next != nullptr) { _end = old->_next; } } if (header->_prev != nullptr) { header->_prev->_next = header->_next; } if (header->_next != nullptr) { header->_next->_prev = header->_prev; } if (old->_prev != nullptr) { old->_prev->_next = header; } if (old->_next != nullptr) { old->_next->_prev = header; } header->_prev = old->_prev; header->_next = old->_next; TRI_ASSERT(_begin != nullptr); TRI_ASSERT(_end != nullptr); TRI_ASSERT(header->_prev != header); TRI_ASSERT(header->_next != header); }