void TRI_headers_t::moveBack (TRI_doc_mptr_t* header,
TRI_doc_mptr_t* old) {
if (header == nullptr) {
return;
}
TRI_ASSERT(_nrAllocated > 0);
TRI_ASSERT(_nrLinked > 0);
TRI_ASSERT(_totalSize > 0);
// we have at least one element in the list
TRI_ASSERT(_begin != nullptr);
TRI_ASSERT(_end != nullptr);
TRI_ASSERT(header->_prev != header);
TRI_ASSERT(header->_next != header);
TRI_ASSERT(old != nullptr);
TRI_ASSERT(old->getDataPtr() != nullptr); // ONLY IN HEADERS, PROTECTED by RUNTIME
int64_t newSize = (int64_t) (((TRI_df_marker_t*) header->getDataPtr())->_size); // ONLY IN HEADERS, PROTECTED by RUNTIME
int64_t oldSize = (int64_t) (((TRI_df_marker_t*) old->getDataPtr())->_size); // ONLY IN HEADERS, PROTECTED by RUNTIME
// we must adjust the size of the collection
_totalSize += ( TRI_DF_ALIGN_BLOCK(newSize)
- TRI_DF_ALIGN_BLOCK(oldSize));
if (_end == header) {
// header is already at the end
TRI_ASSERT(header->_next == nullptr);
return;
}
TRI_ASSERT(_begin != _end);
// unlink the element
if (header->_prev != nullptr) {
header->_prev->_next = header->_next;
}
if (header->_next != nullptr) {
header->_next->_prev = header->_prev;
}
if (_begin == header) {
TRI_ASSERT(header->_next != nullptr);
_begin = header->_next;
}
header->_prev = _end;
header->_next = nullptr;
_end = header;
header->_prev->_next = header;
TRI_ASSERT(_begin != nullptr);
TRI_ASSERT(_end != nullptr);
TRI_ASSERT(header->_prev != header);
TRI_ASSERT(header->_next != header);
TRI_ASSERT(_totalSize > 0);
}
void TRI_headers_t::adjustTotalSize (int64_t oldSize,
int64_t newSize) {
// oldSize = size of marker in WAL
// newSize = size of marker in datafile
_totalSize -= ( TRI_DF_ALIGN_BLOCK(oldSize)
- TRI_DF_ALIGN_BLOCK(newSize));
}
static bool CalculateSize (TRI_df_marker_t const* marker,
void* data,
TRI_datafile_t* datafile,
bool journal) {
TRI_document_collection_t* document;
TRI_primary_collection_t* primary;
compaction_initial_context_t* context;
TRI_voc_size_t alignedSize;
context = data;
document = context->_document;
primary = &document->base;
alignedSize = TRI_DF_ALIGN_BLOCK(marker->_size);
// new or updated document
if (marker->_type == TRI_DOC_MARKER_KEY_DOCUMENT ||
marker->_type == TRI_DOC_MARKER_KEY_EDGE) {
TRI_doc_document_key_marker_t const* d;
TRI_doc_mptr_t const* found;
TRI_voc_key_t key;
bool deleted;
d = (TRI_doc_document_key_marker_t const*) marker;
key = (char*) d + d->_offsetKey;
// check if the document is still active
TRI_READ_LOCK_DOCUMENTS_INDEXES_PRIMARY_COLLECTION(primary);
found = TRI_LookupByKeyAssociativePointer(&primary->_primaryIndex, key);
deleted = (found == NULL || found->_rid > d->_rid);
TRI_READ_UNLOCK_DOCUMENTS_INDEXES_PRIMARY_COLLECTION(primary);
if (deleted) {
return true;
}
context->_keepDeletions = true;
context->_targetSize += alignedSize;
}
else if (marker->_type == TRI_DOC_MARKER_KEY_DELETION &&
context->_keepDeletions) {
context->_targetSize += alignedSize;
}
else if (marker->_type == TRI_DOC_MARKER_BEGIN_TRANSACTION ||
marker->_type == TRI_DOC_MARKER_COMMIT_TRANSACTION ||
marker->_type == TRI_DOC_MARKER_ABORT_TRANSACTION ||
marker->_type == TRI_DOC_MARKER_PREPARE_TRANSACTION) {
context->_targetSize += alignedSize;
}
return true;
}
void TRI_headers_t::unlink (TRI_doc_mptr_t* header) {
int64_t size;
TRI_ASSERT(header != nullptr);
TRI_ASSERT(header->getDataPtr() != nullptr); // ONLY IN HEADERS, PROTECTED by RUNTIME
TRI_ASSERT(header->_prev != header);
TRI_ASSERT(header->_next != header);
size = (int64_t) ((TRI_df_marker_t*) header->getDataPtr())->_size; // ONLY IN HEADERS, PROTECTED by RUNTIME
TRI_ASSERT(size > 0);
// unlink the header
if (header->_prev != nullptr) {
header->_prev->_next = header->_next;
}
if (header->_next != nullptr) {
header->_next->_prev = header->_prev;
}
// adjust begin & end pointers
if (_begin == header) {
_begin = header->_next;
}
if (_end == header) {
_end = header->_prev;
}
TRI_ASSERT(_begin != header);
TRI_ASSERT(_end != header);
TRI_ASSERT(_nrLinked > 0);
_nrLinked--;
_totalSize -= TRI_DF_ALIGN_BLOCK(size);
if (_nrLinked == 0) {
TRI_ASSERT(_begin == nullptr);
TRI_ASSERT(_end == nullptr);
TRI_ASSERT(_totalSize == 0);
}
else {
TRI_ASSERT(_begin != nullptr);
TRI_ASSERT(_end != nullptr);
TRI_ASSERT(_totalSize > 0);
}
TRI_ASSERT(header->_prev != header);
TRI_ASSERT(header->_next != header);
}
void TRI_headers_t::relink (TRI_doc_mptr_t* header,
TRI_doc_mptr_t* old) {
if (header == nullptr) {
return;
}
TRI_ASSERT(header->getDataPtr() != nullptr); // ONLY IN HEADERS, PROTECTED by RUNTIME
int64_t size = (int64_t) ((TRI_df_marker_t*) header->getDataPtr())->_size; // ONLY IN HEADERS, PROTECTED by RUNTIME
TRI_ASSERT(size > 0);
TRI_ASSERT(_begin != header);
TRI_ASSERT(_end != header);
this->move(header, old);
_nrLinked++;
_totalSize += TRI_DF_ALIGN_BLOCK(size);
TRI_ASSERT(_totalSize > 0);
TRI_ASSERT(header->_prev != header);
TRI_ASSERT(header->_next != header);
}
static bool CheckCollection (TRI_collection_t* collection) {
TRI_datafile_t* datafile;
TRI_vector_pointer_t all;
TRI_vector_pointer_t compactors;
TRI_vector_pointer_t datafiles;
TRI_vector_pointer_t journals;
TRI_vector_pointer_t sealed;
TRI_vector_string_t files;
bool stop;
regex_t re;
size_t i;
size_t n;
stop = false;
// check files within the directory
files = TRI_FilesDirectory(collection->_directory);
n = files._length;
regcomp(&re, "^(journal|datafile|index|compactor)-([0-9][0-9]*)\\.(db|json)$", REG_EXTENDED);
TRI_InitVectorPointer(&journals, TRI_UNKNOWN_MEM_ZONE);
TRI_InitVectorPointer(&compactors, TRI_UNKNOWN_MEM_ZONE);
TRI_InitVectorPointer(&datafiles, TRI_UNKNOWN_MEM_ZONE);
TRI_InitVectorPointer(&sealed, TRI_UNKNOWN_MEM_ZONE);
TRI_InitVectorPointer(&all, TRI_UNKNOWN_MEM_ZONE);
for (i = 0; i < n; ++i) {
char const* file = files._buffer[i];
regmatch_t matches[4];
if (regexec(&re, file, sizeof(matches) / sizeof(matches[0]), matches, 0) == 0) {
char const* first = file + matches[1].rm_so;
size_t firstLen = matches[1].rm_eo - matches[1].rm_so;
char const* third = file + matches[3].rm_so;
size_t thirdLen = matches[3].rm_eo - matches[3].rm_so;
// .............................................................................
// file is an index, just store the filename
// .............................................................................
if (TRI_EqualString2("index", first, firstLen) && TRI_EqualString2("json", third, thirdLen)) {
char* filename;
filename = TRI_Concatenate2File(collection->_directory, file);
TRI_PushBackVectorString(&collection->_indexFiles, filename);
}
// .............................................................................
// file is a journal or datafile, open the datafile
// .............................................................................
else if (TRI_EqualString2("db", third, thirdLen)) {
char* filename;
char* ptr;
TRI_col_header_marker_t* cm;
filename = TRI_Concatenate2File(collection->_directory, file);
datafile = TRI_OpenDatafile(filename);
if (datafile == NULL) {
collection->_lastError = TRI_errno();
stop = true;
LOG_ERROR("cannot open datafile '%s': %s", filename, TRI_last_error());
break;
}
TRI_PushBackVectorPointer(&all, datafile);
// check the document header
ptr = datafile->_data;
ptr += TRI_DF_ALIGN_BLOCK(sizeof(TRI_df_header_marker_t));
cm = (TRI_col_header_marker_t*) ptr;
if (cm->base._type != TRI_COL_MARKER_HEADER) {
LOG_ERROR("collection header mismatch in file '%s', expected TRI_COL_MARKER_HEADER, found %lu",
filename,
(unsigned long) cm->base._type);
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
stop = true;
break;
}
if (cm->_cid != collection->_info._cid) {
LOG_ERROR("collection identifier mismatch, expected %llu, found %llu",
(unsigned long long) collection->_info._cid,
(unsigned long long) cm->_cid);
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
stop = true;
break;
}
// file is a journal
if (TRI_EqualString2("journal", first, firstLen)) {
if (datafile->_isSealed) {
LOG_WARNING("strange, journal '%s' is already sealed; must be a left over; will use it as datafile", filename);
TRI_PushBackVectorPointer(&sealed, datafile);
}
else {
TRI_PushBackVectorPointer(&journals, datafile);
}
}
// file is a compactor file
else if (TRI_EqualString2("compactor", first, firstLen)) {
if (datafile->_isSealed) {
LOG_WARNING("strange, compactor journal '%s' is already sealed; must be a left over; will use it as datafile", filename);
TRI_PushBackVectorPointer(&sealed, datafile);
}
else {
TRI_PushBackVectorPointer(&compactors, datafile);
}
}
// file is a datafile
else if (TRI_EqualString2("datafile", first, firstLen)) {
if (! datafile->_isSealed) {
LOG_ERROR("datafile '%s' is not sealed, this should never happen", filename);
collection->_lastError = TRI_set_errno(TRI_ERROR_ARANGO_CORRUPTED_DATAFILE);
stop = true;
break;
}
else {
TRI_PushBackVectorPointer(&datafiles, datafile);
}
}
else {
LOG_ERROR("unknown datafile '%s'", file);
}
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
}
else {
LOG_ERROR("unknown datafile '%s'", file);
}
}
}
TRI_DestroyVectorString(&files);
regfree(&re);
// convert the sealed journals into datafiles
if (! stop) {
n = sealed._length;
for (i = 0; i < n; ++i) {
char* number;
char* dname;
char* filename;
bool ok;
datafile = sealed._buffer[i];
number = TRI_StringUInt64(datafile->_fid);
dname = TRI_Concatenate3String("datafile-", number, ".db");
filename = TRI_Concatenate2File(collection->_directory, dname);
TRI_FreeString(TRI_CORE_MEM_ZONE, dname);
TRI_FreeString(TRI_CORE_MEM_ZONE, number);
ok = TRI_RenameDatafile(datafile, filename);
if (ok) {
TRI_PushBackVectorPointer(&datafiles, datafile);
LOG_DEBUG("renamed sealed journal to '%s'", filename);
}
else {
collection->_lastError = datafile->_lastError;
stop = true;
LOG_ERROR("cannot rename sealed log-file to %s, this should not happen: %s", filename, TRI_last_error());
break;
}
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
}
}
TRI_DestroyVectorPointer(&sealed);
// stop if necessary
if (stop) {
n = all._length;
for (i = 0; i < n; ++i) {
datafile = all._buffer[i];
LOG_TRACE("closing datafile '%s'", datafile->_filename);
TRI_CloseDatafile(datafile);
TRI_FreeDatafile(datafile);
}
TRI_DestroyVectorPointer(&all);
TRI_DestroyVectorPointer(&datafiles);
return false;
}
TRI_DestroyVectorPointer(&all);
// add the datafiles and journals
collection->_datafiles = datafiles;
collection->_journals = journals;
collection->_compactors = compactors;
return true;
}
TRI_doc_mptr_t* TRI_headers_t::request (size_t size) {
TRI_doc_mptr_t* header;
TRI_ASSERT(size > 0);
if (_freelist == nullptr) {
size_t blockSize = GetBlockSize(_blocks._length);
TRI_ASSERT(blockSize > 0);
TRI_doc_mptr_t* begin;
try {
begin = new TRI_doc_mptr_t[blockSize];
}
catch (std::exception&) {
begin = nullptr;
}
// out of memory
if (begin == nullptr) {
TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY);
return nullptr;
}
TRI_doc_mptr_t* ptr = begin + (blockSize - 1);
header = nullptr;
for (; begin <= ptr; ptr--) {
ptr->setDataPtr(header); // ONLY IN HEADERS
header = ptr;
}
_freelist = header;
TRI_PushBackVectorPointer(&_blocks, begin);
}
TRI_ASSERT(_freelist != nullptr);
TRI_doc_mptr_t* result = const_cast<TRI_doc_mptr_t*>(_freelist);
TRI_ASSERT(result != nullptr);
_freelist = static_cast<TRI_doc_mptr_t const*>(result->getDataPtr()); // ONLY IN HEADERS, PROTECTED by RUNTIME
result->setDataPtr(nullptr); // ONLY IN HEADERS
// put new header at the end of the list
if (_begin == nullptr) {
// list of headers is empty
TRI_ASSERT(_nrLinked == 0);
TRI_ASSERT(_totalSize == 0);
_begin = result;
_end = result;
result->_prev = nullptr;
result->_next = nullptr;
}
else {
// list is not empty
TRI_ASSERT(_nrLinked > 0);
TRI_ASSERT(_totalSize > 0);
TRI_ASSERT(_nrAllocated > 0);
TRI_ASSERT(_begin != nullptr);
TRI_ASSERT(_end != nullptr);
_end->_next = result;
result->_prev = _end;
result->_next = nullptr;
_end = result;
}
_nrAllocated++;
_nrLinked++;
_totalSize += (int64_t) TRI_DF_ALIGN_BLOCK(size);
return result;
}
void TRI_headers_t::move (TRI_doc_mptr_t* header,
TRI_doc_mptr_t* old) {
if (header == nullptr) {
return;
}
TRI_ASSERT(_nrAllocated > 0);
TRI_ASSERT(header->_prev != header);
TRI_ASSERT(header->_next != header);
TRI_ASSERT(header->getDataPtr() != nullptr); // ONLY IN HEADERS, PROTECTED by RUNTIME
TRI_ASSERT(((TRI_df_marker_t*) header->getDataPtr())->_size > 0); // ONLY IN HEADERS, PROTECTED by RUNTIME
TRI_ASSERT(old != nullptr);
TRI_ASSERT(old->getDataPtr() != nullptr); // ONLY IN HEADERS, PROTECTED by RUNTIME
int64_t newSize = (int64_t) (((TRI_df_marker_t*) header->getDataPtr())->_size); // ONLY IN HEADERS, PROTECTED by RUNTIME
int64_t oldSize = (int64_t) (((TRI_df_marker_t*) old->getDataPtr())->_size); // ONLY IN HEADERS, PROTECTED by RUNTIME
// Please note the following: This operation is only used to revert an
// update operation. The "new" document is removed again and the "old"
// one is used once more. Therefore, the signs in the following statement
// are actually OK:
_totalSize -= ( TRI_DF_ALIGN_BLOCK(newSize)
- TRI_DF_ALIGN_BLOCK(oldSize));
// adjust list start and end pointers
if (old->_prev == nullptr) {
_begin = header;
}
else if (_begin == header) {
if (old->_prev != nullptr) {
_begin = old->_prev;
}
}
if (old->_next == nullptr) {
_end = header;
}
else if (_end == header) {
if (old->_next != nullptr) {
_end = old->_next;
}
}
if (header->_prev != nullptr) {
header->_prev->_next = header->_next;
}
if (header->_next != nullptr) {
header->_next->_prev = header->_prev;
}
if (old->_prev != nullptr) {
old->_prev->_next = header;
}
if (old->_next != nullptr) {
old->_next->_prev = header;
}
header->_prev = old->_prev;
header->_next = old->_next;
TRI_ASSERT(_begin != nullptr);
TRI_ASSERT(_end != nullptr);
TRI_ASSERT(header->_prev != header);
TRI_ASSERT(header->_next != header);
}