static void CompactifyDatafiles (TRI_document_collection_t* document, TRI_vector_t const* compactions) { TRI_datafile_t* compactor; TRI_primary_collection_t* primary; compaction_initial_context_t initial; compaction_context_t context; size_t i, j, n; n = compactions->_length; assert(n > 0); initial = InitCompaction(document, compactions); if (initial._failed) { LOG_ERROR("could not create initialise compaction"); return; } LOG_TRACE("compactify called for collection '%llu' for %d datafiles of total size %llu", (unsigned long long) document->base.base._info._cid, (int) n, (unsigned long long) initial._targetSize); // now create a new compactor file // we are re-using the _fid of the first original datafile! compactor = CreateCompactor(document, initial._fid, initial._targetSize); if (compactor == NULL) { // some error occurred LOG_ERROR("could not create compactor file"); return; } LOG_DEBUG("created new compactor file '%s'", compactor->getName(compactor)); memset(&context._dfi, 0, sizeof(TRI_doc_datafile_info_t)); // these attributes remain the same for all datafiles we collect context._document = document; context._compactor = compactor; context._dfi._fid = compactor->_fid; // now compact all datafiles for (i = 0; i < n; ++i) { compaction_info_t* compaction; TRI_datafile_t* df; bool ok; compaction = TRI_AtVector(compactions, i); df = compaction->_datafile; LOG_DEBUG("compacting datafile '%s' into '%s', number: %d, keep deletions: %d", df->getName(df), compactor->getName(compactor), (int) i, (int) compaction->_keepDeletions); // if this is the first datafile in the list of datafiles, we can also collect // deletion markers context._keepDeletions = compaction->_keepDeletions; // run the actual compaction of a single datafile ok = TRI_IterateDatafile(df, Compactifier, &context, false, false); if (! ok) { LOG_WARNING("failed to compact datafile '%s'", df->getName(df)); // compactor file does not need to be removed now. will be removed on next startup // TODO: Remove return; } } // next file // locate the compactor // must acquire a write-lock as we're about to change the datafiles vector primary = &document->base; TRI_WRITE_LOCK_DATAFILES_DOC_COLLECTION(primary); if (! LocateDatafile(&primary->base._compactors, compactor->_fid, &j)) { // not found TRI_WRITE_UNLOCK_DATAFILES_DOC_COLLECTION(primary); LOG_ERROR("logic error in CompactifyDatafiles: could not find compactor"); return; } if (! TRI_CloseCompactorPrimaryCollection(primary, j)) { TRI_WRITE_UNLOCK_DATAFILES_DOC_COLLECTION(primary); LOG_ERROR("could not close compactor file"); // TODO: how do we recover from this state? return; } TRI_WRITE_UNLOCK_DATAFILES_DOC_COLLECTION(primary); if (context._dfi._numberAlive == 0 && context._dfi._numberDead == 0 && context._dfi._numberDeletion == 0 && context._dfi._numberTransaction == 0) { TRI_barrier_t* b; if (n > 1) { // create .dead files for all collected files for (i = 0; i < n; ++i) { compaction_info_t* compaction; TRI_datafile_t* datafile; compaction = TRI_AtVector(compactions, i); datafile = compaction->_datafile; if (datafile->isPhysical(datafile)) { char* filename = TRI_Concatenate2String(datafile->getName(datafile), ".dead"); if (filename != NULL) { TRI_WriteFile(filename, "", 0); TRI_FreeString(TRI_CORE_MEM_ZONE, filename); } } } } // compactor is fully empty. remove it RemoveCompactor(document, compactor); for (i = 0; i < n; ++i) { compaction_info_t* compaction; compaction = TRI_AtVector(compactions, i); // datafile is also empty after compaction and thus useless RemoveDatafile(document, compaction->_datafile); // add a deletion marker to the result set container b = TRI_CreateBarrierDropDatafile(&primary->_barrierList, compaction->_datafile, DropDatafileCallback, primary); if (b == NULL) { LOG_ERROR("out of memory when creating datafile-drop barrier"); } } } else { if (n > 1) { // create .dead files for all collected files but the first for (i = 1; i < n; ++i) { compaction_info_t* compaction; TRI_datafile_t* datafile; compaction = TRI_AtVector(compactions, i); datafile = compaction->_datafile; if (datafile->isPhysical(datafile)) { char* filename = TRI_Concatenate2String(datafile->getName(datafile), ".dead"); if (filename != NULL) { TRI_WriteFile(filename, "", 0); TRI_FreeString(TRI_CORE_MEM_ZONE, filename); } } } } for (i = 0; i < n; ++i) { TRI_barrier_t* b; compaction_info_t* compaction; compaction = TRI_AtVector(compactions, i); if (i == 0) { // add a rename marker void* copy; copy = TRI_Allocate(TRI_CORE_MEM_ZONE, sizeof(compaction_context_t), false); memcpy(copy, &context, sizeof(compaction_context_t)); b = TRI_CreateBarrierRenameDatafile(&primary->_barrierList, compaction->_datafile, RenameDatafileCallback, copy); if (b == NULL) { LOG_ERROR("out of memory when creating datafile-rename barrier"); TRI_Free(TRI_CORE_MEM_ZONE, copy); } } else { // datafile is empty after compaction and thus useless RemoveDatafile(document, compaction->_datafile); // add a drop datafile marker b = TRI_CreateBarrierDropDatafile(&primary->_barrierList, compaction->_datafile, DropDatafileCallback, primary); if (b == NULL) { LOG_ERROR("out of memory when creating datafile-drop barrier"); } } } } }
TRI_datafile_t* TRI_CreateCompactorPrimaryCollection (TRI_primary_collection_t* primary, TRI_voc_fid_t fid, TRI_voc_size_t maximalSize) { return CreateCompactor(primary, fid, maximalSize); }