static TRI_datafile_t* SelectJournal (TRI_shape_collection_t* collection,
TRI_voc_size_t size,
TRI_df_marker_t** result) {
TRI_datafile_t* datafile;
bool ok;
int res;
size_t n;
// need to create a new journal?
n = collection->base._journals._length;
if (n == 0) {
ok = CreateJournal(collection);
n = collection->base._journals._length;
if (! ok || n == 0) {
return NULL;
}
}
// select first datafile
datafile = collection->base._journals._buffer[0];
// try to reserve space
res = TRI_ReserveElementDatafile(datafile, size, result);
while (res == TRI_ERROR_ARANGO_DATAFILE_FULL) {
ok = CloseJournal(collection, datafile);
if (! ok) {
return NULL;
}
ok = CreateJournal(collection);
n = collection->base._journals._length;
if (! ok || n == 0) {
return NULL;
}
datafile = collection->base._journals._buffer[0];
res = TRI_ReserveElementDatafile(datafile, size, result);
}
if (res != TRI_ERROR_NO_ERROR) {
collection->base._state = TRI_COL_STATE_WRITE_ERROR;
return NULL;
}
// we got enough space
return datafile;
}
static int CopyMarker (TRI_document_collection_t* document,
TRI_datafile_t* compactor,
TRI_df_marker_t const* marker,
TRI_df_marker_t** result) {
int res;
res = TRI_ReserveElementDatafile(compactor, marker->_size, result, 0);
if (res != TRI_ERROR_NO_ERROR) {
document->base.base._lastError = TRI_set_errno(TRI_ERROR_ARANGO_NO_JOURNAL);
return TRI_ERROR_ARANGO_NO_JOURNAL;
}
return TRI_WriteElementDatafile(compactor, *result, marker, marker->_size, false);
}
static TRI_datafile_t* SelectCompactor (TRI_sim_collection_t* sim,
TRI_voc_size_t size,
TRI_df_marker_t** result) {
TRI_datafile_t* datafile;
int res;
size_t i;
size_t n;
TRI_LOCK_JOURNAL_ENTRIES_SIM_COLLECTION(sim);
while (true) {
n = sim->base.base._compactors._length;
for (i = 0; i < n; ++i) {
// select datafile
datafile = sim->base.base._compactors._buffer[i];
// try to reserve space
res = TRI_ReserveElementDatafile(datafile, size, result);
// in case of full datafile, try next
if (res == TRI_ERROR_NO_ERROR) {
TRI_UNLOCK_JOURNAL_ENTRIES_SIM_COLLECTION(sim);
return datafile;
}
else if (res != TRI_ERROR_ARANGO_DATAFILE_FULL) {
TRI_UNLOCK_JOURNAL_ENTRIES_SIM_COLLECTION(sim);
return NULL;
}
}
TRI_WAIT_JOURNAL_ENTRIES_SIM_COLLECTION(sim);
}
TRI_UNLOCK_JOURNAL_ENTRIES_SIM_COLLECTION(sim);
}
static TRI_datafile_t* CreateJournal (TRI_primary_collection_t* primary, bool compactor) {
TRI_col_header_marker_t cm;
TRI_collection_t* collection;
TRI_datafile_t* journal;
TRI_df_marker_t* position;
int res;
char* filename;
collection = &primary->base;
if (collection->_info._isVolatile) {
// in-memory collection
filename = NULL;
}
else {
char* jname;
char* number;
// construct a suitable filename
number = TRI_StringUInt64(TRI_NewTickVocBase());
if (compactor) {
jname = TRI_Concatenate3String("journal-", number, ".db");
}
else {
jname = TRI_Concatenate3String("compactor-", number, ".db");
}
filename = TRI_Concatenate2File(collection->_directory, jname);
TRI_FreeString(TRI_CORE_MEM_ZONE, number);
TRI_FreeString(TRI_CORE_MEM_ZONE, jname);
}
// create journal file
journal = TRI_CreateDatafile(filename, collection->_info._maximalSize);
if (filename != NULL) {
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
}
if (journal == NULL) {
if (TRI_errno() == TRI_ERROR_OUT_OF_MEMORY_MMAP) {
collection->_lastError = TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY_MMAP);
collection->_state = TRI_COL_STATE_READ;
}
else {
collection->_lastError = TRI_set_errno(TRI_ERROR_ARANGO_NO_JOURNAL);
collection->_state = TRI_COL_STATE_WRITE_ERROR;
}
return NULL;
}
LOG_TRACE("created a new primary journal '%s'", journal->getName(journal));
if (journal->isPhysical(journal)) {
char* jname;
char* number;
bool ok;
// and use the correct name
number = TRI_StringUInt64(journal->_fid);
if (compactor) {
jname = TRI_Concatenate3String("compactor-", number, ".db");
}
else {
jname = TRI_Concatenate3String("journal-", number, ".db");
}
filename = TRI_Concatenate2File(collection->_directory, jname);
TRI_FreeString(TRI_CORE_MEM_ZONE, number);
TRI_FreeString(TRI_CORE_MEM_ZONE, jname);
ok = TRI_RenameDatafile(journal, filename);
if (! ok) {
LOG_WARNING("failed to rename the journal to '%s': %s", filename, TRI_last_error());
}
else {
LOG_TRACE("renamed journal to '%s'", filename);
}
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
}
// create a collection header
res = TRI_ReserveElementDatafile(journal, sizeof(TRI_col_header_marker_t), &position);
if (res != TRI_ERROR_NO_ERROR) {
collection->_lastError = journal->_lastError;
LOG_ERROR("cannot create document header in journal '%s': %s", journal->getName(journal), TRI_last_error());
TRI_FreeDatafile(journal);
return NULL;
}
memset(&cm, 0, sizeof(cm));
cm.base._size = sizeof(TRI_col_header_marker_t);
cm.base._type = TRI_COL_MARKER_HEADER;
cm.base._tick = TRI_NewTickVocBase();
cm._cid = collection->_info._cid;
TRI_FillCrcMarkerDatafile(journal, &cm.base, sizeof(cm), 0, 0, 0, 0);
res = TRI_WriteElementDatafile(journal, position, &cm.base, sizeof(cm), 0, 0, 0, 0, true);
if (res != TRI_ERROR_NO_ERROR) {
collection->_lastError = journal->_lastError;
LOG_ERROR("cannot create document header in journal '%s': %s", journal->getName(journal), TRI_last_error());
TRI_FreeDatafile(journal);
return NULL;
}
// that's it
if (compactor) {
TRI_PushBackVectorPointer(&collection->_compactors, journal);
}
else {
TRI_PushBackVectorPointer(&collection->_journals, journal);
}
return journal;
}
static TRI_datafile_t* CreateJournal (TRI_primary_collection_t* primary,
TRI_voc_size_t maximalSize) {
TRI_col_header_marker_t cm;
TRI_collection_t* collection;
TRI_datafile_t* journal;
TRI_df_marker_t* position;
TRI_voc_fid_t fid;
int res;
collection = &primary->base;
fid = (TRI_voc_fid_t) TRI_NewTickServer();
if (collection->_info._isVolatile) {
// in-memory collection
journal = TRI_CreateDatafile(NULL, fid, maximalSize);
}
else {
char* jname;
char* number;
char* filename;
// construct a suitable filename (which is temporary at the beginning)
number = TRI_StringUInt64(fid);
jname = TRI_Concatenate3String("temp-", number, ".db");
filename = TRI_Concatenate2File(collection->_directory, jname);
TRI_FreeString(TRI_CORE_MEM_ZONE, number);
TRI_FreeString(TRI_CORE_MEM_ZONE, jname);
journal = TRI_CreateDatafile(filename, fid, maximalSize);
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
}
if (journal == NULL) {
if (TRI_errno() == TRI_ERROR_OUT_OF_MEMORY_MMAP) {
collection->_lastError = TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY_MMAP);
collection->_state = TRI_COL_STATE_READ;
}
else {
collection->_lastError = TRI_set_errno(TRI_ERROR_ARANGO_NO_JOURNAL);
collection->_state = TRI_COL_STATE_WRITE_ERROR;
}
return NULL;
}
LOG_TRACE("created new journal '%s'", journal->getName(journal));
// create a collection header, still in the temporary file
res = TRI_ReserveElementDatafile(journal, sizeof(TRI_col_header_marker_t), &position, maximalSize);
if (res != TRI_ERROR_NO_ERROR) {
collection->_lastError = journal->_lastError;
LOG_ERROR("cannot create document header in journal '%s': %s", journal->getName(journal), TRI_last_error());
TRI_FreeDatafile(journal);
return NULL;
}
TRI_InitMarker((char*) &cm, TRI_COL_MARKER_HEADER, sizeof(TRI_col_header_marker_t));
cm.base._tick = (TRI_voc_tick_t) fid;
cm._type = (TRI_col_type_t) collection->_info._type;
cm._cid = collection->_info._cid;
res = TRI_WriteCrcElementDatafile(journal, position, &cm.base, sizeof(cm), true);
if (res != TRI_ERROR_NO_ERROR) {
collection->_lastError = journal->_lastError;
LOG_ERROR("cannot create document header in journal '%s': %s", journal->getName(journal), TRI_last_error());
TRI_FreeDatafile(journal);
return NULL;
}
assert(fid == journal->_fid);
// if a physical file, we can rename it from the temporary name to the correct name
if (journal->isPhysical(journal)) {
char* jname;
char* number;
char* filename;
bool ok;
// and use the correct name
number = TRI_StringUInt64(journal->_fid);
jname = TRI_Concatenate3String("journal-", number, ".db");
filename = TRI_Concatenate2File(collection->_directory, jname);
TRI_FreeString(TRI_CORE_MEM_ZONE, number);
TRI_FreeString(TRI_CORE_MEM_ZONE, jname);
ok = TRI_RenameDatafile(journal, filename);
if (! ok) {
LOG_ERROR("failed to rename the journal to '%s': %s", filename, TRI_last_error());
TRI_FreeDatafile(journal);
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
return NULL;
}
else {
LOG_TRACE("renamed journal from %s to '%s'", journal->getName(journal), filename);
}
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
}
TRI_PushBackVectorPointer(&collection->_journals, journal);
return journal;
}
static TRI_datafile_t* CreateCompactor (TRI_primary_collection_t* primary,
TRI_voc_fid_t fid,
TRI_voc_size_t maximalSize) {
TRI_col_header_marker_t cm;
TRI_collection_t* collection;
TRI_datafile_t* journal;
TRI_df_marker_t* position;
int res;
collection = &primary->base;
if (collection->_info._isVolatile) {
// in-memory collection
journal = TRI_CreateDatafile(NULL, fid, maximalSize);
}
else {
char* jname;
char* number;
char* filename;
number = TRI_StringUInt64(fid);
jname = TRI_Concatenate3String("compaction-", number, ".db");
filename = TRI_Concatenate2File(collection->_directory, jname);
TRI_FreeString(TRI_CORE_MEM_ZONE, number);
TRI_FreeString(TRI_CORE_MEM_ZONE, jname);
if (TRI_ExistsFile(filename)) {
// remove any existing temporary file first
TRI_UnlinkFile(filename);
}
journal = TRI_CreateDatafile(filename, fid, maximalSize);
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
}
if (journal == NULL) {
if (TRI_errno() == TRI_ERROR_OUT_OF_MEMORY_MMAP) {
collection->_lastError = TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY_MMAP);
collection->_state = TRI_COL_STATE_READ;
}
else {
collection->_lastError = TRI_set_errno(TRI_ERROR_ARANGO_NO_JOURNAL);
collection->_state = TRI_COL_STATE_WRITE_ERROR;
}
return NULL;
}
LOG_TRACE("created new compactor '%s'", journal->getName(journal));
// create a collection header, still in the temporary file
res = TRI_ReserveElementDatafile(journal, sizeof(TRI_col_header_marker_t), &position, maximalSize);
if (res != TRI_ERROR_NO_ERROR) {
collection->_lastError = journal->_lastError;
LOG_ERROR("cannot create document header in compactor '%s': %s", journal->getName(journal), TRI_last_error());
TRI_FreeDatafile(journal);
return NULL;
}
TRI_InitMarker((char*) &cm, TRI_COL_MARKER_HEADER, sizeof(TRI_col_header_marker_t));
cm.base._tick = (TRI_voc_tick_t) fid;
cm._type = (TRI_col_type_t) collection->_info._type;
cm._cid = collection->_info._cid;
res = TRI_WriteCrcElementDatafile(journal, position, &cm.base, sizeof(cm), false);
if (res != TRI_ERROR_NO_ERROR) {
collection->_lastError = journal->_lastError;
LOG_ERROR("cannot create document header in compactor '%s': %s", journal->getName(journal), TRI_last_error());
TRI_FreeDatafile(journal);
return NULL;
}
assert(fid == journal->_fid);
return journal;
}
static bool CreateJournal (TRI_shape_collection_t* collection) {
TRI_col_header_marker_t cm;
TRI_datafile_t* journal;
TRI_df_marker_t* position;
char* filename;
int res;
if (collection->base._info._isVolatile) {
// in memory collection
filename = NULL;
}
else {
char* jname;
char* number;
number = TRI_StringUInt32(TRI_NewTickVocBase());
if (! number) {
return false;
}
jname = TRI_Concatenate3String("journal-", number, ".db");
TRI_FreeString(TRI_CORE_MEM_ZONE, number);
filename = TRI_Concatenate2File(collection->base._directory, jname);
TRI_FreeString(TRI_CORE_MEM_ZONE, jname);
}
journal = TRI_CreateDatafile(filename, collection->base._info._maximalSize);
if (filename != NULL) {
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
}
// check that a journal was created
if (journal == NULL) {
if (TRI_errno() == TRI_ERROR_OUT_OF_MEMORY_MMAP) {
collection->base._lastError = TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY_MMAP);
collection->base._state = TRI_COL_STATE_READ;
}
else {
collection->base._lastError = TRI_set_errno(TRI_ERROR_ARANGO_NO_JOURNAL);
collection->base._state = TRI_COL_STATE_WRITE_ERROR;
}
return false;
}
LOG_TRACE("created a new shape journal '%s'", journal->getName(journal));
if (journal->isPhysical(journal)) {
char* jname;
char* number;
bool ok;
// and use the correct name
number = TRI_StringUInt32(journal->_fid);
jname = TRI_Concatenate3String("journal-", number, ".db");
filename = TRI_Concatenate2File(collection->base._directory, jname);
TRI_FreeString(TRI_CORE_MEM_ZONE, number);
TRI_FreeString(TRI_CORE_MEM_ZONE, jname);
ok = TRI_RenameDatafile(journal, filename);
if (! ok) {
// TODO: remove disastrous call to exit() here
LOG_FATAL_AND_EXIT("failed to rename the journal to '%s': %s", filename, TRI_last_error());
}
else {
LOG_TRACE("renamed journal to '%s'", filename);
}
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
}
// create a collection header
res = TRI_ReserveElementDatafile(journal, sizeof(TRI_col_header_marker_t), &position);
if (res != TRI_ERROR_NO_ERROR) {
LOG_ERROR("cannot create document header in journal '%s': %s",
journal->getName(journal),
TRI_last_error());
TRI_FreeDatafile(journal);
return false;
}
// create a header
memset(&cm, 0, sizeof(cm));
cm.base._size = sizeof(TRI_col_header_marker_t);
cm.base._type = TRI_COL_MARKER_HEADER;
cm.base._tick = TRI_NewTickVocBase();
cm._cid = collection->base._info._cid;
TRI_FillCrcMarkerDatafile(journal, &cm.base, sizeof(cm), 0, 0, 0, 0);
// on journal creation, always use waitForSync = true
res = TRI_WriteElementDatafile(journal, position, &cm.base, sizeof(cm), 0, 0, 0, 0, true);
if (res != TRI_ERROR_NO_ERROR) {
LOG_ERROR("cannot create document header in journal '%s': %s",
journal->getName(journal),
TRI_last_error());
TRI_FreeDatafile(journal);
return false;
}
// that's it
TRI_PushBackVectorPointer(&collection->base._journals, journal);
return true;
}
