int TRI_DeflateStringBuffer (TRI_string_buffer_t* self,
size_t bufferSize) {
TRI_string_buffer_t deflated;
const char* ptr;
const char* end;
char* buffer;
int res;
z_stream strm;
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
// initialise deflate procedure
res = deflateInit(&strm, Z_DEFAULT_COMPRESSION);
if (res != Z_OK) {
return TRI_ERROR_OUT_OF_MEMORY;
}
buffer = (char*) TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, bufferSize, false);
if (buffer == NULL) {
(void) deflateEnd(&strm);
return TRI_ERROR_OUT_OF_MEMORY;
}
// we'll use this buffer for the output
TRI_InitStringBuffer(&deflated, TRI_UNKNOWN_MEM_ZONE);
ptr = TRI_BeginStringBuffer(self);
end = ptr + TRI_LengthStringBuffer(self);
while (ptr < end) {
int flush;
strm.next_in = (unsigned char*) ptr;
if (end - ptr > (int) bufferSize) {
strm.avail_in = (int) bufferSize;
flush = Z_NO_FLUSH;
}
else {
strm.avail_in = (uInt) (end - ptr);
flush = Z_FINISH;
}
ptr += strm.avail_in;
do {
strm.avail_out = (int) bufferSize;
strm.next_out = (unsigned char*) buffer;
res = deflate(&strm, flush);
if (res == Z_STREAM_ERROR) {
(void) deflateEnd(&strm);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, buffer);
TRI_DestroyStringBuffer(&deflated);
return TRI_ERROR_INTERNAL;
}
if (TRI_AppendString2StringBuffer(&deflated, (char*) buffer, bufferSize - strm.avail_out) != TRI_ERROR_NO_ERROR) {
(void) deflateEnd(&strm);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, buffer);
TRI_DestroyStringBuffer(&deflated);
return TRI_ERROR_OUT_OF_MEMORY;
}
}
while (strm.avail_out == 0);
}
// deflate successful
(void) deflateEnd(&strm);
TRI_SwapStringBuffer(self, &deflated);
TRI_DestroyStringBuffer(&deflated);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, buffer);
return TRI_ERROR_NO_ERROR;
}
void TRI_DestroyStringBuffer (TRI_string_buffer_t * self) {
if (self->_buffer != NULL) {
TRI_Free(self->_memoryZone, self->_buffer);
}
}
void TRI_FreeStringBuffer (TRI_memory_zone_t* zone, TRI_string_buffer_t * self) {
TRI_DestroyStringBuffer(self);
TRI_Free(zone, self);
}
int extendColumns (TRI_bitarray_t* ba, size_t newBlocks) {
bool ok = true;
size_t j;
// ............................................................................
// allocate memory for the new columns
// ............................................................................
char* newColumns = static_cast<char*>(TRI_Allocate(ba->_memoryZone, sizeof(BitColumn_t) * ba->_numColumns, true));
if (newColumns == NULL) {
return TRI_ERROR_OUT_OF_MEMORY;
}
// ............................................................................
// allocate space for each column
// ............................................................................
for (j = 0; j < ba->_numColumns; ++j) {
BitColumn_t* newColumn;
newColumn = (BitColumn_t*)(newColumns + (sizeof(BitColumn_t) * j));
newColumn->_column = static_cast<bit_column_int_t*>(TRI_Allocate(ba->_memoryZone, sizeof(bit_column_int_t) * newBlocks, true));
if (newColumn->_column == NULL) {
ok = false;
break;
}
}
// ............................................................................
// if memory allocation failed, undo allocation and return
// ............................................................................
if (!ok) { // memory allocation failure
for (j = 0; j < ba->_numColumns; ++j) {
BitColumn_t* newColumn;
newColumn = (BitColumn_t*)(newColumns + (sizeof(BitColumn_t) * j));
if (newColumn->_column != NULL) {
TRI_Free(ba->_memoryZone,newColumn->_column);
}
}
TRI_Free(ba->_memoryZone, newColumns);
return TRI_ERROR_OUT_OF_MEMORY;
}
// ............................................................................
// copy the old columns into the new and free that column
// ............................................................................
for (j = 0; j < ba->_numColumns; ++j) {
BitColumn_t* oldColumn;
BitColumn_t* newColumn;
oldColumn = (BitColumn_t*)(ba->_columns + (sizeof(BitColumn_t) * j));
newColumn = (BitColumn_t*)(newColumns + (sizeof(BitColumn_t) * j));
if (oldColumn->_column == NULL) {
TRI_ASSERT(false);
continue;
}
memcpy(newColumn->_column, oldColumn->_column, sizeof(bit_column_int_t) * ba->_numBlocksInColumn);
TRI_Free(ba->_memoryZone,oldColumn->_column);
}
TRI_Free(ba->_memoryZone, ba->_columns);
ba->_columns = newColumns;
ba->_numBlocksInColumn = newBlocks;
return TRI_ERROR_NO_ERROR;
}
void TRI_ParseQueryFreeString (char* string) {
if (string) {
TRI_Free(string);
string = NULL;
}
}
void TRI_FreeArrayShaper (TRI_memory_zone_t* zone, TRI_shaper_t* shaper) {
TRI_DestroyArrayShaper(shaper);
TRI_Free(zone, shaper);
}
void TRI_CleanupShadowData (TRI_shadow_store_t* const store,
const double maxAge,
const bool force) {
double compareStamp = TRI_microtime() - maxAge; // age must be specified in secs
size_t deleteCount = 0;
// we need an exclusive lock on the index
TRI_LockMutex(&store->_lock);
if (store->_ids._nrUsed == 0) {
// store is empty, nothing to do!
TRI_UnlockMutex(&store->_lock);
return;
}
LOG_TRACE("cleaning shadows. in store: %ld", (unsigned long) store->_ids._nrUsed);
// loop until there's nothing to delete or
// we have deleted SHADOW_MAX_DELETE elements
while (deleteCount++ < SHADOW_MAX_DELETE || force) {
bool deleted = false;
size_t i;
for (i = 0; i < store->_ids._nrAlloc; i++) {
// enum all shadows
TRI_shadow_t* shadow = (TRI_shadow_t*) store->_ids._table[i];
if (shadow == NULL) {
continue;
}
// check if shadow is unused and expired
if (shadow->_rc < 1 || force) {
if (shadow->_type == SHADOW_TRANSIENT ||
shadow->_timestamp < compareStamp ||
shadow->_deleted ||
force) {
LOG_TRACE("cleaning shadow %p, id: %llu, rc: %d, expired: %d, deleted: %d",
shadow,
(unsigned long long) shadow->_id,
(int) shadow->_rc,
(int) (shadow->_timestamp < compareStamp),
(int) shadow->_deleted);
TRI_RemoveKeyAssociativePointer(&store->_ids, &shadow->_id);
TRI_RemoveKeyAssociativePointer(&store->_pointers, shadow->_data);
store->destroyShadow(shadow->_data);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, shadow);
deleted = true;
// the remove might reposition elements in the container.
// therefore break here and start iteration anew
break;
}
}
}
if (! deleted) {
// we did not find anything to delete, so give up
break;
}
}
// release lock
TRI_UnlockMutex(&store->_lock);
}
void TRI_FreeAssociativePointer (TRI_associative_pointer_t* array) {
TRI_DestroyAssociativePointer(array);
TRI_Free(array);
}
void TRI_DestroyAssociativeSynced (TRI_associative_synced_t* array) {
TRI_Free(array->_table);
TRI_DestroyReadWriteLock(&array->_lock);
}
void TRI_FreeAssociativeArray (TRI_associative_array_t* array) {
TRI_DestroyAssociativeArray(array);
TRI_Free(array);
}
void TRI_DestroyAssociativePointer (TRI_associative_pointer_t* array) {
TRI_Free(array->_table);
}
void TRI_DestroyAssociativeArray (TRI_associative_array_t* array) {
TRI_Free(array->_table);
}
void TRI_FreeAssociativeSynced (TRI_associative_synced_t* array) {
TRI_DestroyAssociativeSynced(array);
TRI_Free(array);
}
int TRI_InitBitarray(TRI_bitarray_t** bitArray,
TRI_memory_zone_t* memoryZone,
size_t numArrays,
void* masterTable) {
MasterTable_t* mt;
size_t j;
bool ok;
// ...........................................................................
// The memory for the bitArray is allocated here. We expect a NULL pointer to
// be passed here.
// ...........................................................................
if (*bitArray != NULL) {
return TRI_ERROR_INTERNAL;
}
// ...........................................................................
// Allocate the necessary memory to store the bitArray structure.
// ...........................................................................
*bitArray = static_cast<TRI_bitarray_t*>(TRI_Allocate(memoryZone, sizeof(TRI_bitarray_t), false));
if (*bitArray == NULL) {
return TRI_ERROR_OUT_OF_MEMORY;
}
// ...........................................................................
// For now each set of bit arrays will create and use its own MasteTable.
// ...........................................................................
if (masterTable != NULL) {
TRI_ASSERT(false);
mt = (MasterTable_t*)(masterTable);
}
// ...........................................................................
// Create the new master table here
// ...........................................................................
else {
int result;
mt = NULL;
result = createMasterTable(&mt, memoryZone, false);
if (result != TRI_ERROR_NO_ERROR) {
TRI_Free(memoryZone, *bitArray);
*bitArray = NULL;
return result;
}
}
// ...........................................................................
// Check that we have a valid master table
// ...........................................................................
if (mt == NULL) {
if (*bitArray != NULL) {
TRI_Free(memoryZone, *bitArray);
}
*bitArray = NULL;
return TRI_ERROR_INTERNAL;
}
(*bitArray)->_masterTable = mt;
// ...........................................................................
// Store the number of columns in this set of bit arrays.
// ...........................................................................
(*bitArray)->_numColumns = numArrays;
// ...........................................................................
// Attempt to allocate memory to create the column structures
// ...........................................................................
(*bitArray)->_memoryZone = memoryZone;
(*bitArray)->_columns = static_cast<char*>(TRI_Allocate(memoryZone, sizeof(BitColumn_t) * numArrays, true));
if ((*bitArray)->_columns == NULL) {
TRI_Free(memoryZone,*bitArray);
*bitArray = NULL;
return TRI_ERROR_OUT_OF_MEMORY;
}
// ...........................................................................
// Create the bitarrays (the columns which will contain the bits)
// For each column we allocated the initialise size.
// ...........................................................................
ok = true;
for (j = 0; j < (*bitArray)->_numColumns; ++j) {
BitColumn_t* column;
column = (BitColumn_t*)((*bitArray)->_columns + (sizeof(BitColumn_t) * j));
column->_column = static_cast<bit_column_int_t*>(TRI_Allocate(memoryZone, sizeof(bit_column_int_t) * BITARRAY_INITIAL_NUMBER_OF_COLUMN_BLOCKS_SIZE, true));
if (column->_column == NULL) {
ok = false;
break;
}
}
if (! ok) {
// ...........................................................................
// Oops -- memory failure. Return all allocated memory and make a quick escape.
// ...........................................................................
for (j = 0; j < (*bitArray)->_numColumns; ++j) {
BitColumn_t* column;
column = (BitColumn_t*)((*bitArray)->_columns + (sizeof(BitColumn_t) * j));
if (column->_column != NULL) {
TRI_Free(memoryZone,column->_column);
}
}
TRI_Free(memoryZone, (*bitArray)->_columns);
TRI_Free(memoryZone, *bitArray);
*bitArray = NULL;
return TRI_ERROR_OUT_OF_MEMORY;
}
(*bitArray)->_numBlocksInColumn = BITARRAY_INITIAL_NUMBER_OF_COLUMN_BLOCKS_SIZE;
//(*bitArray)->_usedBitLength = 0;
(*bitArray)->_lastBlockUsed = 0;
// ...........................................................................
// Everything ok
// ...........................................................................
return TRI_ERROR_NO_ERROR;
}
void TRI_FreeStatementWalkerAql (TRI_aql_statement_walker_t* const walker) {
assert(walker);
TRI_DestroyVectorPointer(&walker->_currentScopes);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, walker);
}
void TRI_FreeJson (TRI_memory_zone_t* zone, TRI_json_t* object) {
TRI_DestroyJson(zone, object);
TRI_Free(zone, object);
}
static TRI_shape_pid_t FindNameAttributePath (TRI_shaper_t* shaper, char const* name) {
TRI_shape_aid_t* aids;
TRI_shape_path_t* result;
size_t count;
size_t len;
size_t total;
char* buffer;
char* end;
char* prev;
char* ptr;
void const* f;
void const* p;
p = TRI_LookupByKeyAssociativeSynced(&shaper->_attributePathsByName, name);
if (p != NULL) {
return ((TRI_shape_path_t const*) p)->_pid;
}
// create a attribute path
len = strlen(name);
// lock the index and check that the element is still missing
TRI_LockMutex(&shaper->_attributePathLock);
// if the element appeared, return the pid
p = TRI_LookupByKeyAssociativeSynced(&shaper->_attributePathsByName, name);
if (p != NULL) {
TRI_UnlockMutex(&shaper->_attributePathLock);
return ((TRI_shape_path_t const*) p)->_pid;
}
// split path into attribute pieces
count = 0;
aids = TRI_Allocate(shaper->_memoryZone, len * sizeof(TRI_shape_aid_t), false);
if (aids == NULL) {
return 0;
}
buffer = ptr = TRI_DuplicateString2Z(shaper->_memoryZone, name, len);
if (buffer == NULL) {
TRI_Free(shaper->_memoryZone, aids);
return 0;
}
end = buffer + len + 1;
prev = buffer;
for (; ptr < end; ++ptr) {
if (*ptr == '.' || *ptr == '\0') {
*ptr = '\0';
if (ptr != prev) {
aids[count++] = shaper->findAttributeName(shaper, prev);
}
prev = ptr + 1;
}
}
TRI_FreeString(shaper->_memoryZone, buffer);
// create element
total = sizeof(TRI_shape_path_t) + (len + 1) + (count * sizeof(TRI_shape_aid_t));
result = TRI_Allocate(shaper->_memoryZone, total, false);
if (result == NULL) {
return 0;
}
result->_pid = shaper->_nextPid++;
result->_nameLength = len + 1;
result->_aidLength = count;
memcpy(((char*) result) + sizeof(TRI_shape_path_t), aids, count * sizeof(TRI_shape_aid_t));
memcpy(((char*) result) + sizeof(TRI_shape_path_t) + count * sizeof(TRI_shape_aid_t), name, len + 1);
TRI_Free(shaper->_memoryZone, aids);
f = TRI_InsertKeyAssociativeSynced(&shaper->_attributePathsByName, name, result);
assert(f == NULL);
f = TRI_InsertKeyAssociativeSynced(&shaper->_attributePathsByPid, &result->_pid, result);
assert(f == NULL);
// return pid
TRI_UnlockMutex(&shaper->_attributePathLock);
return result->_pid;
}
static int StringifyJson (TRI_memory_zone_t* zone,
TRI_string_buffer_t* buffer,
TRI_json_t const* object,
bool braces) {
size_t n;
size_t i;
size_t outLength;
char* ptr;
int res;
switch (object->_type) {
case TRI_JSON_UNUSED:
break;
case TRI_JSON_NULL:
res = TRI_AppendStringStringBuffer(buffer, "null");
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
break;
case TRI_JSON_BOOLEAN:
if (object->_value._boolean) {
res = TRI_AppendStringStringBuffer(buffer, "true");
}
else {
res = TRI_AppendStringStringBuffer(buffer, "false");
}
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
break;
case TRI_JSON_NUMBER:
res = TRI_AppendDoubleStringBuffer(buffer, object->_value._number);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
break;
case TRI_JSON_STRING:
res = TRI_AppendStringStringBuffer(buffer, "\"");
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
ptr = TRI_EscapeUtf8StringZ(zone,
object->_value._string.data,
object->_value._string.length - 1,
false,
&outLength);
if (ptr == NULL) {
return TRI_ERROR_OUT_OF_MEMORY;
}
res = TRI_AppendString2StringBuffer(buffer, ptr, outLength);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
TRI_Free(zone, ptr);
res = TRI_AppendStringStringBuffer(buffer, "\"");
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
break;
case TRI_JSON_ARRAY:
if (braces) {
res = TRI_AppendStringStringBuffer(buffer, "{");
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
}
n = object->_value._objects._length;
for (i = 0; i < n; i += 2) {
if (0 < i) {
res = TRI_AppendStringStringBuffer(buffer, ",");
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
}
res = StringifyJson(zone, buffer, TRI_AtVector(&object->_value._objects, i), true);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
res = TRI_AppendCharStringBuffer(buffer, ':');
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
res = StringifyJson(zone, buffer, TRI_AtVector(&object->_value._objects, i + 1), true);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
}
if (braces) {
res = TRI_AppendStringStringBuffer(buffer, "}");
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
}
break;
case TRI_JSON_LIST:
if (braces) {
res = TRI_AppendStringStringBuffer(buffer, "[");
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
}
n = object->_value._objects._length;
for (i = 0; i < n; ++i) {
if (0 < i) {
res = TRI_AppendStringStringBuffer(buffer, ",");
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
}
res = StringifyJson(zone, buffer, TRI_AtVector(&object->_value._objects, i), true);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
}
if (braces) {
res = TRI_AppendStringStringBuffer(buffer, "]");
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
}
break;
}
return TRI_ERROR_NO_ERROR;
}
void TRI_FreeIndexAql (TRI_aql_index_t* const idx) {
assert(idx);
TRI_FreeVectorPointer(TRI_UNKNOWN_MEM_ZONE, idx->_fieldAccesses);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, idx);
}
void TRI_Insert3ArrayJson (TRI_memory_zone_t* zone, TRI_json_t* object, char const* name, TRI_json_t* subobject) {
TRI_Insert2ArrayJson(zone, object, name, subobject);
TRI_Free(zone, subobject);
}
static void FreeDatafileInfo (TRI_doc_datafile_info_t* dfi) {
TRI_Free(TRI_UNKNOWN_MEM_ZONE, dfi);
}
TRI_aql_context_t* TRI_CreateContextAql (TRI_vocbase_t* vocbase,
const char* const query,
const size_t queryLength,
bool isCoordinator,
TRI_json_t* userOptions) {
TRI_aql_context_t* context;
int res;
TRI_ASSERT(vocbase != NULL);
TRI_ASSERT(query != NULL);
LOG_TRACE("creating context");
context = (TRI_aql_context_t*) TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_aql_context_t), false);
if (context == NULL) {
return NULL;
}
context->_type = TRI_AQL_QUERY_READ;
context->_vocbase = vocbase;
context->_userOptions = userOptions;
context->_writeOptions = NULL;
context->_writeCollection = NULL;
context->_variableIndex = 0;
context->_scopeIndex = 0;
context->_subQueries = 0;
// actual bind parameter values
res = TRI_InitAssociativePointer(&context->_parameters._values,
TRI_UNKNOWN_MEM_ZONE,
&TRI_HashStringKeyAssociativePointer,
&TRI_HashBindParameterAql,
&TRI_EqualBindParameterAql,
0);
if (res != TRI_ERROR_NO_ERROR) {
TRI_Free(TRI_UNKNOWN_MEM_ZONE, context);
return NULL;
}
// bind parameter names used in the query
res = TRI_InitAssociativePointer(&context->_parameters._names,
TRI_UNKNOWN_MEM_ZONE,
&TRI_HashStringKeyAssociativePointer,
&TRI_HashStringKeyAssociativePointer,
&TRI_EqualStringKeyAssociativePointer,
0);
if (res != TRI_ERROR_NO_ERROR) {
TRI_DestroyAssociativePointer(&context->_parameters._values);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, context);
return NULL;
}
// collections
res = TRI_InitAssociativePointer(&context->_collectionNames,
TRI_UNKNOWN_MEM_ZONE,
&TRI_HashStringKeyAssociativePointer,
&TRI_HashStringKeyAssociativePointer,
&TRI_EqualStringKeyAssociativePointer,
0);
if (res != TRI_ERROR_NO_ERROR) {
TRI_DestroyAssociativePointer(&context->_parameters._names);
TRI_DestroyAssociativePointer(&context->_parameters._values);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, context);
return NULL;
}
TRI_InitVectorPointer2(&context->_memory._nodes, TRI_UNKNOWN_MEM_ZONE, 16);
TRI_InitVectorPointer2(&context->_memory._strings, TRI_UNKNOWN_MEM_ZONE, 16);
TRI_InitVectorPointer(&context->_collections, TRI_UNKNOWN_MEM_ZONE);
TRI_InitErrorAql(&context->_error);
context->_parser = NULL;
context->_statements = NULL;
context->_query = query;
TRI_InitScopesAql(context);
context->_parser = TRI_CreateParserAql(context->_query, queryLength);
if (context->_parser == NULL) {
// could not create the parser
TRI_FreeContextAql(context);
return NULL;
}
if (! TRI_InitParserAql(context)) {
// could not initialise the lexer
TRI_FreeContextAql(context);
return NULL;
}
context->_statements = TRI_CreateStatementListAql();
if (context->_statements == NULL) {
// could not create statement list
TRI_FreeContextAql(context);
return NULL;
}
ProcessOptions(context);
context->_isCoordinator = isCoordinator;
return context;
}