TRI_shadow_t* TRI_StoreShadowData (TRI_shadow_store_t* const store,
const void* const data) {
TRI_shadow_t* shadow;
assert(store);
shadow = CreateShadow(data);
if (shadow) {
LOG_TRACE("storing shadow %p with data ptr %p and id %llu",
shadow,
shadow->_data,
(unsigned long long) shadow->_id);
TRI_LockMutex(&store->_lock);
if (TRI_InsertKeyAssociativePointer(&store->_ids, &shadow->_id, shadow, false)) {
// duplicate entry
LOG_WARNING("storing shadow failed");
TRI_UnlockMutex(&store->_lock);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, shadow);
return NULL;
}
TRI_InsertKeyAssociativePointer(&store->_pointers, data, shadow, false);
TRI_UnlockMutex(&store->_lock);
}
// might be NULL in case of OOM
return shadow;
}
bool TRI_KillExternalProcess (TRI_external_id_t pid) {
TRI_external_t* external = nullptr; // just to please the compiler
size_t i;
bool ok = true;
LOG_DEBUG("killing process: %d", (int) pid._pid);
TRI_LockMutex(&ExternalProcessesLock);
for (i = 0; i < ExternalProcesses._length; ++i) {
external = static_cast<TRI_external_t*>(TRI_AtVectorPointer(&ExternalProcesses, i));
if (external->_pid == pid._pid) {
break;
}
}
if (i == ExternalProcesses._length) {
TRI_UnlockMutex(&ExternalProcessesLock);
LOG_DEBUG("kill: process not found: %d", (int) pid._pid);
#ifndef _WIN32
// Kill just in case:
if (0 == kill(pid._pid, SIGTERM)) {
int count;
// Otherwise we just let it be.
for (count = 0; count < 10; count++) {
int loc;
pid_t p;
// And wait for it to avoid a zombie:
sleep(1);
p = waitpid(pid._pid, &loc, WUNTRACED | WNOHANG);
if (p == pid._pid) {
return true;
}
if (count == 8) {
kill(pid._pid, SIGKILL);
}
}
}
return false;
#else
return ourKillProcessPID(pid._pid);
#endif
}
if (external->_status == TRI_EXT_RUNNING ||
external->_status == TRI_EXT_STOPPED) {
ok = ourKillProcess(external);
}
TRI_RemoveVectorPointer(&ExternalProcesses, i);
TRI_UnlockMutex(&ExternalProcessesLock);
FreeExternal(external);
return ok;
}
void TRI_EndUsageDataShadowData (TRI_shadow_store_t* const store,
const void* const data) {
TRI_shadow_t* shadow;
assert(store);
TRI_LockMutex(&store->_lock);
shadow = (TRI_shadow_t*) TRI_LookupByKeyAssociativePointer(&store->_pointers, data);
if (shadow && ! shadow->_deleted) {
DecreaseRefCount(store, shadow); // this might delete the shadow
}
TRI_UnlockMutex(&store->_lock);
}
int TRI_WriteShapeCollection (TRI_shape_collection_t* collection,
TRI_df_marker_t* marker,
TRI_voc_size_t markerSize,
void const* body,
TRI_voc_size_t bodySize,
TRI_df_marker_t** result) {
TRI_datafile_t* journal;
int res;
// generate a new tick
marker->_tick = TRI_NewTickVocBase();
// lock the collection
TRI_LockMutex(&collection->_lock);
if (collection->base._state != TRI_COL_STATE_WRITE) {
if (collection->base._state == TRI_COL_STATE_READ) {
TRI_UnlockMutex(&collection->_lock);
return TRI_ERROR_ARANGO_READ_ONLY;
}
TRI_UnlockMutex(&collection->_lock);
return TRI_ERROR_ARANGO_ILLEGAL_STATE;
}
// find and select a journal
journal = SelectJournal(collection, markerSize + bodySize, result);
if (journal == NULL) {
TRI_UnlockMutex(&collection->_lock);
return TRI_ERROR_ARANGO_NO_JOURNAL;
}
// generate crc
TRI_FillCrcMarkerDatafile(journal, marker, markerSize, 0, 0, body, bodySize);
// and write marker and shape
res = WriteElement(collection, journal, *result, marker, markerSize, body, bodySize);
// release lock on collection
TRI_UnlockMutex(&collection->_lock);
return res;
}
void* TRI_BeginUsageIdShadowData (TRI_shadow_store_t* const store,
const TRI_shadow_id id) {
TRI_shadow_t* shadow;
assert(store);
TRI_LockMutex(&store->_lock);
shadow = (TRI_shadow_t*) TRI_LookupByKeyAssociativePointer(&store->_ids, (void const*) &id);
if (shadow && ! shadow->_deleted) {
IncreaseRefCount(store, shadow);
TRI_UnlockMutex(&store->_lock);
return shadow->_data;
}
TRI_UnlockMutex(&store->_lock);
return NULL;
}
bool TRI_DeleteIdShadowData (TRI_shadow_store_t* const store,
const TRI_shadow_id id) {
TRI_shadow_t* shadow;
bool found = false;
assert(store);
TRI_LockMutex(&store->_lock);
shadow = (TRI_shadow_t*) TRI_LookupByKeyAssociativePointer(&store->_ids, &id);
if (shadow && ! shadow->_deleted) {
DeleteShadow(store, shadow, false);
found = true;
}
TRI_UnlockMutex(&store->_lock);
return found;
}
bool TRI_PersistDataShadowData (TRI_shadow_store_t* const store,
const void* const data) {
TRI_shadow_t* shadow;
bool result = false;
assert(store);
TRI_LockMutex(&store->_lock);
shadow = (TRI_shadow_t*) TRI_LookupByKeyAssociativePointer(&store->_pointers, data);
if (shadow && ! shadow->_deleted) {
PersistShadow(shadow);
result = true;
}
TRI_UnlockMutex(&store->_lock);
return result;
}
TRI_shadow_id TRI_GetIdDataShadowData (TRI_shadow_store_t* const store,
const void* const data) {
TRI_shadow_t* shadow;
TRI_shadow_id id = 0;
assert(store);
if (data) {
TRI_LockMutex(&store->_lock);
shadow = (TRI_shadow_t*) TRI_LookupByKeyAssociativePointer(&store->_pointers, data);
if (shadow && ! shadow->_deleted) {
id = shadow->_id;
UpdateTimestampShadow(shadow);
}
TRI_UnlockMutex(&store->_lock);
}
return id;
}
void TRI_CreateExternalProcess (const char* executable,
const char** arguments,
size_t n,
bool usePipes,
TRI_external_id_t* pid) {
// create the external structure
TRI_external_t* external = static_cast<TRI_external_t*>(TRI_Allocate(TRI_CORE_MEM_ZONE, sizeof(TRI_external_t), true));
external->_executable = TRI_DuplicateString(executable);
external->_numberArguments = n + 1;
external->_arguments = static_cast<char**>(TRI_Allocate(TRI_CORE_MEM_ZONE, (n + 2) * sizeof(char*), true));
external->_arguments[0] = TRI_DuplicateString(executable);
for (size_t i = 0; i < n; ++i) {
external->_arguments[i + 1] = TRI_DuplicateString(arguments[i]);
}
external->_arguments[n + 1] = nullptr;
external->_status = TRI_EXT_NOT_STARTED;
StartExternalProcess(external, usePipes);
if (external->_status != TRI_EXT_RUNNING) {
pid->_pid = TRI_INVALID_PROCESS_ID;
FreeExternal(external);
return;
}
LOG_DEBUG("adding process %d to list", (int) external->_pid);
TRI_LockMutex(&ExternalProcessesLock);
TRI_PushBackVectorPointer(&ExternalProcesses, external);
// Note that the following deals with different types under windows,
// however, this code here can be written in a platform-independent
// way:
pid->_pid = external->_pid;
pid->_readPipe = external->_readPipe;
pid->_writePipe = external->_writePipe;
TRI_UnlockMutex(&ExternalProcessesLock);
}
bool TRI_DeleteDataShadowData (TRI_shadow_store_t* const store,
const void* const data) {
bool found = false;
assert(store);
if (data) {
TRI_shadow_t* shadow;
TRI_LockMutex(&store->_lock);
shadow = (TRI_shadow_t*) TRI_LookupByKeyAssociativePointer(&store->_pointers, data);
if (shadow && ! shadow->_deleted) {
DeleteShadow(store, shadow, true);
found = true;
}
TRI_UnlockMutex(&store->_lock);
}
return found;
}
void TRI_LockGeneralCursor (TRI_general_cursor_t* const cursor) {
TRI_LockMutex(&cursor->_lock);
}
static TRI_shape_pid_t FindNameAttributePath (TRI_shaper_t* shaper, char const* name) {
TRI_shape_aid_t* aids;
TRI_shape_path_t* result;
size_t count;
size_t len;
size_t total;
char* buffer;
char* end;
char* prev;
char* ptr;
void const* f;
void const* p;
p = TRI_LookupByKeyAssociativeSynced(&shaper->_attributePathsByName, name);
if (p != NULL) {
return ((TRI_shape_path_t const*) p)->_pid;
}
// create a attribute path
len = strlen(name);
// lock the index and check that the element is still missing
TRI_LockMutex(&shaper->_attributePathLock);
// if the element appeared, return the pid
p = TRI_LookupByKeyAssociativeSynced(&shaper->_attributePathsByName, name);
if (p != NULL) {
TRI_UnlockMutex(&shaper->_attributePathLock);
return ((TRI_shape_path_t const*) p)->_pid;
}
// split path into attribute pieces
count = 0;
aids = TRI_Allocate(shaper->_memoryZone, len * sizeof(TRI_shape_aid_t), false);
if (aids == NULL) {
TRI_UnlockMutex(&shaper->_attributePathLock);
LOG_ERROR("out of memory in shaper");
return 0;
}
buffer = ptr = TRI_DuplicateString2Z(shaper->_memoryZone, name, len);
if (buffer == NULL) {
TRI_UnlockMutex(&shaper->_attributePathLock);
TRI_Free(shaper->_memoryZone, aids);
LOG_ERROR("out of memory in shaper");
return 0;
}
end = buffer + len + 1;
prev = buffer;
for (; ptr < end; ++ptr) {
if (*ptr == '.' || *ptr == '\0') {
*ptr = '\0';
if (ptr != prev) {
aids[count++] = shaper->findAttributeName(shaper, prev);
}
prev = ptr + 1;
}
}
TRI_FreeString(shaper->_memoryZone, buffer);
// create element
total = sizeof(TRI_shape_path_t) + (len + 1) + (count * sizeof(TRI_shape_aid_t));
result = TRI_Allocate(shaper->_memoryZone, total, false);
if (result == NULL) {
TRI_UnlockMutex(&shaper->_attributePathLock);
TRI_Free(shaper->_memoryZone, aids);
LOG_ERROR("out of memory in shaper");
return 0;
}
result->_pid = shaper->_nextPid++;
result->_nameLength = len + 1;
result->_aidLength = count;
memcpy(((char*) result) + sizeof(TRI_shape_path_t), aids, count * sizeof(TRI_shape_aid_t));
memcpy(((char*) result) + sizeof(TRI_shape_path_t) + count * sizeof(TRI_shape_aid_t), name, len + 1);
TRI_Free(shaper->_memoryZone, aids);
f = TRI_InsertKeyAssociativeSynced(&shaper->_attributePathsByName, name, result);
assert(f == NULL);
f = TRI_InsertKeyAssociativeSynced(&shaper->_attributePathsByPid, &result->_pid, result);
assert(f == NULL);
// return pid
TRI_UnlockMutex(&shaper->_attributePathLock);
return result->_pid;
}
TRI_external_status_t TRI_CheckExternalProcess (TRI_external_id_t pid,
bool wait) {
TRI_external_status_t status;
TRI_external_t* external = nullptr; // Just to please the compiler
size_t i;
TRI_LockMutex(&ExternalProcessesLock);
status._status = TRI_EXT_NOT_FOUND;
status._exitStatus = 0;
for (i = 0; i < ExternalProcesses._length; ++i) {
external = static_cast<TRI_external_t*>(TRI_AtVectorPointer(&ExternalProcesses, i));
if (external->_pid == pid._pid) {
break;
}
}
if (i == ExternalProcesses._length) {
TRI_UnlockMutex(&ExternalProcessesLock);
status._errorMessage =
std::string("the pid you're looking for is not in our list: ") +
triagens::basics::StringUtils::itoa(static_cast<int64_t>(pid._pid));
status._status = TRI_EXT_NOT_FOUND;
LOG_WARNING("checkExternal: pid not found: %d", (int) pid._pid);
return status;
}
if (external->_status == TRI_EXT_RUNNING ||
external->_status == TRI_EXT_STOPPED) {
#ifndef _WIN32
TRI_pid_t res;
int opts;
int loc = 0;
if (wait) {
opts = WUNTRACED;
}
else {
opts = WNOHANG | WUNTRACED;
}
res = waitpid(external->_pid, &loc, opts);
if (res == 0) {
if (wait) {
status._errorMessage =
std::string("waitpid returned 0 for pid while it shouldn't ") +
triagens::basics::StringUtils::itoa(external->_pid);
if (WIFEXITED(loc)) {
external->_status = TRI_EXT_TERMINATED;
external->_exitStatus = WEXITSTATUS(loc);
}
else if (WIFSIGNALED(loc)) {
external->_status = TRI_EXT_ABORTED;
external->_exitStatus = WTERMSIG(loc);
}
else if (WIFSTOPPED(loc)) {
external->_status = TRI_EXT_STOPPED;
external->_exitStatus = 0;
}
else {
external->_status = TRI_EXT_ABORTED;
external->_exitStatus = 0;
}
}
else {
external->_exitStatus = 0;
}
}
else if (res == -1) {
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG_WARNING("waitpid returned error for pid %d (%d): %s",
(int) external->_pid,
(int) wait,
TRI_last_error());
status._errorMessage =
std::string("waitpid returned error for pid ") +
triagens::basics::StringUtils::itoa(external->_pid) +
std::string(": ") +
std::string(TRI_last_error());
}
else if (static_cast<TRI_pid_t>(external->_pid) == static_cast<TRI_pid_t>(res)) {
if (WIFEXITED(loc)) {
external->_status = TRI_EXT_TERMINATED;
external->_exitStatus = WEXITSTATUS(loc);
}
else if (WIFSIGNALED(loc)) {
external->_status = TRI_EXT_ABORTED;
external->_exitStatus = WTERMSIG(loc);
}
else if (WIFSTOPPED(loc)) {
external->_status = TRI_EXT_STOPPED;
external->_exitStatus = 0;
}
else {
external->_status = TRI_EXT_ABORTED;
external->_exitStatus = 0;
}
}
else {
LOG_WARNING("unexpected waitpid result for pid %d: %d",
(int) external->_pid,
(int) res);
status._errorMessage =
std::string("unexpected waitpid result for pid ") +
triagens::basics::StringUtils::itoa(external->_pid) +
std::string(": ") +
triagens::basics::StringUtils::itoa(res);
}
#else
{
char windowsErrorBuf[256];
bool wantGetExitCode = wait;
if (wait) {
DWORD result;
result = WaitForSingleObject(external->_process, INFINITE);
if (result == WAIT_FAILED) {
FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM,
NULL,
GetLastError(),
0,
windowsErrorBuf,
sizeof(windowsErrorBuf), NULL);
LOG_WARNING("could not wait for subprocess with PID '%ud': %s",
(unsigned int) external->_pid, windowsErrorBuf);
status._errorMessage =
std::string("could not wait for subprocess with PID '") +
triagens::basics::StringUtils::itoa(static_cast<int64_t>(external->_pid)) +
std::string("'") +
windowsErrorBuf;
status._exitStatus = GetLastError();
}
}
else {
DWORD result;
result = WaitForSingleObject(external->_process, 0);
switch (result) {
case WAIT_ABANDONED:
wantGetExitCode = true;
LOG_WARNING("WAIT_ABANDONED while waiting for subprocess with PID '%ud'",
(unsigned int)external->_pid);
break;
case WAIT_OBJECT_0:
/// this seems to be the exit case - want getExitCodeProcess here.
wantGetExitCode = true;
break;
case WAIT_TIMEOUT:
// success - everything went well.
external->_exitStatus = 0;
break;
case WAIT_FAILED:
FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM,
NULL,
GetLastError(),
0,
windowsErrorBuf,
sizeof(windowsErrorBuf), NULL);
LOG_WARNING("could not wait for subprocess with PID '%ud': %s",
(unsigned int)external->_pid, windowsErrorBuf);
status._errorMessage =
std::string("could not wait for subprocess with PID '") +
triagens::basics::StringUtils::itoa(static_cast<int64_t>(external->_pid)) +
std::string("'") +
windowsErrorBuf;
status._exitStatus = GetLastError();
default:
wantGetExitCode = true;
LOG_WARNING("unexpected status while waiting for subprocess with PID '%ud'",
(unsigned int)external->_pid);
}
}
if (wantGetExitCode) {
DWORD exitCode = STILL_ACTIVE;
if (!GetExitCodeProcess(external->_process, &exitCode)) {
LOG_WARNING("exit status could not be determined for PID '%ud'",
(unsigned int)external->_pid);
status._errorMessage =
std::string("exit status could not be determined for PID '") +
triagens::basics::StringUtils::itoa(static_cast<int64_t>(external->_pid)) +
std::string("'");
}
else {
if (exitCode == STILL_ACTIVE) {
external->_exitStatus = 0;
}
else if (exitCode > 255) {
// this should be one of our signals which we mapped...
external->_status = TRI_EXT_ABORTED;
external->_exitStatus = exitCode - 255;
}
else {
external->_status = TRI_EXT_TERMINATED;
external->_exitStatus = exitCode;
}
}
}
else
{
external->_status = TRI_EXT_RUNNING;
}
}
#endif
}
else {
LOG_WARNING("unexpected process status %d: %d",
(int) external->_status,
(int) external->_exitStatus);
status._errorMessage =
std::string("unexpected process status ") +
triagens::basics::StringUtils::itoa(external->_status) +
std::string(": ") +
triagens::basics::StringUtils::itoa(external->_exitStatus);
}
status._status = external->_status;
status._exitStatus = external->_exitStatus;
// Do we have to free our data?
if (external->_status != TRI_EXT_RUNNING &&
external->_status != TRI_EXT_STOPPED) {
TRI_RemoveVectorPointer(&ExternalProcesses, i);
FreeExternal(external);
}
TRI_UnlockMutex(&ExternalProcessesLock);
return status;
}
void TRI_CleanupShadowData (TRI_shadow_store_t* const store,
const double maxAge,
const bool force) {
double compareStamp = TRI_microtime() - maxAge; // age must be specified in secs
size_t deleteCount = 0;
// we need an exclusive lock on the index
TRI_LockMutex(&store->_lock);
if (store->_ids._nrUsed == 0) {
// store is empty, nothing to do!
TRI_UnlockMutex(&store->_lock);
return;
}
LOG_TRACE("cleaning shadows. in store: %ld", (unsigned long) store->_ids._nrUsed);
// loop until there's nothing to delete or
// we have deleted SHADOW_MAX_DELETE elements
while (deleteCount++ < SHADOW_MAX_DELETE || force) {
bool deleted = false;
size_t i;
for (i = 0; i < store->_ids._nrAlloc; i++) {
// enum all shadows
TRI_shadow_t* shadow = (TRI_shadow_t*) store->_ids._table[i];
if (shadow == NULL) {
continue;
}
// check if shadow is unused and expired
if (shadow->_rc < 1 || force) {
if (shadow->_type == SHADOW_TRANSIENT ||
shadow->_timestamp < compareStamp ||
shadow->_deleted ||
force) {
LOG_TRACE("cleaning shadow %p, id: %llu, rc: %d, expired: %d, deleted: %d",
shadow,
(unsigned long long) shadow->_id,
(int) shadow->_rc,
(int) (shadow->_timestamp < compareStamp),
(int) shadow->_deleted);
TRI_RemoveKeyAssociativePointer(&store->_ids, &shadow->_id);
TRI_RemoveKeyAssociativePointer(&store->_pointers, shadow->_data);
store->destroyShadow(shadow->_data);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, shadow);
deleted = true;
// the remove might reposition elements in the container.
// therefore break here and start iteration anew
break;
}
}
}
if (! deleted) {
// we did not find anything to delete, so give up
break;
}
}
// release lock
TRI_UnlockMutex(&store->_lock);
}
