bool VelocyPackHelper::velocyPackToFile(char const* filename,
VPackSlice const& slice,
bool syncFile) {
std::string const tmp = std::string(filename) + ".tmp";
// remove a potentially existing temporary file
if (TRI_ExistsFile(tmp.c_str())) {
TRI_UnlinkFile(tmp.c_str());
}
int fd = TRI_CREATE(tmp.c_str(), O_CREAT | O_TRUNC | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
if (fd < 0) {
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG(ERR) << "cannot create json file '" << tmp << "': " << TRI_LAST_ERROR_STR;
return false;
}
if (!PrintVelocyPack(fd, slice, true)) {
TRI_CLOSE(fd);
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG(ERR) << "cannot write to json file '" << tmp << "': " << TRI_LAST_ERROR_STR;
TRI_UnlinkFile(tmp.c_str());
return false;
}
if (syncFile) {
LOG(TRACE) << "syncing tmp file '" << tmp << "'";
if (!TRI_fsync(fd)) {
TRI_CLOSE(fd);
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG(ERR) << "cannot sync saved json '" << tmp << "': " << TRI_LAST_ERROR_STR;
TRI_UnlinkFile(tmp.c_str());
return false;
}
}
int res = TRI_CLOSE(fd);
if (res < 0) {
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG(ERR) << "cannot close saved file '" << tmp << "': " << TRI_LAST_ERROR_STR;
TRI_UnlinkFile(tmp.c_str());
return false;
}
res = TRI_RenameFile(tmp.c_str(), filename);
if (res != TRI_ERROR_NO_ERROR) {
TRI_set_errno(res);
LOG(ERR) << "cannot rename saved file '" << tmp << "' to '" << filename << "': " << TRI_LAST_ERROR_STR;
TRI_UnlinkFile(tmp.c_str());
return false;
}
return true;
}
int TRI_RemoveConfigurationReplicationApplier (TRI_vocbase_t* vocbase) {
char* filename;
int res;
if (vocbase->_type == TRI_VOCBASE_TYPE_COORDINATOR) {
return TRI_ERROR_CLUSTER_UNSUPPORTED;
}
filename = GetConfigurationFilename(vocbase);
if (filename == NULL) {
return TRI_ERROR_OUT_OF_MEMORY;
}
if (TRI_ExistsFile(filename)) {
res = TRI_UnlinkFile(filename);
}
else {
res = TRI_ERROR_NO_ERROR;
}
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
return res;
}
int TRI_RemoveStateReplicationApplier (TRI_vocbase_t* vocbase) {
char* filename;
int res;
if (vocbase->_type == TRI_VOCBASE_TYPE_COORDINATOR) {
return TRI_ERROR_CLUSTER_UNSUPPORTED;
}
filename = GetStateFilename(vocbase);
if (filename == nullptr) {
return TRI_ERROR_OUT_OF_MEMORY;
}
if (TRI_ExistsFile(filename)) {
LOG_TRACE("removing replication state file '%s'", filename);
res = TRI_UnlinkFile(filename);
}
else {
res = TRI_ERROR_NO_ERROR;
}
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
return res;
}
int TRI_RemoveConfigurationReplicationApplier (TRI_vocbase_t* vocbase) {
char* filename;
int res;
filename = GetConfigurationFilename(vocbase);
if (filename == NULL) {
return TRI_ERROR_OUT_OF_MEMORY;
}
if (TRI_ExistsFile(filename)) {
res = TRI_UnlinkFile(filename);
}
else {
res = TRI_ERROR_NO_ERROR;
}
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
return res;
}
int TRI_RemoveStateReplicationApplier (TRI_vocbase_t* vocbase) {
char* filename;
int res;
filename = GetStateFilename(vocbase);
if (filename == NULL) {
return TRI_ERROR_OUT_OF_MEMORY;
}
if (TRI_ExistsFile(filename)) {
LOG_TRACE("removing replication state file '%s'", filename);
res = TRI_UnlinkFile(filename);
}
else {
res = TRI_ERROR_NO_ERROR;
}
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
return res;
}
static int LoadConfiguration (TRI_vocbase_t* vocbase,
TRI_replication_applier_configuration_t* config) {
TRI_json_t* json;
TRI_json_t* value;
char* filename;
int res;
TRI_DestroyConfigurationReplicationApplier(config);
TRI_InitConfigurationReplicationApplier(config);
filename = GetConfigurationFilename(vocbase);
if (! TRI_ExistsFile(filename)) {
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
return TRI_ERROR_FILE_NOT_FOUND;
}
json = TRI_JsonFile(TRI_CORE_MEM_ZONE, filename, NULL);
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
if (! TRI_IsArrayJson(json)) {
if (json != NULL) {
TRI_FreeJson(TRI_CORE_MEM_ZONE, json);
}
return TRI_ERROR_REPLICATION_INVALID_APPLIER_CONFIGURATION;
}
res = TRI_ERROR_NO_ERROR;
if (config->_endpoint != NULL) {
TRI_FreeString(TRI_CORE_MEM_ZONE, config->_endpoint);
config->_endpoint = NULL;
}
if (config->_database != NULL) {
TRI_FreeString(TRI_CORE_MEM_ZONE, config->_database);
config->_database = NULL;
}
if (config->_username != NULL) {
TRI_FreeString(TRI_CORE_MEM_ZONE, config->_username);
config->_username = NULL;
}
if (config->_password != NULL) {
TRI_FreeString(TRI_CORE_MEM_ZONE, config->_password);
config->_password = NULL;
}
// read the endpoint
value = TRI_LookupArrayJson(json, "endpoint");
if (! TRI_IsStringJson(value)) {
res = TRI_ERROR_REPLICATION_INVALID_APPLIER_CONFIGURATION;
}
else {
config->_endpoint = TRI_DuplicateString2Z(TRI_CORE_MEM_ZONE,
value->_value._string.data,
value->_value._string.length - 1);
}
// read the database name
value = TRI_LookupArrayJson(json, "database");
if (! TRI_IsStringJson(value)) {
config->_database = TRI_DuplicateStringZ(TRI_CORE_MEM_ZONE,
vocbase->_name);
}
else {
config->_database = TRI_DuplicateString2Z(TRI_CORE_MEM_ZONE,
value->_value._string.data,
value->_value._string.length - 1);
}
// read username / password
value = TRI_LookupArrayJson(json, "username");
if (TRI_IsStringJson(value)) {
config->_username = TRI_DuplicateString2Z(TRI_CORE_MEM_ZONE,
value->_value._string.data,
value->_value._string.length - 1);
}
value = TRI_LookupArrayJson(json, "password");
if (TRI_IsStringJson(value)) {
config->_password = TRI_DuplicateString2Z(TRI_CORE_MEM_ZONE,
value->_value._string.data,
value->_value._string.length - 1);
}
value = TRI_LookupArrayJson(json, "requestTimeout");
if (TRI_IsNumberJson(value)) {
config->_requestTimeout = value->_value._number;
}
value = TRI_LookupArrayJson(json, "connectTimeout");
if (TRI_IsNumberJson(value)) {
config->_connectTimeout = value->_value._number;
}
value = TRI_LookupArrayJson(json, "maxConnectRetries");
if (TRI_IsNumberJson(value)) {
config->_maxConnectRetries = (uint64_t) value->_value._number;
}
value = TRI_LookupArrayJson(json, "chunkSize");
if (TRI_IsNumberJson(value)) {
config->_chunkSize = (uint64_t) value->_value._number;
}
value = TRI_LookupArrayJson(json, "autoStart");
if (TRI_IsBooleanJson(value)) {
config->_autoStart = value->_value._boolean;
}
value = TRI_LookupArrayJson(json, "adaptivePolling");
if (TRI_IsBooleanJson(value)) {
config->_adaptivePolling = value->_value._boolean;
}
TRI_FreeJson(TRI_CORE_MEM_ZONE, json);
return res;
}
int TRI_LoadStateReplicationApplier (TRI_vocbase_t* vocbase,
TRI_replication_applier_state_t* state) {
TRI_json_t* json;
TRI_json_t* serverId;
char* filename;
int res;
TRI_InitStateReplicationApplier(state);
filename = GetStateFilename(vocbase);
if (filename == NULL) {
return TRI_ERROR_OUT_OF_MEMORY;
}
LOG_TRACE("looking for replication state file '%s'", filename);
if (! TRI_ExistsFile(filename)) {
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
return TRI_ERROR_FILE_NOT_FOUND;
}
LOG_TRACE("replication state file '%s' found", filename);
json = TRI_JsonFile(TRI_CORE_MEM_ZONE, filename, NULL);
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
if (! TRI_IsArrayJson(json)) {
if (json != NULL) {
TRI_FreeJson(TRI_CORE_MEM_ZONE, json);
}
return TRI_ERROR_REPLICATION_INVALID_APPLIER_STATE;
}
res = TRI_ERROR_NO_ERROR;
// read the server id
serverId = TRI_LookupArrayJson(json, "serverId");
if (! TRI_IsStringJson(serverId)) {
res = TRI_ERROR_REPLICATION_INVALID_APPLIER_STATE;
}
else {
state->_serverId = TRI_UInt64String2(serverId->_value._string.data,
serverId->_value._string.length - 1);
}
if (res == TRI_ERROR_NO_ERROR) {
// read the ticks
res |= ReadTick(json, "lastAppliedContinuousTick", &state->_lastAppliedContinuousTick);
// set processed = applied
state->_lastProcessedContinuousTick = state->_lastAppliedContinuousTick;
}
TRI_FreeJson(TRI_CORE_MEM_ZONE, json);
LOG_TRACE("replication state file read successfully");
return res;
}
static void DropDatafileCallback (TRI_datafile_t* datafile, void* data) {
TRI_primary_collection_t* primary;
TRI_voc_fid_t fid;
char* filename;
char* name;
char* number;
char* copy;
bool ok;
primary = data;
fid = datafile->_fid;
copy = NULL;
number = TRI_StringUInt64(fid);
name = TRI_Concatenate3String("deleted-", number, ".db");
filename = TRI_Concatenate2File(primary->base._directory, name);
TRI_FreeString(TRI_CORE_MEM_ZONE, number);
TRI_FreeString(TRI_CORE_MEM_ZONE, name);
if (datafile->isPhysical(datafile)) {
copy = TRI_DuplicateStringZ(TRI_CORE_MEM_ZONE, datafile->_filename);
ok = TRI_RenameDatafile(datafile, filename);
if (! ok) {
LOG_ERROR("cannot rename obsolete datafile '%s' to '%s': %s",
copy,
filename,
TRI_last_error());
}
}
LOG_DEBUG("finished compacting datafile '%s'", datafile->getName(datafile));
ok = TRI_CloseDatafile(datafile);
if (! ok) {
LOG_ERROR("cannot close obsolete datafile '%s': %s",
datafile->getName(datafile),
TRI_last_error());
}
else if (datafile->isPhysical(datafile)) {
if (primary->base._vocbase->_settings.removeOnCompacted) {
int res;
LOG_DEBUG("wiping compacted datafile from disk");
res = TRI_UnlinkFile(filename);
if (res != TRI_ERROR_NO_ERROR) {
LOG_ERROR("cannot wipe obsolete datafile '%s': %s",
datafile->getName(datafile),
TRI_last_error());
}
// check for .dead files
if (copy != NULL) {
// remove .dead file for datafile
char* deadfile = TRI_Concatenate2String(copy, ".dead");
if (deadfile != NULL) {
// check if .dead file exists, then remove it
if (TRI_ExistsFile(deadfile)) {
TRI_UnlinkFile(deadfile);
}
TRI_FreeString(TRI_CORE_MEM_ZONE, deadfile);
}
}
}
}
TRI_FreeDatafile(datafile);
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
if (copy != NULL) {
TRI_FreeString(TRI_CORE_MEM_ZONE, copy);
}
}
static int ExtractCurrentFile (unzFile uf,
void* buffer,
const size_t bufferSize,
const char* outPath,
const bool skipPaths,
const bool overwrite,
const char* password) {
char filenameInZip[256];
char* filenameWithoutPath;
char* fullPath;
char* p;
FILE *fout;
unz_file_info64 fileInfo;
if (unzGetCurrentFileInfo64(uf, &fileInfo, filenameInZip, sizeof(filenameInZip), NULL, 0, NULL, 0) != UNZ_OK) {
return TRI_ERROR_INTERNAL;
}
p = filenameWithoutPath = filenameInZip;
// get the file name without any path prefix
while (*p != '\0') {
if (*p == '/' || *p == '\\') {
filenameWithoutPath = p + 1;
}
p++;
}
// found a directory
if (*filenameWithoutPath == '\0') {
if (! skipPaths) {
fullPath = TRI_Concatenate2File(outPath, filenameInZip);
TRI_CreateRecursiveDirectory(fullPath);
TRI_Free(TRI_CORE_MEM_ZONE, fullPath);
}
}
// found a file
else {
const char* writeFilename;
if (! skipPaths) {
writeFilename = filenameInZip;
}
else {
writeFilename = filenameWithoutPath;
}
if (unzOpenCurrentFilePassword(uf, password) != UNZ_OK) {
return TRI_ERROR_INTERNAL;
}
// prefix the name from the zip file with the path specified
fullPath = TRI_Concatenate2File(outPath, writeFilename);
if (! overwrite && TRI_ExistsFile(fullPath)) {
return TRI_ERROR_INTERNAL;
}
// try to write the outfile
fout = fopen(fullPath, "wb");
// cannot write to outfile. this may be due to the target directory missing
if (fout == NULL && ! skipPaths && filenameWithoutPath != (char*) filenameInZip) {
char* d;
char c = *(filenameWithoutPath - 1);
*(filenameWithoutPath - 1) = '\0';
// create target directory recursively
d = TRI_Concatenate2File(outPath, filenameInZip);
TRI_CreateRecursiveDirectory(d);
TRI_Free(TRI_CORE_MEM_ZONE, d);
*(filenameWithoutPath - 1) = c;
// try again
fout = fopen(fullPath, "wb");
}
TRI_Free(TRI_CORE_MEM_ZONE, fullPath);
if (fout == NULL) {
return TRI_ERROR_CANNOT_WRITE_FILE;
}
while (true) {
int result = unzReadCurrentFile(uf, buffer, bufferSize);
if (result < 0) {
fclose(fout);
return TRI_ERROR_INTERNAL;
}
if (result > 0) {
if (fwrite(buffer, result, 1, fout) != 1) {
fclose(fout);
return TRI_set_errno(TRI_ERROR_SYS_ERROR);
}
}
else {
assert(result == 0);
break;
}
}
fclose(fout);
}
unzCloseCurrentFile(uf);
return TRI_ERROR_NO_ERROR;
}
int TRI_ZipFile (const char* filename,
const char* dir,
TRI_vector_string_t const* files,
const char* password) {
void* buffer;
size_t bufferSize;
zipFile zf;
#ifdef USEWIN32IOAPI
zlib_filefunc64_def ffunc;
#endif
size_t i, n;
int res;
if (TRI_ExistsFile(filename)) {
return TRI_ERROR_CANNOT_OVERWRITE_FILE;
}
bufferSize = 16384;
buffer = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, bufferSize, false);
if (buffer == NULL) {
return TRI_ERROR_OUT_OF_MEMORY;
}
#ifdef USEWIN32IOAPI
fill_win32_filefunc64A(&ffunc);
zf = zipOpen2_64(filename, 0, NULL, &ffunc);
#else
zf = zipOpen64(filename, 0);
#endif
if (zf == NULL) {
TRI_Free(TRI_UNKNOWN_MEM_ZONE, buffer);
return ZIP_ERRNO;
}
res = TRI_ERROR_NO_ERROR;
n = files->_length;
for (i = 0; i < n; ++i) {
FILE* fin;
char* file;
char* fullfile;
char* saveName;
zip_fileinfo zi;
uint32_t crc;
int isLarge;
file = TRI_AtVectorString(files, i);
if (*dir == '\0') {
fullfile = TRI_DuplicateString(file);
}
else {
fullfile = TRI_Concatenate2File(dir, file);
}
memset(&zi, 0, sizeof(zi));
res = TRI_Crc32File(fullfile, &crc);
if (res != TRI_ERROR_NO_ERROR) {
break;
}
isLarge = (TRI_SizeFile(file) > 0xFFFFFFFFLL);
saveName = file;
while (*saveName == '\\' || *saveName == '/') {
++saveName;
}
if (zipOpenNewFileInZip3_64(zf,
saveName,
&zi,
NULL,
0,
NULL,
0,
NULL, /* comment*/
Z_DEFLATED,
Z_DEFAULT_COMPRESSION,
0,
-MAX_WBITS,
DEF_MEM_LEVEL,
Z_DEFAULT_STRATEGY,
password,
(unsigned long) crc,
isLarge) != ZIP_OK) {
}
fin = fopen(fullfile, "rb");
TRI_FreeString(TRI_CORE_MEM_ZONE, fullfile);
if (fin == NULL) {
break;
}
while (true) {
int sizeRead;
sizeRead = (int) fread(buffer, 1, bufferSize, fin);
if (sizeRead < bufferSize) {
if (feof(fin) == 0) {
res = TRI_set_errno(TRI_ERROR_SYS_ERROR);
break;
}
}
if (sizeRead > 0) {
res = zipWriteInFileInZip(zf, buffer, sizeRead);
if (res != 0) {
break;
}
}
else if (sizeRead <= 0) {
break;
}
}
fclose(fin);
zipCloseFileInZip(zf);
if (res != TRI_ERROR_NO_ERROR) {
break;
}
}
zipClose(zf, NULL);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, buffer);
return res;
}
int TRI_LoadCollectionInfo (char const* path, TRI_col_info_t* parameter) {
TRI_json_t* json;
char* filename;
char* error = NULL;
size_t i;
size_t n;
memset(parameter, 0, sizeof(TRI_col_info_t));
// find parameter file
filename = TRI_Concatenate2File(path, TRI_COL_PARAMETER_FILE);
if (filename == NULL) {
LOG_ERROR("cannot load parameter info for collection '%s', out of memory", path);
return TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY);
}
if (! TRI_ExistsFile(filename)) {
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
return TRI_set_errno(TRI_ERROR_ARANGO_ILLEGAL_PARAMETER_FILE);
}
json = TRI_JsonFile(TRI_UNKNOWN_MEM_ZONE, filename, &error);
if (json == NULL) {
if (error != NULL) {
LOG_ERROR("cannot open '%s', parameter block not readable: %s", filename, error);
TRI_FreeString(TRI_CORE_MEM_ZONE, error);
}
else {
LOG_ERROR("cannot open '%s', parameter block not readable", filename);
}
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
return TRI_set_errno(TRI_ERROR_ARANGO_ILLEGAL_PARAMETER_FILE);
}
if (json->_type != TRI_JSON_ARRAY) {
LOG_ERROR("cannot open '%s', file does not contain a json array", filename);
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
return TRI_set_errno(TRI_ERROR_ARANGO_ILLEGAL_PARAMETER_FILE);
}
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
// convert json
n = json->_value._objects._length;
for (i = 0; i < n; i += 2) {
TRI_json_t* key;
TRI_json_t* value;
key = TRI_AtVector(&json->_value._objects, i);
value = TRI_AtVector(&json->_value._objects, i + 1);
if (key->_type == TRI_JSON_STRING && value->_type == TRI_JSON_NUMBER) {
if (TRI_EqualString(key->_value._string.data, "version")) {
parameter->_version = value->_value._number;
}
else if (TRI_EqualString(key->_value._string.data, "type")) {
parameter->_type = value->_value._number;
}
else if (TRI_EqualString(key->_value._string.data, "cid")) {
parameter->_cid = value->_value._number;
}
else if (TRI_EqualString(key->_value._string.data, "maximalSize")) {
parameter->_maximalSize = value->_value._number;
}
}
else if (key->_type == TRI_JSON_STRING && value->_type == TRI_JSON_STRING) {
if (TRI_EqualString(key->_value._string.data, "name")) {
TRI_CopyString(parameter->_name, value->_value._string.data, sizeof(parameter->_name));
parameter->_isSystem = TRI_IsSystemCollectionName(parameter->_name);
}
}
else if (key->_type == TRI_JSON_STRING && value->_type == TRI_JSON_BOOLEAN) {
if (TRI_EqualString(key->_value._string.data, "deleted")) {
parameter->_deleted = value->_value._boolean;
}
else if (TRI_EqualString(key->_value._string.data, "isVolatile")) {
parameter->_isVolatile = value->_value._boolean;
}
else if (TRI_EqualString(key->_value._string.data, "waitForSync")) {
parameter->_waitForSync = value->_value._boolean;
}
}
else if (key->_type == TRI_JSON_STRING && value->_type == TRI_JSON_ARRAY) {
if (TRI_EqualString(key->_value._string.data, "keyOptions")) {
parameter->_keyOptions = TRI_CopyJson(TRI_UNKNOWN_MEM_ZONE, value);
}
}
}
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_NO_ERROR;
}
TRI_collection_t* TRI_CreateCollection (TRI_vocbase_t* vocbase,
TRI_collection_t* collection,
char const* path,
const TRI_col_info_t* const parameter) {
char* filename;
// sanity check
if (sizeof(TRI_df_header_marker_t) + sizeof(TRI_df_footer_marker_t) > parameter->_maximalSize) {
TRI_set_errno(TRI_ERROR_ARANGO_DATAFILE_FULL);
LOG_ERROR("cannot create datafile '%s' in '%s', maximal size '%u' is too small",
parameter->_name,
path,
(unsigned int) parameter->_maximalSize);
return NULL;
}
if (! TRI_IsDirectory(path)) {
TRI_set_errno(TRI_ERROR_ARANGO_WRONG_VOCBASE_PATH);
LOG_ERROR("cannot create collection '%s', path is not a directory", path);
return NULL;
}
filename = TRI_GetDirectoryCollection(path, parameter);
if (filename == NULL) {
LOG_ERROR("cannot create collection '%s'", TRI_last_error());
return NULL;
}
// directory must not exist
if (TRI_ExistsFile(filename)) {
TRI_set_errno(TRI_ERROR_ARANGO_COLLECTION_DIRECTORY_ALREADY_EXISTS);
LOG_ERROR("cannot create collection '%s' in '%s', directory already exists",
parameter->_name, filename);
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
return NULL;
}
// create directory
if (! TRI_CreateDirectory(filename)) {
LOG_ERROR("cannot create collection '%s' in '%s' as '%s': %s",
parameter->_name,
path,
filename,
TRI_last_error());
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
return NULL;
}
// create collection structure
if (collection == NULL) {
collection = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_collection_t), false);
if (collection == NULL) {
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
LOG_ERROR("cannot create collection '%s', out of memory", path);
return NULL;
}
}
// we are passing filename to this struct, so we must not free it if you use the struct later
InitCollection(vocbase, collection, filename, parameter);
/* PANAIA: 1) the parameter file if it exists must be removed
2) if collection
*/
return collection;
}
bool TRI_SaveJson (char const* filename,
TRI_json_t const* object,
bool syncFile) {
char* tmp;
int fd;
int res;
tmp = TRI_Concatenate2String(filename, ".tmp");
if (tmp == NULL) {
return false;
}
// remove a potentially existing temporary file
if (TRI_ExistsFile(tmp)) {
TRI_UnlinkFile(tmp);
}
fd = TRI_CREATE(tmp, O_CREAT | O_TRUNC | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR);
if (fd < 0) {
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG_ERROR("cannot create json file '%s': %s", tmp, TRI_LAST_ERROR_STR);
TRI_FreeString(TRI_CORE_MEM_ZONE, tmp);
return false;
}
if (! TRI_PrintJson(fd, object, true)) {
TRI_CLOSE(fd);
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG_ERROR("cannot write to json file '%s': %s", tmp, TRI_LAST_ERROR_STR);
TRI_UnlinkFile(tmp);
TRI_FreeString(TRI_CORE_MEM_ZONE, tmp);
return false;
}
if (syncFile) {
LOG_TRACE("syncing tmp file '%s'", tmp);
if (! TRI_fsync(fd)) {
TRI_CLOSE(fd);
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG_ERROR("cannot sync saved json '%s': %s", tmp, TRI_LAST_ERROR_STR);
TRI_UnlinkFile(tmp);
TRI_FreeString(TRI_CORE_MEM_ZONE, tmp);
return false;
}
}
res = TRI_CLOSE(fd);
if (res < 0) {
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG_ERROR("cannot close saved file '%s': %s", tmp, TRI_LAST_ERROR_STR);
TRI_UnlinkFile(tmp);
TRI_FreeString(TRI_CORE_MEM_ZONE, tmp);
return false;
}
res = TRI_RenameFile(tmp, filename);
if (res != TRI_ERROR_NO_ERROR) {
TRI_set_errno(res);
LOG_ERROR("cannot rename saved file '%s' to '%s': %s", tmp, filename, TRI_LAST_ERROR_STR);
TRI_UnlinkFile(tmp);
TRI_FreeString(TRI_CORE_MEM_ZONE, tmp);
return false;
}
TRI_FreeString(TRI_CORE_MEM_ZONE, tmp);
return true;
}
static TRI_datafile_t* CreateCompactor (TRI_primary_collection_t* primary,
TRI_voc_fid_t fid,
TRI_voc_size_t maximalSize) {
TRI_col_header_marker_t cm;
TRI_collection_t* collection;
TRI_datafile_t* journal;
TRI_df_marker_t* position;
int res;
collection = &primary->base;
if (collection->_info._isVolatile) {
// in-memory collection
journal = TRI_CreateDatafile(NULL, fid, maximalSize);
}
else {
char* jname;
char* number;
char* filename;
number = TRI_StringUInt64(fid);
jname = TRI_Concatenate3String("compaction-", number, ".db");
filename = TRI_Concatenate2File(collection->_directory, jname);
TRI_FreeString(TRI_CORE_MEM_ZONE, number);
TRI_FreeString(TRI_CORE_MEM_ZONE, jname);
if (TRI_ExistsFile(filename)) {
// remove any existing temporary file first
TRI_UnlinkFile(filename);
}
journal = TRI_CreateDatafile(filename, fid, maximalSize);
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
}
if (journal == NULL) {
if (TRI_errno() == TRI_ERROR_OUT_OF_MEMORY_MMAP) {
collection->_lastError = TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY_MMAP);
collection->_state = TRI_COL_STATE_READ;
}
else {
collection->_lastError = TRI_set_errno(TRI_ERROR_ARANGO_NO_JOURNAL);
collection->_state = TRI_COL_STATE_WRITE_ERROR;
}
return NULL;
}
LOG_TRACE("created new compactor '%s'", journal->getName(journal));
// create a collection header, still in the temporary file
res = TRI_ReserveElementDatafile(journal, sizeof(TRI_col_header_marker_t), &position, maximalSize);
if (res != TRI_ERROR_NO_ERROR) {
collection->_lastError = journal->_lastError;
LOG_ERROR("cannot create document header in compactor '%s': %s", journal->getName(journal), TRI_last_error());
TRI_FreeDatafile(journal);
return NULL;
}
TRI_InitMarker((char*) &cm, TRI_COL_MARKER_HEADER, sizeof(TRI_col_header_marker_t));
cm.base._tick = (TRI_voc_tick_t) fid;
cm._type = (TRI_col_type_t) collection->_info._type;
cm._cid = collection->_info._cid;
res = TRI_WriteCrcElementDatafile(journal, position, &cm.base, sizeof(cm), false);
if (res != TRI_ERROR_NO_ERROR) {
collection->_lastError = journal->_lastError;
LOG_ERROR("cannot create document header in compactor '%s': %s", journal->getName(journal), TRI_last_error());
TRI_FreeDatafile(journal);
return NULL;
}
assert(fid == journal->_fid);
return journal;
}
static int ExtractCurrentFile(unzFile uf, void* buffer, size_t const bufferSize,
char const* outPath, bool const skipPaths,
bool const overwrite, char const* password,
std::string& errorMessage) {
char filenameInZip[256];
char* filenameWithoutPath;
char* fullPath;
char* p;
unz_file_info64 fileInfo;
long systemError;
int err;
// cppcheck-suppress *
FILE* fout;
filenameInZip[0] = '\0';
err = unzGetCurrentFileInfo64(uf, &fileInfo, filenameInZip,
sizeof(filenameInZip), NULL, 0, NULL, 0);
if (err != UNZ_OK) {
errorMessage = std::string("Failed to get file info for ") + filenameInZip +
": " + std::to_string(err);
return TRI_ERROR_INTERNAL;
}
// adjust file name
p = filenameInZip;
while (*p != '\0') {
#ifdef WIN32
if (*p == '/') {
*p = '\\';
}
#else
if (*p == '\\') {
*p = '/';
}
#endif
++p;
}
p = filenameWithoutPath = filenameInZip;
// get the file name without any path prefix
while (*p != '\0') {
if (*p == '/' || *p == '\\' || *p == ':') {
filenameWithoutPath = p + 1;
}
p++;
}
// found a directory
if (*filenameWithoutPath == '\0') {
if (!skipPaths) {
fullPath = TRI_Concatenate2File(outPath, filenameInZip);
int res =
TRI_CreateRecursiveDirectory(fullPath, systemError, errorMessage);
if (res != TRI_ERROR_NO_ERROR) {
TRI_Free(TRI_CORE_MEM_ZONE, fullPath);
return res;
}
TRI_Free(TRI_CORE_MEM_ZONE, fullPath);
}
}
// found a file
else {
char const* writeFilename;
if (!skipPaths) {
writeFilename = filenameInZip;
} else {
writeFilename = filenameWithoutPath;
}
err = unzOpenCurrentFilePassword(uf, password);
if (err != UNZ_OK) {
errorMessage = "failed to authenticate the password in the zip: " +
std::to_string(err);
return TRI_ERROR_INTERNAL;
}
// prefix the name from the zip file with the path specified
fullPath = TRI_Concatenate2File(outPath, writeFilename);
if (!overwrite && TRI_ExistsFile(fullPath)) {
TRI_Free(TRI_CORE_MEM_ZONE, fullPath);
errorMessage = std::string("not allowed to overwrite file ") + fullPath;
return TRI_ERROR_CANNOT_OVERWRITE_FILE;
}
// try to write the outfile
fout = fopen(fullPath, "wb");
// cannot write to outfile. this may be due to the target directory missing
if (fout == nullptr && !skipPaths &&
filenameWithoutPath != (char*)filenameInZip) {
char c = *(filenameWithoutPath - 1);
*(filenameWithoutPath - 1) = '\0';
// create target directory recursively
char* d = TRI_Concatenate2File(outPath, filenameInZip);
int res = TRI_CreateRecursiveDirectory(d, systemError, errorMessage);
if (res != TRI_ERROR_NO_ERROR) {
TRI_Free(TRI_CORE_MEM_ZONE, d);
return res;
}
TRI_Free(TRI_CORE_MEM_ZONE, d);
*(filenameWithoutPath - 1) = c;
// try again
fout = fopen(fullPath, "wb");
} else if (fout == nullptr) {
// try to create the target directory recursively
char* d = TRI_Concatenate2File(outPath, filenameInZip);
// strip filename so we only have the directory name
char* dir = TRI_Dirname(d);
int res = TRI_CreateRecursiveDirectory(dir, systemError, errorMessage);
if (res != TRI_ERROR_NO_ERROR) {
TRI_Free(TRI_CORE_MEM_ZONE, d);
TRI_Free(TRI_CORE_MEM_ZONE, dir);
return res;
}
TRI_Free(TRI_CORE_MEM_ZONE, d);
TRI_Free(TRI_CORE_MEM_ZONE, dir);
// try again
fout = fopen(fullPath, "wb");
}
if (fout == NULL) {
errorMessage = std::string("failed to open file for writing: ") +
fullPath + " - " + strerror(errno);
TRI_Free(TRI_CORE_MEM_ZONE, fullPath);
return TRI_ERROR_CANNOT_WRITE_FILE;
}
while (true) {
int result = unzReadCurrentFile(uf, buffer, (unsigned int)bufferSize);
if (result < 0) {
errorMessage = std::string("failed to write file ") + fullPath + " - " +
strerror(errno);
fclose(fout);
TRI_Free(TRI_CORE_MEM_ZONE, fullPath);
return TRI_ERROR_CANNOT_WRITE_FILE;
}
if (result > 0) {
if (fwrite(buffer, result, 1, fout) != 1) {
errorMessage = std::string("failed to write file ") + fullPath +
" - " + strerror(errno);
TRI_Free(TRI_CORE_MEM_ZONE, fullPath);
fclose(fout);
return TRI_set_errno(TRI_ERROR_SYS_ERROR);
}
} else {
TRI_ASSERT(result == 0);
break;
}
}
TRI_Free(TRI_CORE_MEM_ZONE, fullPath);
fclose(fout);
}
unzCloseCurrentFile(uf);
return TRI_ERROR_NO_ERROR;
}
int TRI_ZipFile(char const* filename, char const* dir,
std::vector<std::string> const& files, char const* password) {
void* buffer;
#ifdef USEWIN32IOAPI
zlib_filefunc64_def ffunc;
#endif
if (TRI_ExistsFile(filename)) {
return TRI_ERROR_CANNOT_OVERWRITE_FILE;
}
int bufferSize = 16384;
buffer = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, (size_t)bufferSize, false);
if (buffer == nullptr) {
return TRI_ERROR_OUT_OF_MEMORY;
}
#ifdef USEWIN32IOAPI
fill_win32_filefunc64A(&ffunc);
zipFile zf = zipOpen2_64(filename, 0, NULL, &ffunc);
#else
zipFile zf = zipOpen64(filename, 0);
#endif
if (zf == nullptr) {
TRI_Free(TRI_UNKNOWN_MEM_ZONE, buffer);
return ZIP_ERRNO;
}
int res = TRI_ERROR_NO_ERROR;
size_t n = files.size();
for (size_t i = 0; i < n; ++i) {
std::string fullfile;
if (*dir == '\0') {
fullfile = files[i];
} else {
fullfile = arangodb::basics::FileUtils::buildFilename(dir, files[i]);
}
zip_fileinfo zi;
memset(&zi, 0, sizeof(zi));
uint32_t crc;
res = TRI_Crc32File(fullfile.c_str(), &crc);
if (res != TRI_ERROR_NO_ERROR) {
break;
}
int isLarge = (TRI_SizeFile(files[i].c_str()) > 0xFFFFFFFFLL);
char const* saveName = files[i].c_str();
while (*saveName == '\\' || *saveName == '/') {
++saveName;
}
if (zipOpenNewFileInZip3_64(
zf, saveName, &zi, NULL, 0, NULL, 0, NULL, /* comment*/
Z_DEFLATED, Z_DEFAULT_COMPRESSION, 0, -MAX_WBITS, DEF_MEM_LEVEL,
Z_DEFAULT_STRATEGY, password, (unsigned long)crc,
isLarge) != ZIP_OK) {
res = TRI_ERROR_INTERNAL;
break;
}
FILE* fin = fopen(fullfile.c_str(), "rb");
if (fin == nullptr) {
break;
}
while (true) {
int sizeRead = (int)fread(buffer, 1, bufferSize, fin);
if (sizeRead < bufferSize) {
if (feof(fin) == 0) {
res = TRI_set_errno(TRI_ERROR_SYS_ERROR);
break;
}
}
if (sizeRead > 0) {
res = zipWriteInFileInZip(zf, buffer, sizeRead);
if (res != 0) {
break;
}
} else /* if (sizeRead <= 0) */ {
break;
}
}
fclose(fin);
zipCloseFileInZip(zf);
if (res != TRI_ERROR_NO_ERROR) {
break;
}
}
zipClose(zf, NULL);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, buffer);
return res;
}
