off_t size (string const& path) {
int64_t result = TRI_SizeFile(path.c_str());
if (result < 0) {
return (off_t) 0;
}
return (off_t) result;
}
int TRI_ZipFile (const char* filename,
const char* dir,
TRI_vector_string_t const* files,
const char* password) {
void* buffer;
size_t bufferSize;
zipFile zf;
#ifdef USEWIN32IOAPI
zlib_filefunc64_def ffunc;
#endif
size_t i, n;
int res;
if (TRI_ExistsFile(filename)) {
return TRI_ERROR_CANNOT_OVERWRITE_FILE;
}
bufferSize = 16384;
buffer = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, bufferSize, false);
if (buffer == NULL) {
return TRI_ERROR_OUT_OF_MEMORY;
}
#ifdef USEWIN32IOAPI
fill_win32_filefunc64A(&ffunc);
zf = zipOpen2_64(filename, 0, NULL, &ffunc);
#else
zf = zipOpen64(filename, 0);
#endif
if (zf == NULL) {
TRI_Free(TRI_UNKNOWN_MEM_ZONE, buffer);
return ZIP_ERRNO;
}
res = TRI_ERROR_NO_ERROR;
n = files->_length;
for (i = 0; i < n; ++i) {
FILE* fin;
char* file;
char* fullfile;
char* saveName;
zip_fileinfo zi;
uint32_t crc;
int isLarge;
file = TRI_AtVectorString(files, i);
if (*dir == '\0') {
fullfile = TRI_DuplicateString(file);
}
else {
fullfile = TRI_Concatenate2File(dir, file);
}
memset(&zi, 0, sizeof(zi));
res = TRI_Crc32File(fullfile, &crc);
if (res != TRI_ERROR_NO_ERROR) {
break;
}
isLarge = (TRI_SizeFile(file) > 0xFFFFFFFFLL);
saveName = file;
while (*saveName == '\\' || *saveName == '/') {
++saveName;
}
if (zipOpenNewFileInZip3_64(zf,
saveName,
&zi,
NULL,
0,
NULL,
0,
NULL, /* comment*/
Z_DEFLATED,
Z_DEFAULT_COMPRESSION,
0,
-MAX_WBITS,
DEF_MEM_LEVEL,
Z_DEFAULT_STRATEGY,
password,
(unsigned long) crc,
isLarge) != ZIP_OK) {
}
fin = fopen(fullfile, "rb");
TRI_FreeString(TRI_CORE_MEM_ZONE, fullfile);
if (fin == NULL) {
break;
}
while (true) {
int sizeRead;
sizeRead = (int) fread(buffer, 1, bufferSize, fin);
if (sizeRead < bufferSize) {
if (feof(fin) == 0) {
res = TRI_set_errno(TRI_ERROR_SYS_ERROR);
break;
}
}
if (sizeRead > 0) {
res = zipWriteInFileInZip(zf, buffer, sizeRead);
if (res != 0) {
break;
}
}
else if (sizeRead <= 0) {
break;
}
}
fclose(fin);
zipCloseFileInZip(zf);
if (res != TRI_ERROR_NO_ERROR) {
break;
}
}
zipClose(zf, NULL);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, buffer);
return res;
}
bool ImportHelper::importDelimited (string const& collectionName,
string const& fileName,
DelimitedImportType typeImport) {
_collectionName = collectionName;
_firstLine = "";
_numberLines = 0;
_numberOk = 0;
_numberError = 0;
_outputBuffer.clear();
_lineBuffer.clear();
_errorMessage = "";
_hasError = false;
// read and convert
int fd;
int64_t totalLength;
if (fileName == "-") {
// we don't have a filesize
totalLength = 0;
fd = STDIN_FILENO;
}
else {
// read filesize
totalLength = TRI_SizeFile(fileName.c_str());
fd = TRI_OPEN(fileName.c_str(), O_RDONLY);
if (fd < 0) {
_errorMessage = TRI_LAST_ERROR_STR;
return false;
}
}
// progress display control variables
int64_t totalRead = 0;
double nextProgress = ProgressStep;
size_t separatorLength;
char* separator = TRI_UnescapeUtf8StringZ(TRI_UNKNOWN_MEM_ZONE, _separator.c_str(), _separator.size(), &separatorLength);
if (separator == 0) {
if (fd != STDIN_FILENO) {
TRI_CLOSE(fd);
}
_errorMessage = "out of memory";
return false;
}
TRI_csv_parser_t parser;
TRI_InitCsvParser(&parser,
TRI_UNKNOWN_MEM_ZONE,
ProcessCsvBegin,
ProcessCsvAdd,
ProcessCsvEnd);
TRI_SetSeparatorCsvParser(&parser, separator[0]);
// in csv, we'll use the quote char if set
// in tsv, we do not use the quote char
if (typeImport == ImportHelper::CSV && _quote.size() > 0) {
TRI_SetQuoteCsvParser(&parser, _quote[0], true);
}
else {
TRI_SetQuoteCsvParser(&parser, '\0', false);
}
parser._dataAdd = this;
_rowOffset = 0;
_rowsRead = 0;
char buffer[32768];
while (! _hasError) {
ssize_t n = TRI_READ(fd, buffer, sizeof(buffer));
if (n < 0) {
TRI_Free(TRI_UNKNOWN_MEM_ZONE, separator);
TRI_DestroyCsvParser(&parser);
if (fd != STDIN_FILENO) {
TRI_CLOSE(fd);
}
_errorMessage = TRI_LAST_ERROR_STR;
return false;
}
else if (n == 0) {
break;
}
totalRead += (int64_t) n;
reportProgress(totalLength, totalRead, nextProgress);
TRI_ParseCsvString2(&parser, buffer, n);
}
if (_outputBuffer.length() > 0) {
sendCsvBuffer();
}
TRI_DestroyCsvParser(&parser);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, separator);
if (fd != STDIN_FILENO) {
TRI_CLOSE(fd);
}
_outputBuffer.clear();
return !_hasError;
}
bool ImportHelper::importJson (const string& collectionName, const string& fileName) {
_collectionName = collectionName;
_firstLine = "";
_numberLines = 0;
_numberOk = 0;
_numberError = 0;
_outputBuffer.clear();
_errorMessage = "";
_hasError = false;
// read and convert
int fd;
int64_t totalLength;
if (fileName == "-") {
// we don't have a filesize
totalLength = 0;
fd = STDIN_FILENO;
}
else {
// read filesize
totalLength = TRI_SizeFile(fileName.c_str());
fd = TRI_OPEN(fileName.c_str(), O_RDONLY);
if (fd < 0) {
_errorMessage = TRI_LAST_ERROR_STR;
return false;
}
}
bool isArray = false;
bool checkedFront = false;
// progress display control variables
int64_t totalRead = 0;
double nextProgress = ProgressStep;
static const int BUFFER_SIZE = 32768;
while (! _hasError) {
// reserve enough room to read more data
if (_outputBuffer.reserve(BUFFER_SIZE) == TRI_ERROR_OUT_OF_MEMORY) {
_errorMessage = TRI_errno_string(TRI_ERROR_OUT_OF_MEMORY);
if (fd != STDIN_FILENO) {
TRI_CLOSE(fd);
}
return false;
}
// read directly into string buffer
ssize_t n = TRI_READ(fd, _outputBuffer.end(), BUFFER_SIZE - 1);
if (n < 0) {
_errorMessage = TRI_LAST_ERROR_STR;
if (fd != STDIN_FILENO) {
TRI_CLOSE(fd);
}
return false;
}
else if (n == 0) {
// we're done
break;
}
// adjust size of the buffer by the size of the chunk we just read
_outputBuffer.increaseLength(n);
if (! checkedFront) {
// detect the import file format (single lines with individual JSON objects
// or a JSON array with all documents)
char const* p = _outputBuffer.begin();
char const* e = _outputBuffer.end();
while (p < e &&
(*p == ' ' || *p == '\r' || *p == '\n' || *p == '\t' || *p == '\f' || *p == '\b')) {
++p;
}
isArray = (*p == '[');
checkedFront = true;
}
totalRead += (int64_t) n;
reportProgress(totalLength, totalRead, nextProgress);
if (_outputBuffer.length() > _maxUploadSize) {
if (isArray) {
if (fd != STDIN_FILENO) {
TRI_CLOSE(fd);
}
_errorMessage = "import file is too big. please increase the value of --batch-size (currently " + StringUtils::itoa(_maxUploadSize) + ")";
return false;
}
// send all data before last '\n'
char const* first = _outputBuffer.c_str();
char* pos = (char*) memrchr(first, '\n', _outputBuffer.length());
if (pos != 0) {
size_t len = pos - first + 1;
sendJsonBuffer(first, len, isArray);
_outputBuffer.erase_front(len);
}
}
}
if (_outputBuffer.length() > 0) {
sendJsonBuffer(_outputBuffer.c_str(), _outputBuffer.length(), isArray);
}
_numberLines = _numberError + _numberOk;
if (fd != STDIN_FILENO) {
TRI_CLOSE(fd);
}
_outputBuffer.clear();
return ! _hasError;
}
bool ImportHelper::importJson (const string& collectionName, const string& fileName) {
_collectionName = collectionName;
_firstLine = "";
_numberLines = 0;
_numberOk = 0;
_numberError = 0;
_outputBuffer.clear();
_errorMessage = "";
_hasError = false;
// read and convert
int fd;
int64_t totalLength;
if (fileName == "-") {
// we don't have a filesize
totalLength = 0;
fd = STDIN_FILENO;
}
else {
// read filesize
totalLength = TRI_SizeFile(fileName.c_str());
fd = TRI_OPEN(fileName.c_str(), O_RDONLY);
if (fd < 0) {
_errorMessage = TRI_LAST_ERROR_STR;
return false;
}
}
char buffer[32768];
bool isArray = false;
bool checkedFront = false;
// progress display control variables
int64_t totalRead = 0;
double nextProgress = ProgressStep;
while (! _hasError) {
ssize_t n = TRI_READ(fd, buffer, sizeof(buffer));
if (n < 0) {
_errorMessage = TRI_LAST_ERROR_STR;
if (fd != STDIN_FILENO) {
TRI_CLOSE(fd);
}
return false;
}
else if (n == 0) {
// we're done
break;
}
if (! checkedFront) {
// detect the import file format (single lines with individual JSON objects
// or a JSON array with all documents)
const string firstChar = StringUtils::lTrim(string(buffer, n), "\r\n\t\f\b ").substr(0, 1);
isArray = (firstChar == "[");
checkedFront = true;
}
_outputBuffer.appendText(buffer, n);
totalRead += (int64_t) n;
reportProgress(totalLength, totalRead, nextProgress);
if (_outputBuffer.length() > _maxUploadSize) {
if (isArray) {
if (fd != STDIN_FILENO) {
TRI_CLOSE(fd);
}
_errorMessage = "import file is too big.";
return false;
}
// send all data before last '\n'
const char* first = _outputBuffer.c_str();
char* pos = (char*) memrchr(first, '\n', _outputBuffer.length());
if (pos != 0) {
size_t len = pos - first + 1;
sendJsonBuffer(first, len, isArray);
_outputBuffer.erase_front(len);
}
}
}
if (_outputBuffer.length() > 0) {
sendJsonBuffer(_outputBuffer.c_str(), _outputBuffer.length(), isArray);
}
_numberLines = _numberError + _numberOk;
if (fd != STDIN_FILENO) {
TRI_CLOSE(fd);
}
_outputBuffer.clear();
return ! _hasError;
}
int TRI_ZipFile(char const* filename, char const* dir,
std::vector<std::string> const& files, char const* password) {
void* buffer;
#ifdef USEWIN32IOAPI
zlib_filefunc64_def ffunc;
#endif
if (TRI_ExistsFile(filename)) {
return TRI_ERROR_CANNOT_OVERWRITE_FILE;
}
int bufferSize = 16384;
buffer = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, (size_t)bufferSize, false);
if (buffer == nullptr) {
return TRI_ERROR_OUT_OF_MEMORY;
}
#ifdef USEWIN32IOAPI
fill_win32_filefunc64A(&ffunc);
zipFile zf = zipOpen2_64(filename, 0, NULL, &ffunc);
#else
zipFile zf = zipOpen64(filename, 0);
#endif
if (zf == nullptr) {
TRI_Free(TRI_UNKNOWN_MEM_ZONE, buffer);
return ZIP_ERRNO;
}
int res = TRI_ERROR_NO_ERROR;
size_t n = files.size();
for (size_t i = 0; i < n; ++i) {
std::string fullfile;
if (*dir == '\0') {
fullfile = files[i];
} else {
fullfile = arangodb::basics::FileUtils::buildFilename(dir, files[i]);
}
zip_fileinfo zi;
memset(&zi, 0, sizeof(zi));
uint32_t crc;
res = TRI_Crc32File(fullfile.c_str(), &crc);
if (res != TRI_ERROR_NO_ERROR) {
break;
}
int isLarge = (TRI_SizeFile(files[i].c_str()) > 0xFFFFFFFFLL);
char const* saveName = files[i].c_str();
while (*saveName == '\\' || *saveName == '/') {
++saveName;
}
if (zipOpenNewFileInZip3_64(
zf, saveName, &zi, NULL, 0, NULL, 0, NULL, /* comment*/
Z_DEFLATED, Z_DEFAULT_COMPRESSION, 0, -MAX_WBITS, DEF_MEM_LEVEL,
Z_DEFAULT_STRATEGY, password, (unsigned long)crc,
isLarge) != ZIP_OK) {
res = TRI_ERROR_INTERNAL;
break;
}
FILE* fin = fopen(fullfile.c_str(), "rb");
if (fin == nullptr) {
break;
}
while (true) {
int sizeRead = (int)fread(buffer, 1, bufferSize, fin);
if (sizeRead < bufferSize) {
if (feof(fin) == 0) {
res = TRI_set_errno(TRI_ERROR_SYS_ERROR);
break;
}
}
if (sizeRead > 0) {
res = zipWriteInFileInZip(zf, buffer, sizeRead);
if (res != 0) {
break;
}
} else /* if (sizeRead <= 0) */ {
break;
}
}
fclose(fin);
zipCloseFileInZip(zf);
if (res != TRI_ERROR_NO_ERROR) {
break;
}
}
zipClose(zf, NULL);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, buffer);
return res;
}