RandomDeviceCombined (string path) : fd(0), pos(0), rseed(0) { fd = TRI_OPEN(path.c_str(), O_RDONLY); if (fd < 0) { THROW_INTERNAL_ERROR("cannot open random source '" + path + "'"); } // .............................................................................. // Set the random number generator file to be non-blocking (not for windows) // .............................................................................. { #ifdef _WIN32 abort(); #else long flags = fcntl(fd, F_GETFL, 0); bool ok = (flags >= 0); if (ok) { flags = fcntl(fd, F_SETFL, flags | O_NONBLOCK); ok = (flags >= 0); } if (! ok) { THROW_INTERNAL_ERROR("cannot switch random source '" + path + "' to non-blocking"); } #endif } fillBuffer(); }
RandomDeviceDirect (string path) : fd(1), pos(0) { fd = TRI_OPEN(path.c_str(), O_RDONLY); if (fd < 0) { THROW_INTERNAL_ERROR("cannot open random source '" + path + "'"); } fillBuffer(); }
RandomDeviceCombined (string path) : interval(0, UINT32_MAX), randomGenerator(mersenneDevice, interval), fd(0), pos(0), rseed(0) { fd = TRI_OPEN(path.c_str(), O_RDONLY); if (fd < 0) { THROW_INTERNAL_ERROR("cannot open random source '" + path + "'"); } if (! TRI_SetNonBlockingSocket(fd)) { THROW_INTERNAL_ERROR("cannot switch random source '" + path + "' to non-blocking"); } fillBuffer(); }
string slurp (string const& filename) { int fd = TRI_OPEN(filename.c_str(), O_RDONLY); if (fd == -1) { TRI_set_errno(errno); THROW_FILE_OPEN_ERROR("open", filename, "O_RDONLY", errno); } char buffer[10240]; StringBuffer result(TRI_CORE_MEM_ZONE); while (true) { ssize_t n = TRI_READ(fd, buffer, sizeof(buffer)); if (n == 0) { break; } if (n < 0) { TRI_set_errno(TRI_ERROR_SYS_ERROR); LOG_TRACE("read failed for '%s' with %s and result %d on fd %d", filename.c_str(), strerror(errno), (int) n, fd); TRI_CLOSE(fd); THROW_FILE_FUNC_ERROR("read", "", errno); } result.appendText(buffer, n); } TRI_CLOSE(fd); string r(result.c_str(), result.length()); return r; }
void slurp (string const& filename, StringBuffer& result) { int fd = TRI_OPEN(filename.c_str(), O_RDONLY); if (fd == -1) { THROW_FILE_OPEN_ERROR("open", filename, "O_RDONLY", errno); } // reserve space in the output buffer off_t fileSize = size(filename); if (fileSize > 0) { result.reserve((size_t) fileSize); } char buffer[10240]; while (true) { ssize_t n = TRI_READ(fd, buffer, sizeof(buffer)); if (n == 0) { break; } if (n < 0) { TRI_CLOSE(fd); LOG_TRACE("read failed for '%s' with %s and result %d on fd %d", filename.c_str(), strerror(errno), (int) n, fd); THROW_FILE_FUNC_ERROR("read", "", errno); } result.appendText(buffer, n); } TRI_CLOSE(fd); }
bool ImportHelper::importDelimited (string const& collectionName, string const& fileName, DelimitedImportType typeImport) { _collectionName = collectionName; _firstLine = ""; _numberLines = 0; _numberOk = 0; _numberError = 0; _outputBuffer.clear(); _lineBuffer.clear(); _errorMessage = ""; _hasError = false; // read and convert int fd; int64_t totalLength; if (fileName == "-") { // we don't have a filesize totalLength = 0; fd = STDIN_FILENO; } else { // read filesize totalLength = TRI_SizeFile(fileName.c_str()); fd = TRI_OPEN(fileName.c_str(), O_RDONLY); if (fd < 0) { _errorMessage = TRI_LAST_ERROR_STR; return false; } } // progress display control variables int64_t totalRead = 0; double nextProgress = ProgressStep; size_t separatorLength; char* separator = TRI_UnescapeUtf8StringZ(TRI_UNKNOWN_MEM_ZONE, _separator.c_str(), _separator.size(), &separatorLength); if (separator == 0) { if (fd != STDIN_FILENO) { TRI_CLOSE(fd); } _errorMessage = "out of memory"; return false; } TRI_csv_parser_t parser; TRI_InitCsvParser(&parser, TRI_UNKNOWN_MEM_ZONE, ProcessCsvBegin, ProcessCsvAdd, ProcessCsvEnd); TRI_SetSeparatorCsvParser(&parser, separator[0]); // in csv, we'll use the quote char if set // in tsv, we do not use the quote char if (typeImport == ImportHelper::CSV && _quote.size() > 0) { TRI_SetQuoteCsvParser(&parser, _quote[0], true); } else { TRI_SetQuoteCsvParser(&parser, '\0', false); } parser._dataAdd = this; _rowOffset = 0; _rowsRead = 0; char buffer[32768]; while (! _hasError) { ssize_t n = TRI_READ(fd, buffer, sizeof(buffer)); if (n < 0) { TRI_Free(TRI_UNKNOWN_MEM_ZONE, separator); TRI_DestroyCsvParser(&parser); if (fd != STDIN_FILENO) { TRI_CLOSE(fd); } _errorMessage = TRI_LAST_ERROR_STR; return false; } else if (n == 0) { break; } totalRead += (int64_t) n; reportProgress(totalLength, totalRead, nextProgress); TRI_ParseCsvString2(&parser, buffer, n); } if (_outputBuffer.length() > 0) { sendCsvBuffer(); } TRI_DestroyCsvParser(&parser); TRI_Free(TRI_UNKNOWN_MEM_ZONE, separator); if (fd != STDIN_FILENO) { TRI_CLOSE(fd); } _outputBuffer.clear(); return !_hasError; }
bool ImportHelper::importJson (const string& collectionName, const string& fileName) { _collectionName = collectionName; _firstLine = ""; _numberLines = 0; _numberOk = 0; _numberError = 0; _outputBuffer.clear(); _errorMessage = ""; _hasError = false; // read and convert int fd; int64_t totalLength; if (fileName == "-") { // we don't have a filesize totalLength = 0; fd = STDIN_FILENO; } else { // read filesize totalLength = TRI_SizeFile(fileName.c_str()); fd = TRI_OPEN(fileName.c_str(), O_RDONLY); if (fd < 0) { _errorMessage = TRI_LAST_ERROR_STR; return false; } } bool isArray = false; bool checkedFront = false; // progress display control variables int64_t totalRead = 0; double nextProgress = ProgressStep; static const int BUFFER_SIZE = 32768; while (! _hasError) { // reserve enough room to read more data if (_outputBuffer.reserve(BUFFER_SIZE) == TRI_ERROR_OUT_OF_MEMORY) { _errorMessage = TRI_errno_string(TRI_ERROR_OUT_OF_MEMORY); if (fd != STDIN_FILENO) { TRI_CLOSE(fd); } return false; } // read directly into string buffer ssize_t n = TRI_READ(fd, _outputBuffer.end(), BUFFER_SIZE - 1); if (n < 0) { _errorMessage = TRI_LAST_ERROR_STR; if (fd != STDIN_FILENO) { TRI_CLOSE(fd); } return false; } else if (n == 0) { // we're done break; } // adjust size of the buffer by the size of the chunk we just read _outputBuffer.increaseLength(n); if (! checkedFront) { // detect the import file format (single lines with individual JSON objects // or a JSON array with all documents) char const* p = _outputBuffer.begin(); char const* e = _outputBuffer.end(); while (p < e && (*p == ' ' || *p == '\r' || *p == '\n' || *p == '\t' || *p == '\f' || *p == '\b')) { ++p; } isArray = (*p == '['); checkedFront = true; } totalRead += (int64_t) n; reportProgress(totalLength, totalRead, nextProgress); if (_outputBuffer.length() > _maxUploadSize) { if (isArray) { if (fd != STDIN_FILENO) { TRI_CLOSE(fd); } _errorMessage = "import file is too big. please increase the value of --batch-size (currently " + StringUtils::itoa(_maxUploadSize) + ")"; return false; } // send all data before last '\n' char const* first = _outputBuffer.c_str(); char* pos = (char*) memrchr(first, '\n', _outputBuffer.length()); if (pos != 0) { size_t len = pos - first + 1; sendJsonBuffer(first, len, isArray); _outputBuffer.erase_front(len); } } } if (_outputBuffer.length() > 0) { sendJsonBuffer(_outputBuffer.c_str(), _outputBuffer.length(), isArray); } _numberLines = _numberError + _numberOk; if (fd != STDIN_FILENO) { TRI_CLOSE(fd); } _outputBuffer.clear(); return ! _hasError; }
bool ImportHelper::importJson (const string& collectionName, const string& fileName) { _collectionName = collectionName; _firstLine = ""; _numberLines = 0; _numberOk = 0; _numberError = 0; _outputBuffer.clear(); _errorMessage = ""; _hasError = false; // read and convert int fd; int64_t totalLength; if (fileName == "-") { // we don't have a filesize totalLength = 0; fd = STDIN_FILENO; } else { // read filesize totalLength = TRI_SizeFile(fileName.c_str()); fd = TRI_OPEN(fileName.c_str(), O_RDONLY); if (fd < 0) { _errorMessage = TRI_LAST_ERROR_STR; return false; } } char buffer[32768]; bool isArray = false; bool checkedFront = false; // progress display control variables int64_t totalRead = 0; double nextProgress = ProgressStep; while (! _hasError) { ssize_t n = TRI_READ(fd, buffer, sizeof(buffer)); if (n < 0) { _errorMessage = TRI_LAST_ERROR_STR; if (fd != STDIN_FILENO) { TRI_CLOSE(fd); } return false; } else if (n == 0) { // we're done break; } if (! checkedFront) { // detect the import file format (single lines with individual JSON objects // or a JSON array with all documents) const string firstChar = StringUtils::lTrim(string(buffer, n), "\r\n\t\f\b ").substr(0, 1); isArray = (firstChar == "["); checkedFront = true; } _outputBuffer.appendText(buffer, n); totalRead += (int64_t) n; reportProgress(totalLength, totalRead, nextProgress); if (_outputBuffer.length() > _maxUploadSize) { if (isArray) { if (fd != STDIN_FILENO) { TRI_CLOSE(fd); } _errorMessage = "import file is too big."; return false; } // send all data before last '\n' const char* first = _outputBuffer.c_str(); char* pos = (char*) memrchr(first, '\n', _outputBuffer.length()); if (pos != 0) { size_t len = pos - first + 1; sendJsonBuffer(first, len, isArray); _outputBuffer.erase_front(len); } } } if (_outputBuffer.length() > 0) { sendJsonBuffer(_outputBuffer.c_str(), _outputBuffer.length(), isArray); } _numberLines = _numberError + _numberOk; if (fd != STDIN_FILENO) { TRI_CLOSE(fd); } _outputBuffer.clear(); return ! _hasError; }