void S3KeyReader::open(const S3Params& params) {
S3_CHECK_OR_DIE(this->s3Interface != NULL, S3RuntimeError, "s3Interface must not be NULL");
this->sharedError = false;
this->numOfChunks = params.getNumOfChunks();
S3_CHECK_OR_DIE(this->numOfChunks > 0, S3RuntimeError, "numOfChunks must not be zero");
this->offsetMgr.setKeySize(params.getKeySize());
this->offsetMgr.setChunkSize(params.getChunkSize());
S3_CHECK_OR_DIE(params.getChunkSize() > 0, S3RuntimeError,
"chunk size must be greater than zero");
this->chunkBuffers.reserve(this->numOfChunks);
for (uint64_t i = 0; i < this->numOfChunks; i++) {
this->chunkBuffers.emplace_back(params.getS3Url(), *this, params.getMemoryContext());
}
for (uint64_t i = 0; i < this->numOfChunks; i++) {
this->chunkBuffers[i].setS3InterfaceService(this->s3Interface);
pthread_t thread;
pthread_create(&thread, NULL, DownloadThreadFunc, &this->chunkBuffers[i]);
this->threads.push_back(thread);
}
}
S3RESTfulService::S3RESTfulService(const S3Params ¶ms)
: s3MemContext(const_cast<S3MemoryContext &>(params.getMemoryContext())) {
// This function is not thread safe, must NOT call it when any other
// threads are running, that is, do NOT put it in threads.
curl_global_init(CURL_GLOBAL_ALL);
this->lowSpeedLimit = params.getLowSpeedLimit();
this->lowSpeedTime = params.getLowSpeedTime();
this->debugCurl = params.isDebugCurl();
this->chunkBufferSize = params.getChunkSize();
this->verifyCert = params.isVerifyCert();
}
// invoked by s3_export(), need to be exception safe
GPWriter* writer_init(const char* url_with_options, const char* format) {
GPWriter* writer = NULL;
s3extErrorMessage.clear();
try {
if (!url_with_options) {
return NULL;
}
string urlWithOptions(url_with_options);
S3Params params = InitConfig(urlWithOptions);
InitRemoteLog();
// Prepare memory to be used for thread chunk buffer.
PrepareS3MemContext(params);
string extName = params.isAutoCompress() ? string(format) + ".gz" : format;
writer = new GPWriter(params, extName);
if (writer == NULL) {
return NULL;
}
memoryContextHolder = new S3MemoryContext(params.getMemoryContext());
writer->open(params);
return writer;
} catch (S3Exception& e) {
if (writer != NULL) {
delete writer;
delete memoryContextHolder;
}
s3extErrorMessage =
"writer_init caught a " + e.getType() + " exception: " + e.getFullMessage();
S3ERROR("writer_init caught %s: %s", e.getType().c_str(), s3extErrorMessage.c_str());
return NULL;
} catch (...) {
if (writer != NULL) {
delete writer;
delete memoryContextHolder;
}
S3ERROR("Caught an unexpected exception.");
s3extErrorMessage = "Caught an unexpected exception.";
return NULL;
}
}