void S3KeyReader::open(const S3Params& params) {
S3_CHECK_OR_DIE(this->s3Interface != NULL, S3RuntimeError, "s3Interface must not be NULL");
this->sharedError = false;
this->numOfChunks = params.getNumOfChunks();
S3_CHECK_OR_DIE(this->numOfChunks > 0, S3RuntimeError, "numOfChunks must not be zero");
this->offsetMgr.setKeySize(params.getKeySize());
this->offsetMgr.setChunkSize(params.getChunkSize());
S3_CHECK_OR_DIE(params.getChunkSize() > 0, S3RuntimeError,
"chunk size must be greater than zero");
this->chunkBuffers.reserve(this->numOfChunks);
for (uint64_t i = 0; i < this->numOfChunks; i++) {
this->chunkBuffers.emplace_back(params.getS3Url(), *this, params.getMemoryContext());
}
for (uint64_t i = 0; i < this->numOfChunks; i++) {
this->chunkBuffers[i].setS3InterfaceService(this->s3Interface);
pthread_t thread;
pthread_create(&thread, NULL, DownloadThreadFunc, &this->chunkBuffers[i]);
this->threads.push_back(thread);
}
}
S3Params S3BucketReader::constructReaderParams(BucketContent& key) {
// encode the key name but leave the "/"
// "/encoded_path/encoded_name"
string keyEncoded = UriEncode(key.getName());
FindAndReplace(keyEncoded, "%2F", "/");
S3Params readerParams = this->params.setPrefix(keyEncoded);
readerParams.setKeySize(key.getSize());
S3DEBUG("key: %s, size: %" PRIu64, readerParams.getS3Url().getFullUrlForCurl().c_str(),
readerParams.getKeySize());
return readerParams;
}
void S3CommonReader::open(const S3Params ¶ms) {
this->keyReader.setS3InterfaceService(s3InterfaceService);
S3CompressionType compressionType = s3InterfaceService->checkCompressionType(params.getS3Url());
switch (compressionType) {
case S3_COMPRESSION_GZIP:
this->upstreamReader = &this->decompressReader;
this->decompressReader.setReader(&this->keyReader);
break;
case S3_COMPRESSION_PLAIN:
this->upstreamReader = &this->keyReader;
break;
default:
S3_CHECK_OR_DIE(false, S3RuntimeError, "unknown file type");
};
this->upstreamReader->open(params);
}