Пример #1
0
void S3CommonReader::open(const ReaderParams &params) {
    this->keyReader.setS3interface(s3service);
    S3CompressionType compressionType =
        s3service->checkCompressionType(params.getKeyUrl(), params.getRegion(), params.getCred());
    switch (compressionType) {
        case S3_COMPRESSION_GZIP:
            this->upstreamReader = &this->decompressReader;
            this->decompressReader.setReader(&this->keyReader);
            break;
        case S3_COMPRESSION_PLAIN:
            this->upstreamReader = &this->keyReader;
            break;
        default:
            CHECK_OR_DIE_MSG(false, "%s", "unknown file type");
    };
    this->upstreamReader->open(params);
}
Пример #2
0
void S3KeyReader::open(const ReaderParams& params) {
    this->sharedError = false;
    this->sharedErrorMessage.clear();

    this->numOfChunks = params.getNumOfChunks();
    CHECK_OR_DIE_MSG(this->numOfChunks > 0, "%s", "numOfChunks must not be zero");

    this->region = params.getRegion();
    this->credential = params.getCred();

    this->offsetMgr.setKeySize(params.getKeySize());
    this->offsetMgr.setChunkSize(params.getChunkSize());

    CHECK_OR_DIE_MSG(params.getChunkSize() > 0, "%s", "chunk size must be greater than zero");

    for (uint64_t i = 0; i < this->numOfChunks; i++) {
        // when vector reallocate memory, it will copy object.
        // chunkData must be initialized after all copy.
        this->chunkBuffers.push_back(ChunkBuffer(params.getKeyUrl(), *this));
    }

    for (uint64_t i = 0; i < this->numOfChunks; i++) {
        this->chunkBuffers[i].init();
        this->chunkBuffers[i].setS3interface(this->s3interface);

        pthread_t thread;
        pthread_create(&thread, NULL, DownloadThreadFunc, &this->chunkBuffers[i]);
        this->threads.push_back(thread);
    }

    return;
}