Beispiel #1
0
TEST_F(GPWriterTest, ReGenerateKeyName) {
    string url = "https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";

    S3Params p = InitConfig(url + " config=data/s3test.conf");
    p.setAutoCompress(false);

    MockS3RESTfulService mockRESTfulService(p);
    MockGPWriter gpwriter(p, &mockRESTfulService);

    EXPECT_CALL(mockRESTfulService, head(_, _)).WillOnce(Return(200)).WillOnce(Return(404));

    uint8_t xml[] =
        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
        "<InitiateMultipartUploadResult"
        " xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
        "<Bucket>example-bucket</Bucket>"
        "<Key>example-object</Key>"
        "<UploadId>VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA</UploadId>"
        "</InitiateMultipartUploadResult>";
    vector<uint8_t> raw(xml, xml + sizeof(xml) - 1);
    Response response(RESPONSE_OK, raw);

    EXPECT_CALL(mockRESTfulService, post(_, _, vector<uint8_t>())).WillOnce(Return(response));

    gpwriter.open(p);

    // expect the restfulService->head() was called twice
}
Beispiel #2
0
void CheckEssentialConfig(const S3Params& params) {
    if (params.getCred().accessID.empty()) {
        S3_CHECK_OR_DIE(false, S3ConfigError, "\"FATAL: access id not set\"", "accessid");
    }

    if (params.getCred().secret.empty()) {
        S3_CHECK_OR_DIE(false, S3ConfigError, "\"FATAL: secret id not set\"", "secret");
    }

    if (s3ext_segnum <= 0) {
        S3_CHECK_OR_DIE(false, S3ConfigError, "\"FATAL: segment info is invalid\"", "segment");
    }
}
Beispiel #3
0
S3Params S3BucketReader::constructReaderParams(BucketContent& key) {
    // encode the key name but leave the "/"
    // "/encoded_path/encoded_name"
    string keyEncoded = UriEncode(key.getName());
    FindAndReplace(keyEncoded, "%2F", "/");

    S3Params readerParams = this->params.setPrefix(keyEncoded);

    readerParams.setKeySize(key.getSize());

    S3DEBUG("key: %s, size: %" PRIu64, readerParams.getS3Url().getFullUrlForCurl().c_str(),
            readerParams.getKeySize());
    return readerParams;
}
Beispiel #4
0
// invoked by s3_export(), need to be exception safe
GPWriter* writer_init(const char* url_with_options, const char* format) {
    GPWriter* writer = NULL;
    s3extErrorMessage.clear();

    try {
        if (!url_with_options) {
            return NULL;
        }

        string urlWithOptions(url_with_options);

        S3Params params = InitConfig(urlWithOptions);

        InitRemoteLog();

        // Prepare memory to be used for thread chunk buffer.
        PrepareS3MemContext(params);

        string extName = params.isAutoCompress() ? string(format) + ".gz" : format;
        writer = new GPWriter(params, extName);
        if (writer == NULL) {
            return NULL;
        }

        memoryContextHolder = new S3MemoryContext(params.getMemoryContext());

        writer->open(params);
        return writer;
    } catch (S3Exception& e) {
        if (writer != NULL) {
            delete writer;
            delete memoryContextHolder;
        }
        s3extErrorMessage =
            "writer_init caught a " + e.getType() + " exception: " + e.getFullMessage();
        S3ERROR("writer_init caught %s: %s", e.getType().c_str(), s3extErrorMessage.c_str());
        return NULL;
    } catch (...) {
        if (writer != NULL) {
            delete writer;
            delete memoryContextHolder;
        }
        S3ERROR("Caught an unexpected exception.");
        s3extErrorMessage = "Caught an unexpected exception.";
        return NULL;
    }
}
Beispiel #5
0
void S3KeyReader::open(const S3Params& params) {
    S3_CHECK_OR_DIE(this->s3Interface != NULL, S3RuntimeError, "s3Interface must not be NULL");

    this->sharedError = false;

    this->numOfChunks = params.getNumOfChunks();
    S3_CHECK_OR_DIE(this->numOfChunks > 0, S3RuntimeError, "numOfChunks must not be zero");

    this->offsetMgr.setKeySize(params.getKeySize());
    this->offsetMgr.setChunkSize(params.getChunkSize());

    S3_CHECK_OR_DIE(params.getChunkSize() > 0, S3RuntimeError,
                    "chunk size must be greater than zero");

    this->chunkBuffers.reserve(this->numOfChunks);

    for (uint64_t i = 0; i < this->numOfChunks; i++) {
        this->chunkBuffers.emplace_back(params.getS3Url(), *this, params.getMemoryContext());
    }

    for (uint64_t i = 0; i < this->numOfChunks; i++) {
        this->chunkBuffers[i].setS3InterfaceService(this->s3Interface);

        pthread_t thread;
        pthread_create(&thread, NULL, DownloadThreadFunc, &this->chunkBuffers[i]);
        this->threads.push_back(thread);
    }
}
Beispiel #6
0
S3RESTfulService::S3RESTfulService(const S3Params &params)
    : s3MemContext(const_cast<S3MemoryContext &>(params.getMemoryContext())) {
    // This function is not thread safe, must NOT call it when any other
    // threads are running, that is, do NOT put it in threads.
    curl_global_init(CURL_GLOBAL_ALL);

    this->lowSpeedLimit = params.getLowSpeedLimit();
    this->lowSpeedTime = params.getLowSpeedTime();
    this->debugCurl = params.isDebugCurl();
    this->chunkBufferSize = params.getChunkSize();
    this->verifyCert = params.isVerifyCert();
}
Beispiel #7
0
void S3CommonReader::open(const S3Params &params) {
    this->keyReader.setS3InterfaceService(s3InterfaceService);

    S3CompressionType compressionType = s3InterfaceService->checkCompressionType(params.getS3Url());

    switch (compressionType) {
        case S3_COMPRESSION_GZIP:
            this->upstreamReader = &this->decompressReader;
            this->decompressReader.setReader(&this->keyReader);
            break;
        case S3_COMPRESSION_PLAIN:
            this->upstreamReader = &this->keyReader;
            break;
        default:
            S3_CHECK_OR_DIE(false, S3RuntimeError, "unknown file type");
    };

    this->upstreamReader->open(params);
}
S3Params S3BucketReader::constructReaderParams(BucketContent& key) {
    S3Params readerParams = this->params;

    // encode the key name but leave the "/"
    // "/encoded_path/encoded_name"
    string keyEncoded = uri_encode(key.getName());
    find_replace(keyEncoded, "%2F", "/");

    readerParams.setKeyUrl(this->getKeyURL(keyEncoded));
    readerParams.setRegion(this->region);
    readerParams.setKeySize(key.getSize());

    S3DEBUG("key: %s, size: %" PRIu64, readerParams.getKeyUrl().c_str(), readerParams.getKeySize());
    return readerParams;
}