Beispiel #1
0
void S3Reader::getNextDownloader() {
    if (this->filedownloader) {  // reset old downloader
        filedownloader->destroy();
        delete this->filedownloader;
        this->filedownloader = NULL;
    }

    if (this->contentindex >= this->keylist->contents.size()) {
        S3DEBUG("No more files to download");
        return;
    }
    this->filedownloader = new Downloader(this->concurrent_num);

    if (!this->filedownloader) {
        S3ERROR("Failed to create filedownloader");
        return;
    }
    BucketContent *c = this->keylist->contents[this->contentindex];
    string keyurl = this->getKeyURL(c->Key());
    S3DEBUG("key: %s, size: %llu", keyurl.c_str(), c->Size());

    // XXX don't use strdup()
    if (!filedownloader->init(keyurl.c_str(), c->Size(),
                              this->chunksize, &this->cred)) {
        delete this->filedownloader;
        this->filedownloader = NULL;
    } else {  // move to next file
        // for now, every segment downloads its assigned files(mod)
        // better to build a workqueue in case not all segments are available
        this->contentindex += this->segnum;
    }
    return;
}
Beispiel #2
0
S3Params S3BucketReader::constructReaderParams(BucketContent& key) {
    // encode the key name but leave the "/"
    // "/encoded_path/encoded_name"
    string keyEncoded = UriEncode(key.getName());
    FindAndReplace(keyEncoded, "%2F", "/");

    S3Params readerParams = this->params.setPrefix(keyEncoded);

    readerParams.setKeySize(key.getSize());

    S3DEBUG("key: %s, size: %" PRIu64, readerParams.getS3Url().getFullUrlForCurl().c_str(),
            readerParams.getKeySize());
    return readerParams;
}
S3Params S3BucketReader::constructReaderParams(BucketContent& key) {
    S3Params readerParams = this->params;

    // encode the key name but leave the "/"
    // "/encoded_path/encoded_name"
    string keyEncoded = uri_encode(key.getName());
    find_replace(keyEncoded, "%2F", "/");

    readerParams.setKeyUrl(this->getKeyURL(keyEncoded));
    readerParams.setRegion(this->region);
    readerParams.setKeySize(key.getSize());

    S3DEBUG("key: %s, size: %" PRIu64, readerParams.getKeyUrl().c_str(), readerParams.getKeySize());
    return readerParams;
}
TEST(ListBucket, fake) {
    ListBucketResult *r = ListBucket_FakeHTTP(HOSTSTR, BUCKETSTR);

    ASSERT_NE(r, (void *)NULL);
    EXPECT_EQ(r->contents.size(), 16);

    char urlbuf[256];
    vector<BucketContent *>::iterator i;
    for (i = r->contents.begin(); i != r->contents.end(); i++) {
        BucketContent *p = *i;
        sprintf(urlbuf, "http://%s/%s/%s", HOSTSTR, BUCKETSTR,
                p->Key().c_str());
        printf("%s, %llu\n", urlbuf, p->Size());
    }
    delete r;
}
TEST(ListBucket, s3) {
    InitConfig("test/s3.conf", "default");

    S3Credential g_cred = {s3ext_accessid, s3ext_secret};

    ListBucketResult *r =
        ListBucket("https", S3HOST, S3BUCKET, S3PREFIX, g_cred);

    ASSERT_NE(r, (void *)NULL);
    EXPECT_EQ(r->contents.size(), 16);

    char urlbuf[256];
    vector<BucketContent *>::iterator i;
    for (i = r->contents.begin(); i != r->contents.end(); i++) {
        BucketContent *p = *i;
        sprintf(urlbuf, "http://%s/%s/%s", S3HOST, S3BUCKET, p->Key().c_str());
        printf("%s, %d\n", urlbuf, p->Size());
    }

    delete r;
}