void S3Reader::getNextDownloader() {
if (this->filedownloader) { // reset old downloader
filedownloader->destroy();
delete this->filedownloader;
this->filedownloader = NULL;
}
if (this->contentindex >= this->keylist->contents.size()) {
S3DEBUG("No more files to download");
return;
}
this->filedownloader = new Downloader(this->concurrent_num);
if (!this->filedownloader) {
S3ERROR("Failed to create filedownloader");
return;
}
BucketContent *c = this->keylist->contents[this->contentindex];
string keyurl = this->getKeyURL(c->Key());
S3DEBUG("key: %s, size: %llu", keyurl.c_str(), c->Size());
// XXX don't use strdup()
if (!filedownloader->init(keyurl.c_str(), c->Size(),
this->chunksize, &this->cred)) {
delete this->filedownloader;
this->filedownloader = NULL;
} else { // move to next file
// for now, every segment downloads its assigned files(mod)
// better to build a workqueue in case not all segments are available
this->contentindex += this->segnum;
}
return;
}
S3Params S3BucketReader::constructReaderParams(BucketContent& key) {
// encode the key name but leave the "/"
// "/encoded_path/encoded_name"
string keyEncoded = UriEncode(key.getName());
FindAndReplace(keyEncoded, "%2F", "/");
S3Params readerParams = this->params.setPrefix(keyEncoded);
readerParams.setKeySize(key.getSize());
S3DEBUG("key: %s, size: %" PRIu64, readerParams.getS3Url().getFullUrlForCurl().c_str(),
readerParams.getKeySize());
return readerParams;
}
S3Params S3BucketReader::constructReaderParams(BucketContent& key) {
S3Params readerParams = this->params;
// encode the key name but leave the "/"
// "/encoded_path/encoded_name"
string keyEncoded = uri_encode(key.getName());
find_replace(keyEncoded, "%2F", "/");
readerParams.setKeyUrl(this->getKeyURL(keyEncoded));
readerParams.setRegion(this->region);
readerParams.setKeySize(key.getSize());
S3DEBUG("key: %s, size: %" PRIu64, readerParams.getKeyUrl().c_str(), readerParams.getKeySize());
return readerParams;
}
TEST(ListBucket, fake) {
ListBucketResult *r = ListBucket_FakeHTTP(HOSTSTR, BUCKETSTR);
ASSERT_NE(r, (void *)NULL);
EXPECT_EQ(r->contents.size(), 16);
char urlbuf[256];
vector<BucketContent *>::iterator i;
for (i = r->contents.begin(); i != r->contents.end(); i++) {
BucketContent *p = *i;
sprintf(urlbuf, "http://%s/%s/%s", HOSTSTR, BUCKETSTR,
p->Key().c_str());
printf("%s, %llu\n", urlbuf, p->Size());
}
delete r;
}
TEST(ListBucket, s3) {
InitConfig("test/s3.conf", "default");
S3Credential g_cred = {s3ext_accessid, s3ext_secret};
ListBucketResult *r =
ListBucket("https", S3HOST, S3BUCKET, S3PREFIX, g_cred);
ASSERT_NE(r, (void *)NULL);
EXPECT_EQ(r->contents.size(), 16);
char urlbuf[256];
vector<BucketContent *>::iterator i;
for (i = r->contents.begin(); i != r->contents.end(); i++) {
BucketContent *p = *i;
sprintf(urlbuf, "http://%s/%s/%s", S3HOST, S3BUCKET, p->Key().c_str());
printf("%s, %d\n", urlbuf, p->Size());
}
delete r;
}