void S3Reader::getNextDownloader() { if (this->filedownloader) { // reset old downloader filedownloader->destroy(); delete this->filedownloader; this->filedownloader = NULL; } if (this->contentindex >= this->keylist->contents.size()) { S3DEBUG("No more files to download"); return; } this->filedownloader = new Downloader(this->concurrent_num); if (!this->filedownloader) { S3ERROR("Failed to create filedownloader"); return; } BucketContent *c = this->keylist->contents[this->contentindex]; string keyurl = this->getKeyURL(c->Key()); S3DEBUG("key: %s, size: %llu", keyurl.c_str(), c->Size()); // XXX don't use strdup() if (!filedownloader->init(keyurl.c_str(), c->Size(), this->chunksize, &this->cred)) { delete this->filedownloader; this->filedownloader = NULL; } else { // move to next file // for now, every segment downloads its assigned files(mod) // better to build a workqueue in case not all segments are available this->contentindex += this->segnum; } return; }
S3Params S3BucketReader::constructReaderParams(BucketContent& key) { // encode the key name but leave the "/" // "/encoded_path/encoded_name" string keyEncoded = UriEncode(key.getName()); FindAndReplace(keyEncoded, "%2F", "/"); S3Params readerParams = this->params.setPrefix(keyEncoded); readerParams.setKeySize(key.getSize()); S3DEBUG("key: %s, size: %" PRIu64, readerParams.getS3Url().getFullUrlForCurl().c_str(), readerParams.getKeySize()); return readerParams; }
S3Params S3BucketReader::constructReaderParams(BucketContent& key) { S3Params readerParams = this->params; // encode the key name but leave the "/" // "/encoded_path/encoded_name" string keyEncoded = uri_encode(key.getName()); find_replace(keyEncoded, "%2F", "/"); readerParams.setKeyUrl(this->getKeyURL(keyEncoded)); readerParams.setRegion(this->region); readerParams.setKeySize(key.getSize()); S3DEBUG("key: %s, size: %" PRIu64, readerParams.getKeyUrl().c_str(), readerParams.getKeySize()); return readerParams; }
TEST(ListBucket, fake) { ListBucketResult *r = ListBucket_FakeHTTP(HOSTSTR, BUCKETSTR); ASSERT_NE(r, (void *)NULL); EXPECT_EQ(r->contents.size(), 16); char urlbuf[256]; vector<BucketContent *>::iterator i; for (i = r->contents.begin(); i != r->contents.end(); i++) { BucketContent *p = *i; sprintf(urlbuf, "http://%s/%s/%s", HOSTSTR, BUCKETSTR, p->Key().c_str()); printf("%s, %llu\n", urlbuf, p->Size()); } delete r; }
TEST(ListBucket, s3) { InitConfig("test/s3.conf", "default"); S3Credential g_cred = {s3ext_accessid, s3ext_secret}; ListBucketResult *r = ListBucket("https", S3HOST, S3BUCKET, S3PREFIX, g_cred); ASSERT_NE(r, (void *)NULL); EXPECT_EQ(r->contents.size(), 16); char urlbuf[256]; vector<BucketContent *>::iterator i; for (i = r->contents.begin(); i != r->contents.end(); i++) { BucketContent *p = *i; sprintf(urlbuf, "http://%s/%s/%s", S3HOST, S3BUCKET, p->Key().c_str()); printf("%s, %d\n", urlbuf, p->Size()); } delete r; }