TEST_F(GPReaderTest, ReadAndGetFailedKeyReaderResponse) {
string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";
MockS3RESTfulService mockRESTfulService(this->params);
MockGPReader gpreader(this->params, url, &mockRESTfulService);
XMLGenerator generator;
XMLGenerator* gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("threebytes/")
->setIsTruncated(false)
->pushBuckentContent(BucketContent("threebytes/", 0))
->pushBuckentContent(BucketContent("threebytes/threebytes", 3));
Response listBucketResponse(RESPONSE_OK, gen->toXML());
EXPECT_CALL(mockRESTfulService, get(_, _))
.WillOnce(Return(listBucketResponse))
.WillRepeatedly(Throw(S3FailedAfterRetry("", 3, "")));
gpreader.open(this->params);
const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList();
EXPECT_EQ((uint64_t)1, keyList.contents.size());
EXPECT_EQ("threebytes/threebytes", keyList.contents[0].getName());
EXPECT_EQ((uint64_t)3, keyList.contents[0].getSize());
char buffer[64];
EXPECT_THROW(gpreader.read(buffer, sizeof(buffer)), S3FailedAfterRetry);
}
TEST_F(GPReaderTest, Close) {
string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";
MockS3RESTfulService mockRESTfulService(this->params);
MockGPReader gpreader(this->params, url, &mockRESTfulService);
XMLGenerator generator;
XMLGenerator* gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("threebytes/")
->setIsTruncated(false)
->pushBuckentContent(BucketContent("threebytes/", 0))
->pushBuckentContent(BucketContent("threebytes/threebytes", 3));
Response response(RESPONSE_OK, gen->toXML());
EXPECT_CALL(mockRESTfulService, get(_, _)).WillOnce(Return(response));
gpreader.open(this->params);
const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList();
EXPECT_EQ((uint64_t)1, keyList.contents.size());
gpreader.close();
EXPECT_TRUE(keyList.contents.empty());
}
TEST_F(GPReaderTest, ReadAndGetFailedKeyReaderResponse) {
string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";
MockS3RESTfulService mockRestfulService;
MockGPReader gpreader(url, &mockRestfulService);
XMLGenerator generator;
XMLGenerator* gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("threebytes/")
->setIsTruncated(false)
->pushBuckentContent(BucketContent("threebytes/", 0))
->pushBuckentContent(BucketContent("threebytes/threebytes", 3));
Response listBucketResponse(RESPONSE_OK, gen->toXML());
Response keyReaderResponse(RESPONSE_FAIL, vector<uint8_t>());
keyReaderResponse.setMessage(
"Mocked error in test 'GPReader.ReadAndGetFailedKeyReaderResponse'");
EXPECT_CALL(mockRestfulService, get(_, _, _))
.WillOnce(Return(listBucketResponse))
.WillOnce(Return(keyReaderResponse));
ReaderParams params;
gpreader.open(params);
const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList();
EXPECT_EQ(1, keyList.contents.size());
EXPECT_EQ("threebytes/threebytes", keyList.contents[0].getName());
EXPECT_EQ(3, keyList.contents[0].getSize());
char buffer[64];
EXPECT_THROW(gpreader.read(buffer, sizeof(buffer)), std::runtime_error);
}
TEST_F(S3InterfaceServiceTest, ListBucketWithNormalBucket) {
XMLGenerator generator;
XMLGenerator *gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("threebytes/")
->setIsTruncated(false)
->pushBuckentContent(BucketContent("threebytes/", 0))
->pushBuckentContent(BucketContent("threebytes/threebytes", 3));
Response response(RESPONSE_OK, gen->toXML());
EXPECT_CALL(mockRESTfulService, get(_, _)).WillOnce(Return(response));
result = this->listBucket(this->params.getS3Url());
EXPECT_EQ((uint64_t)1, result.contents.size());
}
TEST_F(GPReaderTest, ReadHugeData) {
InitConfig("data/s3test.conf", "smallchunk");
string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";
// We don't know the chunksize before we load_config()
// so we create a big enough data (128M), to force
// multiple-threads to be constructed and loop-read to be triggered.
uint64_t totalData = 1024 * 1024 * 128;
MockS3RESTfulServiceForMultiThreads mockRestfulService(totalData);
MockGPReader gpreader(url, &mockRestfulService);
XMLGenerator generator;
XMLGenerator* gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("bigdata/")
->setIsTruncated(false)
->pushBuckentContent(BucketContent("bigdata/", 0))
->pushBuckentContent(BucketContent("bigdata/bigdata", totalData));
Response listBucketResponse(RESPONSE_OK, gen->toXML());
// call mockGet() instead of simply return a Response.
EXPECT_CALL(mockRestfulService, get(_, _, _))
.WillOnce(Return(listBucketResponse))
.WillRepeatedly(Invoke(&mockRestfulService, &MockS3RESTfulServiceForMultiThreads::mockGet));
ReaderParams params;
gpreader.open(params);
// compare the data size
const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList();
EXPECT_EQ(1, keyList.contents.size());
EXPECT_EQ("bigdata/bigdata", keyList.contents[0].getName());
EXPECT_EQ(totalData, keyList.contents[0].getSize());
// compare the data content
static char buffer[1024 * 1024];
uint64_t size = sizeof(buffer);
for (uint64_t i = 0; i < totalData; i += size) {
EXPECT_EQ(size, gpreader.read(buffer, size));
ASSERT_EQ(0, memcmp(buffer, mockRestfulService.getData() + i, size));
}
// Guarantee the last call
EXPECT_EQ(0, gpreader.read(buffer, sizeof(buffer)));
}
TEST_F(GPReaderTest, ReadSmallData) {
string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";
MockS3RESTfulService mockRestfulService;
MockGPReader gpreader(url, &mockRestfulService);
XMLGenerator generator;
XMLGenerator* gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("threebytes/")
->setIsTruncated(false)
->pushBuckentContent(BucketContent("threebytes/", 0))
->pushBuckentContent(BucketContent("threebytes/threebytes", 3));
Response listBucketResponse(RESPONSE_OK, gen->toXML());
vector<uint8_t> keyContent;
// generate 3 bytes in random way
srand(time(NULL));
keyContent.push_back(rand() & 0xFF);
keyContent.push_back(rand() & 0xFF);
keyContent.push_back(rand() & 0xFF);
Response keyReaderResponse(RESPONSE_OK, keyContent);
EXPECT_CALL(mockRestfulService, get(_, _, _))
.WillOnce(Return(listBucketResponse))
// first 4 bytes is retrieved once for format detection.
.WillOnce(Return(keyReaderResponse))
// whole file content
.WillOnce(Return(keyReaderResponse));
ReaderParams params;
gpreader.open(params);
const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList();
EXPECT_EQ(1, keyList.contents.size());
EXPECT_EQ("threebytes/threebytes", keyList.contents[0].getName());
EXPECT_EQ(3, keyList.contents[0].getSize());
char buffer[64];
EXPECT_EQ(3, gpreader.read(buffer, sizeof(buffer)));
EXPECT_EQ(0, memcmp(buffer, keyContent.data(), 3));
EXPECT_EQ(0, gpreader.read(buffer, sizeof(buffer)));
}
Response buildListBucketResponse(int numOfContent, bool isTruncated, int numOfZeroKeys = 0) {
XMLGenerator generator;
XMLGenerator *gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("s3files/")
->setIsTruncated(isTruncated)
->pushBuckentContent(BucketContent("s3files/", 0));
char buffer[32] = {0};
for (int i = 0; i < numOfContent; ++i) {
snprintf(buffer, 32, "files%d", i);
gen->pushBuckentContent(BucketContent(buffer, i + 1));
}
for (int i = 0; i < numOfZeroKeys; i++) {
snprintf(buffer, 32, "zerofiles%d", i);
gen->pushBuckentContent(BucketContent(buffer, 0));
}
return Response(RESPONSE_OK, gen->toXML());
}
