TEST_F(GPReaderTest, ReadAndGetFailedKeyReaderResponse) { string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal"; MockS3RESTfulService mockRESTfulService(this->params); MockGPReader gpreader(this->params, url, &mockRESTfulService); XMLGenerator generator; XMLGenerator* gen = &generator; gen->setName("s3test.pivotal.io") ->setPrefix("threebytes/") ->setIsTruncated(false) ->pushBuckentContent(BucketContent("threebytes/", 0)) ->pushBuckentContent(BucketContent("threebytes/threebytes", 3)); Response listBucketResponse(RESPONSE_OK, gen->toXML()); EXPECT_CALL(mockRESTfulService, get(_, _)) .WillOnce(Return(listBucketResponse)) .WillRepeatedly(Throw(S3FailedAfterRetry("", 3, ""))); gpreader.open(this->params); const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList(); EXPECT_EQ((uint64_t)1, keyList.contents.size()); EXPECT_EQ("threebytes/threebytes", keyList.contents[0].getName()); EXPECT_EQ((uint64_t)3, keyList.contents[0].getSize()); char buffer[64]; EXPECT_THROW(gpreader.read(buffer, sizeof(buffer)), S3FailedAfterRetry); }
TEST_F(GPReaderTest, Close) { string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal"; MockS3RESTfulService mockRESTfulService(this->params); MockGPReader gpreader(this->params, url, &mockRESTfulService); XMLGenerator generator; XMLGenerator* gen = &generator; gen->setName("s3test.pivotal.io") ->setPrefix("threebytes/") ->setIsTruncated(false) ->pushBuckentContent(BucketContent("threebytes/", 0)) ->pushBuckentContent(BucketContent("threebytes/threebytes", 3)); Response response(RESPONSE_OK, gen->toXML()); EXPECT_CALL(mockRESTfulService, get(_, _)).WillOnce(Return(response)); gpreader.open(this->params); const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList(); EXPECT_EQ((uint64_t)1, keyList.contents.size()); gpreader.close(); EXPECT_TRUE(keyList.contents.empty()); }
TEST_F(GPReaderTest, ReadAndGetFailedKeyReaderResponse) { string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal"; MockS3RESTfulService mockRestfulService; MockGPReader gpreader(url, &mockRestfulService); XMLGenerator generator; XMLGenerator* gen = &generator; gen->setName("s3test.pivotal.io") ->setPrefix("threebytes/") ->setIsTruncated(false) ->pushBuckentContent(BucketContent("threebytes/", 0)) ->pushBuckentContent(BucketContent("threebytes/threebytes", 3)); Response listBucketResponse(RESPONSE_OK, gen->toXML()); Response keyReaderResponse(RESPONSE_FAIL, vector<uint8_t>()); keyReaderResponse.setMessage( "Mocked error in test 'GPReader.ReadAndGetFailedKeyReaderResponse'"); EXPECT_CALL(mockRestfulService, get(_, _, _)) .WillOnce(Return(listBucketResponse)) .WillOnce(Return(keyReaderResponse)); ReaderParams params; gpreader.open(params); const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList(); EXPECT_EQ(1, keyList.contents.size()); EXPECT_EQ("threebytes/threebytes", keyList.contents[0].getName()); EXPECT_EQ(3, keyList.contents[0].getSize()); char buffer[64]; EXPECT_THROW(gpreader.read(buffer, sizeof(buffer)), std::runtime_error); }
TEST_F(S3InterfaceServiceTest, ListBucketWithNormalBucket) { XMLGenerator generator; XMLGenerator *gen = &generator; gen->setName("s3test.pivotal.io") ->setPrefix("threebytes/") ->setIsTruncated(false) ->pushBuckentContent(BucketContent("threebytes/", 0)) ->pushBuckentContent(BucketContent("threebytes/threebytes", 3)); Response response(RESPONSE_OK, gen->toXML()); EXPECT_CALL(mockRESTfulService, get(_, _)).WillOnce(Return(response)); result = this->listBucket(this->params.getS3Url()); EXPECT_EQ((uint64_t)1, result.contents.size()); }
TEST_F(GPReaderTest, ReadHugeData) { InitConfig("data/s3test.conf", "smallchunk"); string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal"; // We don't know the chunksize before we load_config() // so we create a big enough data (128M), to force // multiple-threads to be constructed and loop-read to be triggered. uint64_t totalData = 1024 * 1024 * 128; MockS3RESTfulServiceForMultiThreads mockRestfulService(totalData); MockGPReader gpreader(url, &mockRestfulService); XMLGenerator generator; XMLGenerator* gen = &generator; gen->setName("s3test.pivotal.io") ->setPrefix("bigdata/") ->setIsTruncated(false) ->pushBuckentContent(BucketContent("bigdata/", 0)) ->pushBuckentContent(BucketContent("bigdata/bigdata", totalData)); Response listBucketResponse(RESPONSE_OK, gen->toXML()); // call mockGet() instead of simply return a Response. EXPECT_CALL(mockRestfulService, get(_, _, _)) .WillOnce(Return(listBucketResponse)) .WillRepeatedly(Invoke(&mockRestfulService, &MockS3RESTfulServiceForMultiThreads::mockGet)); ReaderParams params; gpreader.open(params); // compare the data size const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList(); EXPECT_EQ(1, keyList.contents.size()); EXPECT_EQ("bigdata/bigdata", keyList.contents[0].getName()); EXPECT_EQ(totalData, keyList.contents[0].getSize()); // compare the data content static char buffer[1024 * 1024]; uint64_t size = sizeof(buffer); for (uint64_t i = 0; i < totalData; i += size) { EXPECT_EQ(size, gpreader.read(buffer, size)); ASSERT_EQ(0, memcmp(buffer, mockRestfulService.getData() + i, size)); } // Guarantee the last call EXPECT_EQ(0, gpreader.read(buffer, sizeof(buffer))); }
TEST_F(GPReaderTest, ReadSmallData) { string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal"; MockS3RESTfulService mockRestfulService; MockGPReader gpreader(url, &mockRestfulService); XMLGenerator generator; XMLGenerator* gen = &generator; gen->setName("s3test.pivotal.io") ->setPrefix("threebytes/") ->setIsTruncated(false) ->pushBuckentContent(BucketContent("threebytes/", 0)) ->pushBuckentContent(BucketContent("threebytes/threebytes", 3)); Response listBucketResponse(RESPONSE_OK, gen->toXML()); vector<uint8_t> keyContent; // generate 3 bytes in random way srand(time(NULL)); keyContent.push_back(rand() & 0xFF); keyContent.push_back(rand() & 0xFF); keyContent.push_back(rand() & 0xFF); Response keyReaderResponse(RESPONSE_OK, keyContent); EXPECT_CALL(mockRestfulService, get(_, _, _)) .WillOnce(Return(listBucketResponse)) // first 4 bytes is retrieved once for format detection. .WillOnce(Return(keyReaderResponse)) // whole file content .WillOnce(Return(keyReaderResponse)); ReaderParams params; gpreader.open(params); const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList(); EXPECT_EQ(1, keyList.contents.size()); EXPECT_EQ("threebytes/threebytes", keyList.contents[0].getName()); EXPECT_EQ(3, keyList.contents[0].getSize()); char buffer[64]; EXPECT_EQ(3, gpreader.read(buffer, sizeof(buffer))); EXPECT_EQ(0, memcmp(buffer, keyContent.data(), 3)); EXPECT_EQ(0, gpreader.read(buffer, sizeof(buffer))); }
Response buildListBucketResponse(int numOfContent, bool isTruncated, int numOfZeroKeys = 0) { XMLGenerator generator; XMLGenerator *gen = &generator; gen->setName("s3test.pivotal.io") ->setPrefix("s3files/") ->setIsTruncated(isTruncated) ->pushBuckentContent(BucketContent("s3files/", 0)); char buffer[32] = {0}; for (int i = 0; i < numOfContent; ++i) { snprintf(buffer, 32, "files%d", i); gen->pushBuckentContent(BucketContent(buffer, i + 1)); } for (int i = 0; i < numOfZeroKeys; i++) { snprintf(buffer, 32, "zerofiles%d", i); gen->pushBuckentContent(BucketContent(buffer, 0)); } return Response(RESPONSE_OK, gen->toXML()); }