TEST_F(GPReaderTest, Close) {
string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";
MockS3RESTfulService mockRESTfulService(this->params);
MockGPReader gpreader(this->params, url, &mockRESTfulService);
XMLGenerator generator;
XMLGenerator* gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("threebytes/")
->setIsTruncated(false)
->pushBuckentContent(BucketContent("threebytes/", 0))
->pushBuckentContent(BucketContent("threebytes/threebytes", 3));
Response response(RESPONSE_OK, gen->toXML());
EXPECT_CALL(mockRESTfulService, get(_, _)).WillOnce(Return(response));
gpreader.open(this->params);
const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList();
EXPECT_EQ((uint64_t)1, keyList.contents.size());
gpreader.close();
EXPECT_TRUE(keyList.contents.empty());
}
TEST_F(GPReaderTest, ReadAndGetFailedKeyReaderResponse) {
string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";
MockS3RESTfulService mockRestfulService;
MockGPReader gpreader(url, &mockRestfulService);
XMLGenerator generator;
XMLGenerator* gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("threebytes/")
->setIsTruncated(false)
->pushBuckentContent(BucketContent("threebytes/", 0))
->pushBuckentContent(BucketContent("threebytes/threebytes", 3));
Response listBucketResponse(RESPONSE_OK, gen->toXML());
Response keyReaderResponse(RESPONSE_FAIL, vector<uint8_t>());
keyReaderResponse.setMessage(
"Mocked error in test 'GPReader.ReadAndGetFailedKeyReaderResponse'");
EXPECT_CALL(mockRestfulService, get(_, _, _))
.WillOnce(Return(listBucketResponse))
.WillOnce(Return(keyReaderResponse));
ReaderParams params;
gpreader.open(params);
const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList();
EXPECT_EQ(1, keyList.contents.size());
EXPECT_EQ("threebytes/threebytes", keyList.contents[0].getName());
EXPECT_EQ(3, keyList.contents[0].getSize());
char buffer[64];
EXPECT_THROW(gpreader.read(buffer, sizeof(buffer)), std::runtime_error);
}
TEST_F(GPReaderTest, ReadAndGetFailedKeyReaderResponse) {
string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";
MockS3RESTfulService mockRESTfulService(this->params);
MockGPReader gpreader(this->params, url, &mockRESTfulService);
XMLGenerator generator;
XMLGenerator* gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("threebytes/")
->setIsTruncated(false)
->pushBuckentContent(BucketContent("threebytes/", 0))
->pushBuckentContent(BucketContent("threebytes/threebytes", 3));
Response listBucketResponse(RESPONSE_OK, gen->toXML());
EXPECT_CALL(mockRESTfulService, get(_, _))
.WillOnce(Return(listBucketResponse))
.WillRepeatedly(Throw(S3FailedAfterRetry("", 3, "")));
gpreader.open(this->params);
const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList();
EXPECT_EQ((uint64_t)1, keyList.contents.size());
EXPECT_EQ("threebytes/threebytes", keyList.contents[0].getName());
EXPECT_EQ((uint64_t)3, keyList.contents[0].getSize());
char buffer[64];
EXPECT_THROW(gpreader.read(buffer, sizeof(buffer)), S3FailedAfterRetry);
}
/**
Generates the XML files with the metadata of the parsed files, which is obtained from the pubmed MySql database
@param XMLG: XMLGenerator object to use
@param dir: directory wheret the files will be saved
@param tArticles: variable where the time needed to generate the Articles.xml file will be saved
@param tExtArticles:variable where the time needed to generate the ExtendedArticles.xml file will be saved
@param tCitations:variable where the time needed to generate the Citations.xml file will be saved
@param nArt: maximum number of articles that the XML files, or -1 if the files will contain all the articles of the DB
@param dir: directory where the generated files will be saved
@output ArticleIndex.xml
@output ExtendedArticleIndex.xml
@output CitationIndex.xml
*/
void DatasetGenerator::generateXML(XMLGenerator &XMLG, const string &dir, int &tArticles, int &tExtArticles, int &tCitations, int nArt)
{
struct timeval start, stop;
long sec, msec;
cout <<"----------N="<<nArt<<"------------"<<endl;
//GENERATE Citations.xml
//Init
gettimeofday(&start, NULL);
string name="Citations.xml";
if(nArt!=-1)name="Citations_"+to_string(nArt)+".xml";
//Create XML file
XMLG.createCitationsXML(name, dir, nArt);
//Get execution time
gettimeofday(&stop, NULL);
sec=stop.tv_sec-start.tv_sec;
msec=sec*1000+(stop.tv_usec-start.tv_usec)/1000;
tCitations=msec;
cout << "Time lapsed for the generation of the Citation XML: " << msec << endl;
//GENERATE ExtendedArticles.xml
//Init
gettimeofday(&start, NULL);
name="ExtendedArticles.xml";
if(nArt!=-1)name="ExtendedArticles_"+to_string(nArt)+".xml";
//Create XML file
XMLG.createExtendedArticlesXML(name, dir, nArt);
//Get execution time
gettimeofday(&stop, NULL);
sec=stop.tv_sec-start.tv_sec;
msec=sec*1000+(stop.tv_usec-start.tv_usec)/1000;
tExtArticles=msec;
cout << "Time lapsed for the generation of the Extended Article XML: " << msec << endl;
//GENERATE Articles.xml
//Init
gettimeofday(&start, NULL);
name="Articles.xml";
if(nArt!=-1)name="Articles_"+to_string(nArt)+".xml";
//Create XML file
XMLG.createArticlesXML(name, dir, nArt);
//Get execution time
gettimeofday(&stop, NULL);
sec=stop.tv_sec-start.tv_sec;
msec=sec*1000+(stop.tv_usec-start.tv_usec)/1000;
tArticles=msec;
cout << "Time lapsed for the generation of the Article XML: " << msec << endl;
}
TEST_F(S3InterfaceServiceTest, ListBucketWithNormalBucket) {
XMLGenerator generator;
XMLGenerator *gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("threebytes/")
->setIsTruncated(false)
->pushBuckentContent(BucketContent("threebytes/", 0))
->pushBuckentContent(BucketContent("threebytes/threebytes", 3));
Response response(RESPONSE_OK, gen->toXML());
EXPECT_CALL(mockRESTfulService, get(_, _)).WillOnce(Return(response));
result = this->listBucket(this->params.getS3Url());
EXPECT_EQ((uint64_t)1, result.contents.size());
}
TEST_F(GPReaderTest, ReadHugeData) {
InitConfig("data/s3test.conf", "smallchunk");
string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";
// We don't know the chunksize before we load_config()
// so we create a big enough data (128M), to force
// multiple-threads to be constructed and loop-read to be triggered.
uint64_t totalData = 1024 * 1024 * 128;
MockS3RESTfulServiceForMultiThreads mockRestfulService(totalData);
MockGPReader gpreader(url, &mockRestfulService);
XMLGenerator generator;
XMLGenerator* gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("bigdata/")
->setIsTruncated(false)
->pushBuckentContent(BucketContent("bigdata/", 0))
->pushBuckentContent(BucketContent("bigdata/bigdata", totalData));
Response listBucketResponse(RESPONSE_OK, gen->toXML());
// call mockGet() instead of simply return a Response.
EXPECT_CALL(mockRestfulService, get(_, _, _))
.WillOnce(Return(listBucketResponse))
.WillRepeatedly(Invoke(&mockRestfulService, &MockS3RESTfulServiceForMultiThreads::mockGet));
ReaderParams params;
gpreader.open(params);
// compare the data size
const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList();
EXPECT_EQ(1, keyList.contents.size());
EXPECT_EQ("bigdata/bigdata", keyList.contents[0].getName());
EXPECT_EQ(totalData, keyList.contents[0].getSize());
// compare the data content
static char buffer[1024 * 1024];
uint64_t size = sizeof(buffer);
for (uint64_t i = 0; i < totalData; i += size) {
EXPECT_EQ(size, gpreader.read(buffer, size));
ASSERT_EQ(0, memcmp(buffer, mockRestfulService.getData() + i, size));
}
// Guarantee the last call
EXPECT_EQ(0, gpreader.read(buffer, sizeof(buffer)));
}
TEST_F(GPReaderTest, ReadSmallData) {
string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";
MockS3RESTfulService mockRestfulService;
MockGPReader gpreader(url, &mockRestfulService);
XMLGenerator generator;
XMLGenerator* gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("threebytes/")
->setIsTruncated(false)
->pushBuckentContent(BucketContent("threebytes/", 0))
->pushBuckentContent(BucketContent("threebytes/threebytes", 3));
Response listBucketResponse(RESPONSE_OK, gen->toXML());
vector<uint8_t> keyContent;
// generate 3 bytes in random way
srand(time(NULL));
keyContent.push_back(rand() & 0xFF);
keyContent.push_back(rand() & 0xFF);
keyContent.push_back(rand() & 0xFF);
Response keyReaderResponse(RESPONSE_OK, keyContent);
EXPECT_CALL(mockRestfulService, get(_, _, _))
.WillOnce(Return(listBucketResponse))
// first 4 bytes is retrieved once for format detection.
.WillOnce(Return(keyReaderResponse))
// whole file content
.WillOnce(Return(keyReaderResponse));
ReaderParams params;
gpreader.open(params);
const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList();
EXPECT_EQ(1, keyList.contents.size());
EXPECT_EQ("threebytes/threebytes", keyList.contents[0].getName());
EXPECT_EQ(3, keyList.contents[0].getSize());
char buffer[64];
EXPECT_EQ(3, gpreader.read(buffer, sizeof(buffer)));
EXPECT_EQ(0, memcmp(buffer, keyContent.data(), 3));
EXPECT_EQ(0, gpreader.read(buffer, sizeof(buffer)));
}
Response buildListBucketResponse(int numOfContent, bool isTruncated, int numOfZeroKeys = 0) {
XMLGenerator generator;
XMLGenerator *gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("s3files/")
->setIsTruncated(isTruncated)
->pushBuckentContent(BucketContent("s3files/", 0));
char buffer[32] = {0};
for (int i = 0; i < numOfContent; ++i) {
snprintf(buffer, 32, "files%d", i);
gen->pushBuckentContent(BucketContent(buffer, i + 1));
}
for (int i = 0; i < numOfZeroKeys; i++) {
snprintf(buffer, 32, "zerofiles%d", i);
gen->pushBuckentContent(BucketContent(buffer, 0));
}
return Response(RESPONSE_OK, gen->toXML());
}