void* worker(void* d) {
shared_ptr<UploadBuffer> pdata;
srand(time(NULL));
// get data from fq
while (1) {
fq.deQ(pdata);
if (pdata->index == 0) {
// inited data??
cout << "inited data" << std::endl;
}
if (pdata->index == -1) { // finished
// cout<<"finished"<<std::endl;
break;
}
MD5Calc m;
m.Update(pdata->data.getdata(), pdata->data.len());
cout << pdata->index << "\t" << m.Get() << std::endl;
// sleep(rand()%5);
// reset pdata
pdata->data.reset();
pdata->index = 0;
// re queue to empty queue
eq.enQ(pdata);
}
// cout<<"quit...\n";
}
TEST(Utils, md5) {
MD5Calc m;
m.Update(MD5TESTBUF, strlen(MD5TESTBUF));
EXPECT_STREQ("e302f9ecd2d189fa80aac1c3392e9b9c", m.Get());
m.Update(MD5TESTBUF, strlen(MD5TESTBUF));
m.Update(MD5TESTBUF, strlen(MD5TESTBUF));
m.Update(MD5TESTBUF, strlen(MD5TESTBUF));
EXPECT_STREQ("3f8c2c6e2579e864071c33919fac61ee", m.Get());
}
TEST(databuffer, simple) {
MD5Calc m;
shared_ptr<DataBuffer> pdata = make_shared<DataBuffer>((uint64_t)128);
EXPECT_EQ(pdata->len(), 0);
EXPECT_TRUE(pdata->empty());
EXPECT_FALSE(pdata->full());
EXPECT_EQ(pdata->append(TEST_STRING, strlen(TEST_STRING)), 43);
EXPECT_FALSE(pdata->empty());
EXPECT_EQ(pdata->len(), 43);
m.Update(pdata->getdata(), 43);
EXPECT_STREQ("9e107d9d372bb6826bd81d3542a419d6", m.Get());
EXPECT_EQ(pdata->append(TEST_STRING, strlen(TEST_STRING)), 43);
EXPECT_EQ(pdata->append(TEST_STRING, strlen(TEST_STRING)), 42);
EXPECT_EQ(pdata->append(TEST_STRING, strlen(TEST_STRING)), 0);
EXPECT_TRUE(pdata->full());
m.Update(pdata->getdata(), 128);
EXPECT_STREQ("67e3cb156ceb62e5f114f075a8ef3063", m.Get());
pdata->reset();
EXPECT_EQ(pdata->len(), 0);
EXPECT_TRUE(pdata->empty());
EXPECT_FALSE(pdata->full());
EXPECT_EQ(pdata->append(TEST_STRING, strlen(TEST_STRING)), 43);
EXPECT_FALSE(pdata->empty());
EXPECT_EQ(pdata->len(), 43);
m.Update(pdata->getdata(), 43);
EXPECT_STREQ("9e107d9d372bb6826bd81d3542a419d6", m.Get());
EXPECT_EQ(pdata->append(TEST_STRING, strlen(TEST_STRING)), 43);
EXPECT_EQ(pdata->append(TEST_STRING, strlen(TEST_STRING)), 42);
EXPECT_EQ(pdata->append(TEST_STRING, strlen(TEST_STRING)), 0);
}
void ExtWrapperTest(const char *url, uint64_t buffer_size, const char *md5_str,
int segid, int segnum, uint64_t chunksize) {
InitConfig("test/s3.conf", "default");
s3ext_segid = segid;
s3ext_segnum = segnum;
MD5Calc m;
S3ExtBase *myData;
uint64_t nread = 0;
uint64_t buf_len = buffer_size;
char *buf = (char *)malloc(buffer_size);
ASSERT_NE((void *)NULL, buf);
if (strncmp(url, "http://localhost/", 17) == 0) {
myData = new S3Reader_fake(url);
} else {
myData = new S3Reader(url);
}
ASSERT_NE((void *)NULL, myData);
ASSERT_TRUE(myData->Init(segid, segnum, chunksize));
while (1) {
nread = buf_len;
ASSERT_TRUE(myData->TransferData(buf, nread));
if (nread == 0) break;
m.Update(buf, nread);
}
EXPECT_STREQ(md5_str, m.Get());
myData->Destroy();
delete myData;
free(buf);
}
void DownloadTest(const char *url, uint64_t file_size, const char *md5_str,
uint8_t thread_num, uint64_t chunk_size, uint64_t buffer_size,
bool use_credential) {
InitConfig("test/s3.conf", "default");
S3Credential g_cred = {s3ext_accessid, s3ext_secret};
uint64_t buf_len = buffer_size;
char *buf = (char *)malloc(buffer_size);
Downloader *d = new Downloader(thread_num);
MD5Calc m;
bool result = false;
ASSERT_NE((void *)NULL, buf);
if (use_credential) {
result = d->init(url, file_size, chunk_size, &g_cred);
} else {
result = d->init(url, file_size, chunk_size, NULL);
}
ASSERT_TRUE(result);
while (1) {
ASSERT_TRUE(d->get(buf, buf_len));
if (buf_len == 0) {
break;
}
m.Update(buf, buf_len);
buf_len = buffer_size;
}
ASSERT_STREQ(md5_str, m.Get());
delete d;
free(buf);
}