// There is currently the possibility of a race condition if a chunk
// upload timed-out. It's possible that a second upload succeeds,
// has the chunk marked as "complete" and then the first request makes
// its way through the queue and marks the chunk as pending again.
// Since we are just about to close the file, we'll check to see if any
// chunks are marked as pending, and if so, we'll retry them.
void check_for_complete_chunks(vector<File> &files) {
for (int currCheckNum=0; currCheckNum < NUM_CHUNK_CHECKS; ++currCheckNum){
map<string, JSON> fileDescriptions;
while (!chunksFinished.empty()) {
Chunk *c = chunksFinished.consume();
// Cache file descriptions so we only have to do once per file,
// not once per chunk.
if (fileDescriptions.find(c->fileID) == fileDescriptions.end())
fileDescriptions[c->fileID] = fileDescribe(c->fileID);
if (!is_chunk_complete(c, fileDescriptions[c->fileID])) {
// After the chunk was uploaded, it was cleared, removing the data
// from the buffer. We need to reload if we're going to upload again.
chunksToRead.produce(c);
}
}
// All of the chunks were marked as complete, so let's exit and we
// should be safeish to close the file.
if(chunksToRead.size() == 0)
return;
// Set the totalChunks variable to the # of chunks we're going
// to retry now plus the number of chunks in the failed queue. The monitor
// thread will be busy until the size of chunksFinished + chunksFailed
// equals totalChunks.
DXLOG(logINFO) << "Retrying " << chunksToRead.size() << " chunks that did not complete.";
totalChunks = chunksToRead.size() + chunksFailed.size();
// Read, compress, and upload the chunks which weren't marked as complete.
createWorkerThreads(files);
boost::thread monitorThread(monitor);
monitorThread.join();
interruptWorkerThreads();
joinWorkerThreads();
}
// We have tried to upload incomplete chunks NUM_CHUNK_CHECKS times!
// Check to see if there are any chunks still not complete and if so,
// print warning.
map<string, JSON> fileDescriptions;
while (!chunksFinished.empty()) {
Chunk *c = chunksFinished.consume();
// Cache file descriptions so we only have to do once per file,
// not once per chunk.
if (fileDescriptions.find(c->fileID) == fileDescriptions.end())
fileDescriptions[c->fileID] = fileDescribe(c->fileID);
if (!is_chunk_complete(c, fileDescriptions[c->fileID])) {
cerr << "Chunk " << c->index << " of file " << c->fileID << " did not complete. This file will not be accessible. PLease try to upload this file again." << endl;
}
}
}
int main()
{
std::cout << "\n Demonstrating C++11 Blocking Queue";
std::cout << "\n ====================================";
BlockingQueue<std::string> q;
std::thread t(test, &q);
for(int i=0; i<15; ++i)
{
std::ostringstream temp;
temp << i;
std::string msg = std::string("msg#") + temp.str();
{
std::lock_guard<std::mutex> l(ioLock);
std::cout << "\n main enQing " << msg.c_str();
}
q.enQ(msg);
std::this_thread::sleep_for(std::chrono::milliseconds(3));
}
q.enQ("quit");
t.join();
std::cout << "\n";
std::cout << "\n Making copy of BlockingQueue";
std::cout << "\n ------------------------------";
std::string msg = "test";
q.enQ(msg);
std::cout << "\n q.size() = " << q.size();
BlockingQueue<std::string> q2 = q; // make default copy
std::cout << "\n q2.size() = " << q2.size();
std::cout << "\n q element = " << (msg = q.deQ());
q.enQ(msg);
std::cout << "\n q2 element = " << q2.deQ() << "\n";
std::cout << "\n Assigning state of BlockingQueue";
std::cout << "\n ----------------------------------";
BlockingQueue<std::string> q3;
q3 = q;
std::cout << "\n q.size() = " << q.size();
std::cout << "\n q3.size() = " << q3.size();
std::cout << "\n q element = " << q.deQ();
q.enQ(msg);
std::cout << "\n q3 element = " << q3.deQ() << "\n";
std::cout << "\n\n";
}
void ClientHandler::operator()(Socket& socket_)
{
Display *disp = new Display();
BlockingQueue<string> q_;
while (true)
{
Receiver *rs = new Receiver();
std::string command = socket_.recvString();
q_.enQ(command);
if (command.size() == 0)
break;
size_t size= q_.size();
string ackmsg;
for (size_t i = 0; i < size; i++)
{
command = q_.deQ();
ackmsg = rs->start(command, socket_);
}
Sender *s = new Sender();
if (ackmsg != "")
{
s->sendAck(socket_, ackmsg);
}
}
disp->show("ClientHandler socket connection closing");
socket_.shutDown();
socket_.close();
disp->show("ClientHandler thread terminating");
}
TEST(LinkedBlockingQueue, OrderingTest) {
BlockingQueue *queue = new LinkedBlockingQueue();
ASSERT_TRUE(queue->offer(new DummyObject(1)));
ASSERT_TRUE(queue->offer(new DummyObject(2)));
ASSERT_TRUE(queue->offer(new DummyObject(3)));
DummyObject *ret1 = dynamic_cast<DummyObject *>(queue->take());
ASSERT_TRUE(ret1 != NULL);
ASSERT_EQ(1, ret1->getData());
delete ret1;
DummyObject *ret2 = dynamic_cast<DummyObject *>(queue->take());
ASSERT_TRUE(ret2 != NULL);
ASSERT_EQ(2, ret2->getData());
delete ret2;
ASSERT_TRUE(queue->offer(new DummyObject(4)));
DummyObject *ret3 = dynamic_cast<DummyObject *>(queue->take());
ASSERT_TRUE(ret3 != NULL);
ASSERT_EQ(3, ret3->getData());
delete ret3;
DummyObject *ret4 = dynamic_cast<DummyObject *>(queue->take());
ASSERT_TRUE(ret4 != NULL);
ASSERT_EQ(4, ret4->getData());
delete ret4;
ASSERT_EQ(0, queue->size());
delete queue;
}
void monitor() {
while (true) {
boost::this_thread::sleep(boost::posix_time::milliseconds(1000));
{
DXLOG(logINFO) << "[monitor]"
<< " to read: " << chunksToRead.size()
<< " to compress: " << chunksToCompress.size()
<< " to upload: " << chunksToUpload.size()
<< " finished: " << chunksFinished.size()
<< " failed: " << chunksFailed.size();
if (finished()) {
return;
}
}
}
}
void monitor() {
while (true) {
boost::this_thread::sleep(boost::posix_time::milliseconds(1000));
{
LOG << "[monitor]"
<< " to read: " << chunksToRead.size()
<< " to compute md5: " << chunksToComputeMD5.size()
<< " skipped: " << chunksSkipped.size()
<< " finished: " << chunksFinished.size()
<< " failed: " << chunksFailed.size() << endl;
if (finished()) {
return;
}
}
}
}
TEST(LinkedBlockingQueue, MixedTest) {
BlockingQueue *queue = new LinkedBlockingQueue(5);
SuperObject *dummy1 = new DummyObject(1);
ASSERT_TRUE(queue->offer(dummy1));
ASSERT_TRUE(queue->offer(new DummyObject(2)));
ASSERT_TRUE(queue->offer(new DummyObject(3)));
ASSERT_TRUE(queue->offer(new DummyObject(4)));
ASSERT_TRUE(queue->add(new DummyObject(5)));
ASSERT_EQ(5, queue->size());
SuperObject *tmp1 = queue->take();
ASSERT_EQ(dummy1, tmp1);
ASSERT_EQ(4, queue->size());
delete tmp1;
SuperObject *tmp2 = queue->peek();
ASSERT_EQ(4, queue->size());
ASSERT_TRUE(tmp2 != NULL);
delete queue->take();
ASSERT_EQ(3, queue->size());
delete queue->take();
ASSERT_EQ(2, queue->size());
delete queue->take();
ASSERT_EQ(1, queue->size());
SuperObject *tmp3 = queue->poll(3);
ASSERT_TRUE(tmp3 != NULL);
if (tmp3 != NULL) {
delete tmp3;
}
ASSERT_EQ(0, queue->size());
ASSERT_EQ(NULL, queue->poll(3));
ASSERT_EQ(NULL, queue->peek());
delete queue;
}
void PopAndShow(BlockingQueue<double>& q)
{
long save = cout.flags(); // save format state
cout.flags(ios::showpoint); // show decimal point
cout.precision(3); // show only 3 digits on right of decimal
double item = q.deQ();
cout << "\n " << setw(4) << item << " : ";
cout << "Q size = " << q.size();
cout.flags(save); // restore format state
}
TEST(LinkedBlockingQueue, PeekAndRemoveTest) {
BlockingQueue *queue = new LinkedBlockingQueue();
SuperObject *dummy1 = new DummyObject(1);
ASSERT_TRUE(queue->offer(dummy1));
ASSERT_EQ(dummy1, queue->peek());
ASSERT_TRUE(queue->contains(dummy1));
ASSERT_TRUE(queue->remove(dummy1));
ASSERT_FALSE(queue->remove(dummy1));
ASSERT_FALSE(queue->contains(dummy1));
ASSERT_EQ(0, queue->size());
delete queue;
delete dummy1;
}
// Function for reterieving files which contains search text specified in the message
string Receiver::performTextSearch(string msg1, string &result)
{
Display * disp = new Display();
TextSearch *ts = new TextSearch();
Message *m = new Message();
BlockingQueue<std::string> resultfiles;
std::vector<string> file_patterns, all_files;
string sstr = m->getmsgBody1(msg1), file_path = m->getFileName(msg1), filepattern = m->getPatterns(msg1), token, resultfiles1;
disp->show("\n*******************************************\n");
string prtmsg ="Performing Text Search for string: "+ sstr + " on files with patterns " + filepattern+"\n";
disp->show(prtmsg);
std::istringstream ss(filepattern);
while (std::getline(ss, token, ','))
{
file_patterns.push_back(token);
}
all_files = showFiles(file_path, file_patterns);
for each (string file in all_files)
{
if (ts->search(sstr, file))
resultfiles.enQ(file);
}
size_t vecsize = resultfiles.size();
for (size_t i = 0; i < vecsize; i++)
{
resultfiles1 = resultfiles1 + resultfiles.deQ();
resultfiles1.append(";");
}
disp->show("Sending Result File list to client \n");
disp->show("*******************************************\n\n");
Message *resmsg = new Message();
Message orgmsg;
orgmsg.setMessage(msg1);
resmsg->setCommand("textsearch");
resmsg->setFileAttrs(orgmsg.getFileName(msg1));
resmsg->setsendAddr(orgmsg.getrecvAddr(msg1));
resmsg->setrecAddr(orgmsg.getsendAddr(msg1));
resmsg->setmsgBody(resultfiles1);
result = resmsg->getMessage();
return result;
}
bool finished() {
return (chunksFinished.size() + chunksFailed.size() == totalChunks);
}
bool RequestProcessor::OutQueueEmpty()
{
bool test = outQ.size() == 0;
return test;
}
