int multireceive(int PORT, std::string group, std::string ip, BlockingQueue<std::string> &q, BlockingQueue<std::string> &foreign){
int pack = makesocket(PORT, group, ip); // call function above to make socket
int recvlen;
char buf[BUFSIZE] = "";
std::string foreignIP; // string to store IP of sender
recvlen = 0;
struct sockaddr_in peer_address; // makes structure for peer_address
socklen_t peer_address_len;
peer_address_len = sizeof(struct sockaddr_storage);
while(true){ // while loop to keep receiving
//std::cout << "waiting on port " << PORT << std::endl;
recvlen = recvfrom(pack, buf, BUFSIZE, 0, (struct sockaddr *)&peer_address, &peer_address_len); //actual receiving mechanism, returns amount of bytes received
//std::cout << "received bytes: " << recvlen << std::endl;
if (recvlen > 0) { // if bytes available push them to the BlockingQueue
foreignIP = inet_ntoa(peer_address.sin_addr); // stores IP of sender
foreign.push(std::string(foreignIP, foreignIP.size()));
q.push(std::string(buf, recvlen));
}
}
exit(0);
}
void run() {
BlockingQueue<int> q;
Timer t;
int x;
ASSERT(!q.blockingPop(x, 5));
ASSERT(t.seconds() > 3 && t.seconds() < 9);
}
int main()
{
pthread_t ptids[5];
for (int i = 0; i < 5; ++i)
{
pthread_create(&ptids[i], NULL, thread_func, NULL);
}
Functor task1 = ::std::bind(add_func, 1, 2);
taskQueue.post(task1);
Functor task2 = ::std::bind(add_func, 3, 4);
taskQueue.post(task2);
Functor task3 = ::std::bind(add_func, 5, 6);
taskQueue.post(task3);
for (int i = 0; i < 5; ++i)
{
pthread_join(ptids[i], NULL);
}
return 0;
}
void verifyChunkMD5(vector<File> &files) {
try {
while (true) {
Chunk * c = chunksToComputeMD5.consume();
if (files[c->parentFileIndex].matchStatus == File::Status::FAILED_TO_MATCH_REMOTE_FILE) {
// We have already marked file as a non-match, don't waste time reading more chunks from it
c->log("File status == FAILED_TO_MATCH_REMOTE_FILE, Skipping the MD5 compute...");
c->clear();
chunksSkipped.produce(c);
} else {
c->log("Computing MD5...");
string computedMD5 = c->computeMD5();
c->clear();
if (c->expectedMD5 != computedMD5) {
c->log("MISMATCH between expected MD5 '" + c->expectedMD5 + "', and computed MD5 '" + computedMD5 + "' ... marking the file as Mismatch");
files[c->parentFileIndex].matchStatus = File::Status::FAILED_TO_MATCH_REMOTE_FILE;
chunksFailed.produce(c);
} else {
c->log("Expected and computed MD5 match!");
chunksFinished.produce(c);
}
}
}
} catch (boost::thread_interrupted &ti) {
return;
}
}
// ------------------------------------ Starts Receiver for the this component ---------------------------
void Server::startReceiverThread()
{
BlockingQueue<HttpMessage> msgQ;
try
{
SocketSystem ss;
SocketListener sl(getServerPort(), Socket::IP6);
Receiver cp(msgQ,"Server");
sl.start(cp);
/*
* Since this is a server the loop below never terminates.
* We could easily change that by sending a distinguished
* message for shutdown.
*/
while (true)
{
HttpMessage msg = msgQ.deQ();
std::string cPort = msg.findValue("From-Port");
if (msg.attributes()[0].first == "GET") { // Checks if its an file extraction request
std::string filename = msg.findValue("file");
Show::write("\n\n Client @: "+cPort+ " => requested file :: " + filename);
sendServerFiles(msg);
}
else
Show::write("\n\n Message from client @:"+cPort +" => " + msg.bodyString());
}
}
catch (std::exception& exc)
{
Show::write("\n Exeception caught: ");
std::string exMsg = "\n " + std::string(exc.what()) + "\n\n";
Show::write(exMsg);
}
}
void ClientHandler::operator()(Socket& socket_)
{
Display *disp = new Display();
BlockingQueue<string> q_;
while (true)
{
Receiver *rs = new Receiver();
std::string command = socket_.recvString();
q_.enQ(command);
if (command.size() == 0)
break;
size_t size= q_.size();
string ackmsg;
for (size_t i = 0; i < size; i++)
{
command = q_.deQ();
ackmsg = rs->start(command, socket_);
}
Sender *s = new Sender();
if (ackmsg != "")
{
s->sendAck(socket_, ackmsg);
}
}
disp->show("ClientHandler socket connection closing");
socket_.shutDown();
socket_.close();
disp->show("ClientHandler thread terminating");
}
int main()
{
std::cout << "\n Demonstrating BlockingQueue<std::string> Operation";
std::cout << "\n ====================================================\n";
BlockingQueue<std::string> q;
TestQThread td(q);
td.start();
gCSLock<1> IOlock; // global lock shares a static CS
std::string msg;
for(size_t i=0; i<25; ++i)
{
std::ostringstream convert; // convert size_t value to string
convert << i;
msg = std::string("Msg #") + convert.str();
q.enQ(msg);
IOlock.lock();
std::cout << "\n main thread enQ'd " << msg.c_str();
IOlock.unlock();
::Sleep(5);
}
q.enQ("quit");
td.join();
std::cout << "\n parent exiting\n\n";
}
int main()
{
title("Testing Sender", '=');
try
{
Verbose v(true);
SocketSystem ss;
SocketListener sl(1000, Socket::IP6);
BlockingQueue<Message> rQueue;
BlockingQueue<Message> sQueue;
Sender sd(&sQueue);
string test = "Test Message";
string fileName = "test1.txt";
Message msg("STRING_HANDLING_REQUEST");
msg.buildMessage(msg, 1000, 2000, test, test.size());
displayMessage(msg, "Constructing");
Message msg2("FILE_HANDLING_REQUEST");
msg.buildMessage(msg2, 1000, 2000, fileName);
sQueue.enQ(msg);
sQueue.enQ(msg2);
std::thread sThread(&Sender::getQueueMessage, Sender(&sQueue));
sThread.join();
}
catch (std::exception& ex)
{
Verbose::show(" Exception caught:", always);
Verbose::show(std::string("\n ") + ex.what() + "\n\n");
}
}
void compressChunks() {
try {
while (true) {
Chunk * c = chunksToCompress.consume();
if (c->toCompress) {
c->log("Compressing...");
c->compress();
c->log("Finished compressing");
} else {
c->log("Not compressing");
}
chunksToUpload.produce(c);
// Sleep for tiny amount of time, to make sure we yield to other threads.
// Note: boost::this_thread::yield() is not a valid interruption point,
// so we have to use sleep()
boost::this_thread::sleep(boost::posix_time::microseconds(100));
}
} catch(std::bad_alloc &e) {
boost::call_once(bad_alloc_once, boost::bind(&handle_bad_alloc, e));
} catch (boost::thread_interrupted &ti) {
return;
}
}
int main() {
::SetConsoleTitle("ClientHandler");
Show::attach(&std::cout);
Show::start();
Show::title("\n ClientHandler Server started");
BlockingQueue<HttpMessage> msgQ;
SocketSystem ss;
SocketListener sl(8080, Socket::IP6);
ClientHandler cp(msgQ);
sl.start(cp);
try
{
while (true)
{
HttpMessage msg = msgQ.deQ();
std::cout << "\n\n clienthandler " + msg.findValue("toAddr") + " recvd message contents:\n "
+ msg.bodyString() + "\n from client " + msg.findValue("fromAddr") + "\n";
if (msg.bodyString() == "closeServer") {
::Sleep(100);
}
}
}
catch (std::exception& exc)
{
Show::write("\n Exeception caught: ");
std::string exMsg = "\n " + std::string(exc.what()) + "\n\n";
Show::write(exMsg);
}
std::cout << "\n\n";
return 0;
}
void PopAndShow(BlockingQueue<double>& q)
{
long save = cout.flags(); // save format state
cout.flags(ios::showpoint); // show decimal point
cout.precision(3); // show only 3 digits on right of decimal
double item = q.deQ();
cout << "\n " << setw(4) << item << " : ";
cout << "Q size = " << q.size();
cout.flags(save); // restore format state
}
TEST(LinkedBlockingQueue, OrderingTest) {
BlockingQueue *queue = new LinkedBlockingQueue();
ASSERT_TRUE(queue->offer(new DummyObject(1)));
ASSERT_TRUE(queue->offer(new DummyObject(2)));
ASSERT_TRUE(queue->offer(new DummyObject(3)));
DummyObject *ret1 = dynamic_cast<DummyObject *>(queue->take());
ASSERT_TRUE(ret1 != NULL);
ASSERT_EQ(1, ret1->getData());
delete ret1;
DummyObject *ret2 = dynamic_cast<DummyObject *>(queue->take());
ASSERT_TRUE(ret2 != NULL);
ASSERT_EQ(2, ret2->getData());
delete ret2;
ASSERT_TRUE(queue->offer(new DummyObject(4)));
DummyObject *ret3 = dynamic_cast<DummyObject *>(queue->take());
ASSERT_TRUE(ret3 != NULL);
ASSERT_EQ(3, ret3->getData());
delete ret3;
DummyObject *ret4 = dynamic_cast<DummyObject *>(queue->take());
ASSERT_TRUE(ret4 != NULL);
ASSERT_EQ(4, ret4->getData());
delete ret4;
ASSERT_EQ(0, queue->size());
delete queue;
}
int main(int argc, char *argv[])
{
BlockingQueue<std::string> queue;
queue.put("Hello");
queue.put("World");
std::cout << queue.take() << std::endl;
std::cout << queue.take() << std::endl;
return 0;
}
// There is currently the possibility of a race condition if a chunk
// upload timed-out. It's possible that a second upload succeeds,
// has the chunk marked as "complete" and then the first request makes
// its way through the queue and marks the chunk as pending again.
// Since we are just about to close the file, we'll check to see if any
// chunks are marked as pending, and if so, we'll retry them.
void check_for_complete_chunks(vector<File> &files) {
for (int currCheckNum=0; currCheckNum < NUM_CHUNK_CHECKS; ++currCheckNum){
map<string, JSON> fileDescriptions;
while (!chunksFinished.empty()) {
Chunk *c = chunksFinished.consume();
// Cache file descriptions so we only have to do once per file,
// not once per chunk.
if (fileDescriptions.find(c->fileID) == fileDescriptions.end())
fileDescriptions[c->fileID] = fileDescribe(c->fileID);
if (!is_chunk_complete(c, fileDescriptions[c->fileID])) {
// After the chunk was uploaded, it was cleared, removing the data
// from the buffer. We need to reload if we're going to upload again.
chunksToRead.produce(c);
}
}
// All of the chunks were marked as complete, so let's exit and we
// should be safeish to close the file.
if(chunksToRead.size() == 0)
return;
// Set the totalChunks variable to the # of chunks we're going
// to retry now plus the number of chunks in the failed queue. The monitor
// thread will be busy until the size of chunksFinished + chunksFailed
// equals totalChunks.
DXLOG(logINFO) << "Retrying " << chunksToRead.size() << " chunks that did not complete.";
totalChunks = chunksToRead.size() + chunksFailed.size();
// Read, compress, and upload the chunks which weren't marked as complete.
createWorkerThreads(files);
boost::thread monitorThread(monitor);
monitorThread.join();
interruptWorkerThreads();
joinWorkerThreads();
}
// We have tried to upload incomplete chunks NUM_CHUNK_CHECKS times!
// Check to see if there are any chunks still not complete and if so,
// print warning.
map<string, JSON> fileDescriptions;
while (!chunksFinished.empty()) {
Chunk *c = chunksFinished.consume();
// Cache file descriptions so we only have to do once per file,
// not once per chunk.
if (fileDescriptions.find(c->fileID) == fileDescriptions.end())
fileDescriptions[c->fileID] = fileDescribe(c->fileID);
if (!is_chunk_complete(c, fileDescriptions[c->fileID])) {
cerr << "Chunk " << c->index << " of file " << c->fileID << " did not complete. This file will not be accessible. PLease try to upload this file again." << endl;
}
}
}
/* Test to see if the queue is sorted within reasonable limits */
bool is_sorted(BlockingQueue<timeout_t>& bq) {
LogContext& log = LogManager::instance().getLogContext("Test", "Scheduler");
timeout_t last = 0;
while(!bq.empty()) {
timeout_t cur = bq.front();
bq.pop();
if(cur < last && (last - cur) > 5 MILLIS ) {
/* we give a 5 ms grace period */
log.printfln(ERROR, "%ld is less than %ld", cur, last);
return false;
}
last = cur;
}
return true;
}
int main()
{
std::cout << "\n Demonstrating C++11 Blocking Queue";
std::cout << "\n ====================================";
BlockingQueue<std::string> q;
std::thread t(test, &q);
for(int i=0; i<15; ++i)
{
std::ostringstream temp;
temp << i;
std::string msg = std::string("msg#") + temp.str();
{
std::lock_guard<std::mutex> l(ioLock);
std::cout << "\n main enQing " << msg.c_str();
}
q.enQ(msg);
std::this_thread::sleep_for(std::chrono::milliseconds(3));
}
q.enQ("quit");
t.join();
std::cout << "\n";
std::cout << "\n Making copy of BlockingQueue";
std::cout << "\n ------------------------------";
std::string msg = "test";
q.enQ(msg);
std::cout << "\n q.size() = " << q.size();
BlockingQueue<std::string> q2 = q; // make default copy
std::cout << "\n q2.size() = " << q2.size();
std::cout << "\n q element = " << (msg = q.deQ());
q.enQ(msg);
std::cout << "\n q2 element = " << q2.deQ() << "\n";
std::cout << "\n Assigning state of BlockingQueue";
std::cout << "\n ----------------------------------";
BlockingQueue<std::string> q3;
q3 = q;
std::cout << "\n q.size() = " << q.size();
std::cout << "\n q3.size() = " << q3.size();
std::cout << "\n q element = " << q.deQ();
q.enQ(msg);
std::cout << "\n q3 element = " << q3.deQ() << "\n";
std::cout << "\n\n";
}
void consumer_take(int id, BlockingQueue & queue) {
for (int i = 0; i < 50; ++i) {
int data = queue.take();
cout << "Consumer " << id << " take " << data << endl;
this_thread::sleep_for(chrono::milliseconds(250));
}
}
void Cities::CreateCities(std::set<City> &_cities,
Countries &_countries,
BlockingQueue<std::string> &_q,
unsigned long &_read_cities,
unsigned long &_collisions,
const unsigned int &_read_count){
std::string *data;
_read_cities = 0;
_collisions = 0;
City city;
while (_q.Remove(&data)){
if (data){
bool load_city = true;
if (_read_count > 0){
load_city = _read_count > _read_cities;
}
if (!(*data).empty() && load_city){
if (create_city(*data, _countries, city)){
_read_cities++;
if (_read_cities % 50000 == 0){
Log("read " + StringUtils::get_string(_read_cities) + "...");
}
_cities.insert(city);
/*auto iter = _cities.find(city);
if (iter != _cities.end())
_collisions++;
else*/
}
}
delete data;
}
}
}
int receivePacket(string ip, uint port, string group, BlockingQueue<string> &receiveQueue)
{
try
{
int rsock;
rsock = get_receive_socket(ip, port, group);
// prepare a structure to put peer data into
struct sockaddr_in peer_address;
socklen_t peer_address_len;
peer_address_len = sizeof(struct sockaddr_storage);
// allocate memory to put the received data into
char data[1500];
int len;
len = 0;
while (1) {
// Receive packet and put its contents in data, recvfrom will block until a packet for this socket has been received
len = recvfrom(rsock, data, sizeof(data), 0, (struct sockaddr *) &peer_address, &peer_address_len);
if(len > 0){
//cout << "receiver received a packet" << endl;
receiveQueue.push(std::string(data, len));
}
}
} catch(std::exception &e)
{
std::cout << e.what() << std::endl;
exit(0);
}
return 0;
}
void monitor() {
while (true) {
boost::this_thread::sleep(boost::posix_time::milliseconds(1000));
{
LOG << "[monitor]"
<< " to read: " << chunksToRead.size()
<< " to compute md5: " << chunksToComputeMD5.size()
<< " skipped: " << chunksSkipped.size()
<< " finished: " << chunksFinished.size()
<< " failed: " << chunksFailed.size() << endl;
if (finished()) {
return;
}
}
}
}
void operator()(BlockingQueue<int> &queue) {
for (int i = 0; i < size; ++i) {
int value = product_item.fetch_add(1);
if (!queue.Push(value)) {
std::cout << "failed to push item " << value << " into queue\n";
}
}
}
void monitor() {
while (true) {
boost::this_thread::sleep(boost::posix_time::milliseconds(1000));
{
DXLOG(logINFO) << "[monitor]"
<< " to read: " << chunksToRead.size()
<< " to compress: " << chunksToCompress.size()
<< " to upload: " << chunksToUpload.size()
<< " finished: " << chunksFinished.size()
<< " failed: " << chunksFailed.size();
if (finished()) {
return;
}
}
}
}
void producer_put(int id, BlockingQueue & queue) {
for (int i = 0; i < 75; ++i) {
int data = i;
queue.put(data);
cout << "Producer " << id << " put " << data << endl;
this_thread::sleep_for(chrono::milliseconds(100));
}
}
void* thread_func(void *arg)
{
while(running)
{
Functor task = taskQueue.take();
task();
}
pthread_exit(NULL);
}
void test2()
{
for(int i = 0; i < 2500; i++)
{
//std::unique_lock<std::mutex> mlock(this->bqMutex);
std::cout<<"odtwarzam"<<i;
Pa_WriteStream(outputStream, bq.pop().getSample(), FRAMES_PER_BUFFER);
//mlock.unlock();
}
}
void readChunks(const vector<File> &files) {
try {
while (true) {
Chunk * c = chunksToRead.consume();
if (files[c->parentFileIndex].matchStatus == File::Status::FAILED_TO_MATCH_REMOTE_FILE) {
// We have already marked file as a non-match, don't waste time reading more chunks from it
c->log("File status == FAILED_TO_MATCH_REMOTE_FILE, Skipping the read...");
chunksSkipped.produce(c);
} else {
c->log("Reading...");
c->read();
c->log("Finished reading");
chunksToComputeMD5.produce(c);
}
}
} catch (boost::thread_interrupted &ti) {
return;
}
}
// Function for reterieving files which contains search text specified in the message
string Receiver::performTextSearch(string msg1, string &result)
{
Display * disp = new Display();
TextSearch *ts = new TextSearch();
Message *m = new Message();
BlockingQueue<std::string> resultfiles;
std::vector<string> file_patterns, all_files;
string sstr = m->getmsgBody1(msg1), file_path = m->getFileName(msg1), filepattern = m->getPatterns(msg1), token, resultfiles1;
disp->show("\n*******************************************\n");
string prtmsg ="Performing Text Search for string: "+ sstr + " on files with patterns " + filepattern+"\n";
disp->show(prtmsg);
std::istringstream ss(filepattern);
while (std::getline(ss, token, ','))
{
file_patterns.push_back(token);
}
all_files = showFiles(file_path, file_patterns);
for each (string file in all_files)
{
if (ts->search(sstr, file))
resultfiles.enQ(file);
}
size_t vecsize = resultfiles.size();
for (size_t i = 0; i < vecsize; i++)
{
resultfiles1 = resultfiles1 + resultfiles.deQ();
resultfiles1.append(";");
}
disp->show("Sending Result File list to client \n");
disp->show("*******************************************\n\n");
Message *resmsg = new Message();
Message orgmsg;
orgmsg.setMessage(msg1);
resmsg->setCommand("textsearch");
resmsg->setFileAttrs(orgmsg.getFileName(msg1));
resmsg->setsendAddr(orgmsg.getrecvAddr(msg1));
resmsg->setrecAddr(orgmsg.getsendAddr(msg1));
resmsg->setmsgBody(resultfiles1);
result = resmsg->getMessage();
return result;
}
void consumer_poll(int id, BlockingQueue & queue) {
int i = 0;
while (i < 50) {
int data;
if (queue.poll(data, chrono::milliseconds(250))) {
cout << "Consumer " << id << " poll " << data << endl;
i++;
continue;
}
cout << "Consumer " << id << " retry" << endl;
}
}
void producer_offer(int id, BlockingQueue & queue) {
int i = 0;
while (i < 75) {
int data = i;
if (queue.offer(data, chrono::milliseconds(100))) {
cout << "Producer " << id << " offer " << data << endl;
i++;
continue;
}
cout << "Producer " << id << " retry" << endl;
}
}
int main (int argc, char* argv[])
{
BlockingQueue<int> q;
auto t1 = std::async (std::launch::async, [&q] () {
for (int i = 0; i < 10; ++i) {
q.Put (i);
}
});
auto t2 = std::async (std::launch::async, [&q] () {
while (q.Size ()) {
std::cout << q.Take () << std::endl;
}
});
auto t3 = std::async (std::launch::async, [&q] () {
while (q.Size ()) {
std::cout << q.Take () << std::endl;
}
});
t1.wait ();
t2.wait ();
t3.wait ();
return 0;
}
