예제 #1
0
// Actually only used for movie requests
vector<pair<char, string> > KeyValueStore::forwardLongRequest(Protocol::KEYVALUE_STORE type, const string& req, Config::ThreadControl& control)
{
  vector<pair<char, string> > ret;
  Mutex& outMutex = control.getOutMutex();
  Condition& cond = control.getOutCondition();

  procForward.lock();
  pair<char, string> freq(type, req);
  forwardQueue.enqueue(freq);
  
  outMutex.lock();
  cond.signal();
  outMutex.unlock();
  forwardCond.wait(procForward);

  while(true) {
    pair<char, string> resp(respBuffer.dequeue());
    ret.push_back(resp);
    if(resp.first == Protocol::MLOOKUP_DONE || resp.first == Protocol::LOOKUP_NO) {
      procForward.unlock();
      break;
    }
  }

  printKv("Done serving movie list");

  return ret;
}
 void destroy_ConnectionPool(void) {
     // pop until queue is empty
     photodb_client client;
     while (!client_queue.size() != 0) {
         client_queue.pop(client, 1, 0);
     }
     cout << "release OK" << endl;
 }
예제 #3
0
void *ExporterSink::exporterSinkProcess(void *arg)
{
	ExporterSink *sink = (ExporterSink *)arg;
	ConcurrentQueue<Packet*> *queue = sink->getQueue();
	Packet *p;
	bool result;
	// our deadline
	struct timeval deadline;
	int pckCount;

	msg(MSG_INFO, "Sink: now running ExporterSink thread");

	while(!sink->exitFlag) {
		sink->startNewPacketStream();

		// let's get the first packet
		gettimeofday(&deadline, 0);
		
		result = queue->pop(&p);

	    if(result == true) {
			// we got a packet, so let's add the record
			result = sink->addPacket(p);
		}
			
		pckCount = 1;

		// now calculate the deadline by which the packet has to leave the exporter
		gettimeofday(&deadline, 0);
		deadline.tv_usec += sink->exportTimeout * 1000L;
		if(deadline.tv_usec > 1000000L) {
			deadline.tv_sec += (deadline.tv_usec / 1000000L);
			deadline.tv_usec %= 1000000L;
		}

		while(!sink->exitFlag && (pckCount < sink->ipfix_maxrecords)) {
			// Try to get next packet from queue before our deadline
			result = queue->popAbs(deadline, &p);

			// check for timeout and break loop if neccessary
			if (!result) break;

			// no timeout received, continue waiting, but

			// count only if packet was added
			if(sink->addPacket(p) == true)
			    pckCount++;
		}
		sink->flushPacketStream();
	}
	return 0;
}
예제 #4
0
pair<char, string> KeyValueStore::forwardRequest(Protocol::KEYVALUE_STORE type, const string& req, Config::ThreadControl& control)
{
  Mutex& outMutex = control.getOutMutex();
  Condition& cond = control.getOutCondition();

  procForward.lock();
  pair<char, string> freq(type, req);
  forwardQueue.enqueue(freq);
  outMutex.lock();
  cond.signal();
  outMutex.unlock();
  forwardCond.wait(procForward);
  pair<char, string> resp(respBuffer.dequeue());
  procForward.unlock();

  return resp;
}
void AggregationPerfTest::start(unsigned int numpackets)
{
	char packetdata[] = { 0x00, 0x12, 0x1E, 0x08, 0xE0, 0x1F, 0x00, 0x15, 0x2C, 0xDB, 0xE4, 0x00, 0x08, 0x00, 0x45, 0x00, 
						  0x00, 0x2C, 0xEF, 0x42, 0x40, 0x00, 0x3C, 0x06, 0xB3, 0x51, 0xC3, 0x25, 0x84, 0xBE, 0x5B, 0x20, 
						  0xF9, 0x33, 0x13, 0x8B, 0x07, 0x13, 0x63, 0xF2, 0xA0, 0x06, 0x2D, 0x07, 0x36, 0x2B, 0x50, 0x18, 
						  0x3B, 0x78, 0x67, 0xC9, 0x00, 0x00, 0x6F, 0x45, 0x7F, 0x40 };
	unsigned int packetdatalen = 58;

	// just push our sample packet a couple of times into the filter
	struct timeval curtime;
	BOOST_REQUIRE(gettimeofday(&curtime, 0) == 0);

	ConcurrentQueue<Packet*>* filterq = filter->getQueue();
	for (unsigned int i=0; i<numpackets; i++) {
		Packet* p = packetManager->getNewInstance();
		p->init((char*)packetdata, packetdatalen, curtime);
		filterq->push(p);
	}
}
예제 #6
0
TEST(ConcurrentQueue, ParallelPushPopAscIntegerAndCalculateTotalSum) {
    tlx::ThreadPool pool(8);

    ConcurrentQueue<size_t, std::allocator<size_t> > queue;
    std::atomic<size_t> count(0);
    std::atomic<size_t> total_sum(0);

    static constexpr size_t num_threads = 4;
    static constexpr size_t num_pushes = 10000;

    // have threads push items

    for (size_t i = 0; i != num_threads; ++i) {
        pool.enqueue([&queue]() {
                         for (size_t i = 0; i != num_pushes; ++i) {
                             queue.push(i);
                         }
                     });
    }

    // have threads try to pop items.

    for (size_t i = 0; i != num_threads; ++i) {
        pool.enqueue([&]() {
                         while (count != num_threads * num_pushes) {
                             size_t item;
                             while (queue.try_pop(item)) {
                                 total_sum += item;
                                 ++count;
                             }
                         }
                     });
    }

    pool.loop_until_empty();

    ASSERT_TRUE(queue.empty());
    ASSERT_EQ(count, num_threads * num_pushes);
    // check total sum, no item gets lost?
    ASSERT_EQ(total_sum, num_threads * num_pushes * (num_pushes - 1) / 2);
}
예제 #7
0
void KeyValueStore::doReplicate(Protocol::KEYVALUE_STORE type, const string& req, Config::ThreadControl& control, int remaining)
{
  Mutex& mutex = control.getOutMutex();

  // First craft message
  pair<char, string> entry(type, KeyValueStore::createReplMsg(req, remaining));

  forwardQueue.enqueue(entry);

  mutex.lock();
  control.getOutCondition().signal();
  mutex.unlock();
}
    void init_thrift_connection_pool(unsigned int max_client, char* meta_server_address, int meta_server_port, char* content_server_address, int content_server_port) {
        if (client_queue.size() > 0) return;
        for (unsigned int i = 0; i < max_client; i++) {
            // init a client
            photodb_client *client = new photodb_client;

            boost::shared_ptr<TTransport> content_socket(new TSocket(content_server_address, content_server_port));
            boost::shared_ptr<TTransport> content_transport(new TFramedTransport(content_socket));
            boost::shared_ptr<TProtocol> content_protocol(new TBinaryProtocol(content_transport));
            try {
                content_transport->open();
                PhotoDBClient *client_content = new PhotoDBClient(content_protocol);
                boost::shared_ptr<TTransport> metadata_socket(new TSocket(meta_server_address, meta_server_port));
                boost::shared_ptr<TTransport> metadata_transport(new TFramedTransport(metadata_socket));
                boost::shared_ptr<TProtocol> metadata_protocol(new TBinaryProtocol(metadata_transport));
                try {
                    //cout << i << endl;
                    metadata_transport->open();
                    PhotoDBClient *client_metadata = new PhotoDBClient(metadata_protocol);

                    client->content = client_content;
                    client->metadata = client_metadata;

                    // Put to queue
                    client_queue.put(*client, 1, 0);

                } catch (TException& tx) {
                    cout << "Can not Open meta PORT number: " << meta_server_port << endl;
                    cout << "Can not Open meta host : " << meta_server_address << endl;
                    cout << "ERROR OPEN: " << tx.what() << endl;
                }
            } catch (TException& tx) {
                cout << "Can not Open content PORT number: " << content_server_port << endl;
                cout << "Can not Open meta host : " << content_server_address << endl;
                cout << "ERROR OPEN: " << tx.what() << endl;
            }
        }
    }
예제 #9
0
// Color image call back
void imgRcvd( const sensor_msgs::ImageConstPtr& msg )
{
    try
    {
        cv::Mat im = cv_bridge::toCvCopy(msg, "bgr8")->image;
        cv::Mat im_resized;
        cv::resize( im, im_resized, cv::Size(), 0.5, 0.5 );

        colorQueue.push(im_resized);
//        cv::imshow("view", cv_bridge::toCvShare(msg, "bgr8")->image);
//        cv::waitKey(30);
    }
    catch (cv_bridge::Exception& e)
    {
        ROS_ERROR("Could not convert from '%s' to 'bgr8'.", msg->encoding.c_str());
    }
}
예제 #10
0
//depth image call back
void depthRcvd( const sensor_msgs::ImageConstPtr& msg )
{
    try
    {
        cv::Mat depth = cv_bridge::toCvCopy(msg, "")->image;
        //depth mask nan
        cv::Mat mask = cv::Mat(depth != depth);
        depth.setTo( 0, mask );

        cv::Mat depth_resized;
        cv::resize( depth, depth_resized, cv::Size(), 0.5, 0.5 );

        depthQueue.push(depth_resized);

//        cv::imshow("view-depth", cv_bridge::toCvShare(msg, "")->image);
//        cv::waitKey(30);
    }
    catch (cv_bridge::Exception& e)
    {
        ROS_ERROR("Could not convert from '%s' to 'bgr8'.", msg->encoding.c_str());
    }
}
예제 #11
0
void queueConsumer( )
{
    cv::Mat im, depth;
    ros::Rate rate(30);
    int nFrame = 0;
    while( ros::ok() )
    {
        //TODO :
        // This synchronization is not perfect. Should ideally also push the timestamps to the queue and do
        // it based on time stamps of the pushed images
        cout << "size : "<< colorQueue.getSize() << "  " << depthQueue.getSize() << endl;
        if( colorQueue.getSize() < 1 || depthQueue.getSize() < 1 )
        {
            ros::spinOnce();
            rate.sleep();
            continue;
        }

        bool pop_im_flag = colorQueue.try_pop(im);
        bool pop_depth_flag = depthQueue.try_pop(depth);



        cout << "im.type() : " << type2str( im.type() ) << endl;
        cout << "depth.type() : " << type2str( depth.type() ) << endl;
        double minVal, maxVal;
        cv::minMaxLoc( depth, &minVal, &maxVal );
        cout << "depth min : "<< minVal << endl;
        cout << "depth max : "<< maxVal << endl;

        // save to file
        write_to_opencv_file( nFrame, im, depth );

        cv::imshow( "im", im );
        cv::Mat falseColorsMap;
        applyColorMap(depth, falseColorsMap, cv::COLORMAP_AUTUMN);
        cv::imshow( "depth", depth );
        cv::waitKey(1);


        ros::spinOnce();
        rate.sleep();
        nFrame++;
    }
}
예제 #12
0
void KeyValueStore::InternalThread::outgoingRoutine()
{
  unsigned currId = -1;
  const Config::ServerInformation& info = config.getServerInformation();
  Config::ThreadControl& control = config.getThreadControl();
  Mutex& outMutex = control.getOutMutex();
  Condition& cond = control.getOutCondition();
  Timespan timeout(3, 0); // 3 seconds to receive response
  bool connected = false;
  unsigned contentLength = 0;
  char* buf = NULL;

  while(control.isLive()) {
    outMutex.lock();
    cond.wait(outMutex);

    // Check if server changed
    if(control.getReconnectedTo()) {
      printKv("Reconnecting");
      currId = control.getConnectedToId();
      const Config::ServerInformation& nextInfo = config.getServerInformation(currId);
      try {
        if(connected)
          sock.close();
        sock.connect(SocketAddress(nextInfo.address, nextInfo.internalPort), Timespan(5, 0));
        connected = true;
        printKv("Connected to outgoing server: "<< nextInfo.id);
        //sock.setReceiveTimeout(timeout);
        KeyValueStore::replicateDB(sock, control.getMachineCount());
      } catch(Exception& e) {
        printKv("Could not connect to next key-value internal server ("<< e.displayText() <<")" << endl
          << "System may be inconsistent until next reconnect.");
        connected = false;
      }
    }

    while(!forwardQueue.isEmpty()) {
      pair<char, string> req(forwardQueue.dequeue());
      if(req.first == Protocol::REPL_INS || req.first == Protocol::REPL_REM) {
        try {
          KeyValueStore::sendRequest(sock, req);
        } catch(Exception& e) {
          printKv("Could not replicate: "<< e.displayText());
        }
      } else if(req.first == Protocol::INS_MOVIE) { // Requests without responses
        KeyValueStore::sendRequest(sock, req); // Not waiting for a response
      } else if(req.first == Protocol::LUP_MOVIE && false) { // This returns many responses need to get all
        printKv("Received movie lookup forward");
        KeyValueStore::sendRequest(sock, req);
        
        pair<char, string> resp(KeyValueStore::rcvResponse(sock, buf));
        while(resp.first != Protocol::MLOOKUP_DONE && resp.first != Protocol::LOOKUP_NO) {
          resp = KeyValueStore::rcvResponse(sock, buf);
          respBuffer.enqueue(resp);
//          replyForward(); // Send as we get them
        }
        respBuffer.enqueue(resp); // Make sure we also send our done response

        printKv("Served movies");

        replyForward();
      } else {
        // Forward and wait for response.
        try {
          KeyValueStore::sendRequest(sock, req);

          pair<char, string> resp(KeyValueStore::rcvResponse(sock, buf));
          respBuffer.enqueue(resp);

          replyForward();
        } catch(Exception& e) {
          // If we timeout or anything, treat request as failed
          printKv("Exception while forwarding: "<< e.displayText());
          respBuffer.enqueue(pair<char, string>(Protocol::REQ_FAIL, ""));
          replyForward();
        }
      }
    }
 //   if(!respBuffer.isEmpty())
   //   replyForward();

    outMutex.unlock();
  }
}
예제 #13
0
// 模仿java.util.concurrent库并发容器及Mircosoft的并发容器而写的一个简单并发队列
// http://docs.oracle.com/javase/7/docs/api/java/util/concurrent/BlockingQueue.html
// http://docs.oracle.com/javase/7/docs/api/java/util/concurrent/LinkedBlockingQueue.html
// http://docs.oracle.com/javase/7/docs/api/java/util/concurrent/ConcurrentLinkedQueue.html
// http://docs.oracle.com/javase/7/docs/api/java/util/concurrent/PriorityBlockingQueue.html
int main()
{
    {
        FifoConcurrentQueue<int> cq;
        cq.push(3);
        cq.push(1);
        cq.push(2);

        int i;
        cq.pop(i);
        assert(i == 3);
        cq.pop(i);
        assert(i == 1);
        cq.pop(i);
        assert(i == 2);
    }

    {
        FiloConcurrentQueue<int> cq;
        cq.push(3);
        cq.push(1);
        cq.push(2);

        int i;
        cq.pop(i);
        assert(i == 2);
        cq.pop(i);
        assert(i == 1);
        cq.pop(i);
        assert(i == 3);
    }

    {
        PrioConcurrentQueue<int> cq;
        cq.push(3);
        cq.push(1);
        cq.push(2);

        int i;
        cq.pop(i);
        assert(i == 3);
        cq.pop(i);
        assert(i == 2);
        cq.pop(i);
        assert(i == 1);
    }
    {
        ConcurrentQueue<int, std::priority_queue<int, std::vector<int>, std::greater<int> >, ConcurrentQueueTraits::tagPRIO> cq;
        cq.push(3);
        cq.push(1);
        cq.push(2);

        int i;
        cq.pop(i);
        assert(i == 1);
        cq.pop(i);
        assert(i == 2);
        cq.pop(i);
        assert(i == 3);
    }

}
예제 #14
0
int main(int argc, char *argv[]) {
	// read config.ini file
	Config config;
	auto nthreads  = config.get<int>("nthreads");
	auto device    = config.get<const char *>("disk_guest");
	auto traceFile = config.get<const char *>("trace_file");
	auto logDir    = config.get<string>("log_dir");
	
	// parse arg, prioritize argv over config
	if (argc > 1) traceFile = argv[1]; 
	if (argc > 2) nthreads = atoi(argv[2]);
	if (strstr(device, "/dev/sda")) { // avoid accidentally writing to system part 
		fprintf(stderr, "Error trying to write to system partition %s\n", device);
		return 1;
	}
	// use default value if not supplied
	if (strcmp(device, "") == 0) device = DEFAULT_DEVICE;
	if (strcmp(traceFile, "") == 0) traceFile = DEFAULT_TRACE_FILE;
	if (strcmp(logDir.c_str(), "") == 0) logDir = DEFAULT_LOG_DIR;
	if (nthreads == 0) nthreads = DEFAULT_NTHREADS;
	
	srand(time(NULL)); 	// initialize seed

	// print configuration
	printf("trace     : %s\n", traceFile);
	printf("nthreads  : %d\n", nthreads);
	printf("device    : %s\n", device);
	printf("log       : %s\n", logDir.c_str());
	printf("precision : %fms\n", Timer::getResolution());

	printf("Opening device %s\n", device);
	int fd = open(device, O_DIRECT | O_RDWR | O_SYNC); 
	if (fd < 0) {
		fprintf(stderr, "Error opening device '%s'\n", device);
		return 1;
	}

	printf("Allocating buffer\n");
	void *buf; 
	if (posix_memalign(&buf, MEM_ALIGN, LARGEST_REQUEST_SIZE * BYTE_PER_BLOCK)) {
		fprintf(stderr, "Error allocating buffer\n");
		return 1;
	}
	//memset(buf, rand() % 256, LARGEST_REQUEST_SIZE * BYTE_PER_BLOCK);

	printf("Opening trace file\n");
	TraceReader trace(traceFile); // open trace file
	ConcurrentQueue<TraceEvent> queue; // queue of trace events
	bool readDone = false; // whether or not we're done reading trace file
	
	printf("Start reading trace\n");
	thread fileThread([&] { // thread to read trace file 
		TraceEvent event;
		while (trace.read(event)) {
			event.time = event.time * 1000; // to microseconds
			event.size = event.bcount * BYTE_PER_BLOCK;
			queue.push(event);
		}
		readDone = true; 
		queue.notifyAll(); // notify worker we're done  
	});	
	queue.waitUntilFull(); // wait until at least queue's full

	printf("Start replaying trace\n");
	vector<thread> workers(nthreads); // generate worker threads
	atomic<int> lateCount(0), threadId(0); // late I/O count and threadId
	for (auto& t : workers) t = thread([&] { // launch workers 
		int myId = ++threadId; // id for this thread
		int myLateCount = 0; // local lateCount for this thread 
		Logger logger(logDir + traceFile + to_string(myId));
		
		Timer timer; // mark the beginning of worker thread	
		while (!readDone or !queue.empty()) { 
			TraceEvent event;
			if (not queue.pop(event)) continue; // retry 
			long currentTime = timer.elapsedTime(), nextIoTime = event.time;
			if (currentTime <= nextIoTime) { // we're early/on-time
				//printf(". next=%ld current=%ld \n", nextIoTime, currentTime);
				Timer::delay(nextIoTime - currentTime); // delay until ~specified time
			} else { // we're late
				//printf("x next=%ld current=%ld delta=%ld\n", 
				// nextIoTime, currentTime, currentTime-nextIoTime);
				++myLateCount;
			}
			
			performIo(fd, buf, event, logger);
		}
		lateCount += myLateCount; // update global lateCount
	});

	fileThread.join(); // wait for all threads to finish
	for (auto& t : workers) t.join(); 

	printf("Late count: %d\n", lateCount.load());
	Logger logger(logDir + traceFile + to_string(0));
	logger.printf("%d\n", lateCount.load());
	printf("Done\n");
	return 0;
}
    return_value *kv_up_get(unsigned long key_get, unsigned long size) {
        try {
            return_value *value_return = new return_value;

            photodb_client *client = new photodb_client;

            // wait 2 seconds 
            client_queue.pop(*client, 1, 2);
            // get metadata
            MetaValueResult metadata_result;
            try {
                client->metadata->getMeta(metadata_result, (long int) key_get); // Should check error
                if (metadata_result.error != 0) {
                    client_queue.put(*client, 1, 0);
                    delete value_return;
                    return NULL;
                }
            } catch (TException& tx) {
                cout << "ERROR GET META: " << tx.what() << endl;
                client_queue.put(*client, 1, 0);
                delete value_return;
                return NULL;
            }

            // get content of image
            ImgValueResult content_result;
            try {
                client->content->getImg(content_result, (long int) key_get, (int) size);
                if (content_result.error != 0) {
                    client_queue.put(*client, 1, 0);
                    delete value_return;
                    return NULL;
                }
            } catch (TException& tx) {
                cout << "ERROR GET META: " << tx.what() << endl;
                client_queue.put(*client, 1, 0);
                delete value_return;
                return NULL;
            }

            client_queue.put(*client, 1, 0);

            char *writable_content = new char[content_result.value.img.size() + 1];
            std::copy(content_result.value.img.begin(), content_result.value.img.end(), writable_content);
            writable_content[content_result.value.img.size() + 1] = '\0';
            // Prepare date to return
            value_return->content = writable_content;
            value_return->size = content_result.value.img.size();

            char *writable_etag = new char[metadata_result.value.etag.size() + 1];
            std::copy(metadata_result.value.etag.begin(), metadata_result.value.etag.end(), writable_etag);
            writable_etag[metadata_result.value.etag.size() + 1] = '\0';
            // Prepare etag to return
            value_return->etag = writable_etag;
            value_return->etag_size = metadata_result.value.etag.size();

            char *writable_contentType = new char[metadata_result.value.contentType.size() + 1];
            std::copy(metadata_result.value.contentType.begin(), metadata_result.value.contentType.end(), writable_contentType);
            writable_contentType[metadata_result.value.contentType.size()] = '\0';
            value_return->contentType = writable_contentType;
            value_return->content_type_size = metadata_result.value.contentType.size();

            return value_return;
        } catch (const exception& e) {
            cerr << "EXCEPTION: " << e.what() << endl;
            return NULL;
        }
    }