Example #1
0
/**
 * Abstract method Implementation - See MsgBusInterface.hpp for details
 *
 * TODO: Consolidate this to single produce method
 */
void msgBus_kafka::send_bmp_raw(u_char *r_hash, obj_bgp_peer &peer, u_char *data, size_t data_len) {
    string r_hash_str;
    string p_hash_str;
    RdKafka::Topic *topic = NULL;

    hash_toStr(peer.hash_id, p_hash_str);
    hash_toStr(r_hash, r_hash_str);

    if (data_len == 0)
        return;

    while (isConnected == false) {
        LOG_WARN("rtr=%s: Not connected to Kafka, attempting to reconnect", router_ip.c_str());
        connect();

        sleep(2);
    }

    char headers[256];
    size_t hdr_len = snprintf(headers, sizeof(headers), "V: %s\nC_HASH_ID: %s\nR_HASH: %s\nR_IP: %s\nL: %lu\n\n",
             MSGBUS_API_VERSION, collector_hash.c_str(), r_hash_str.c_str(), router_ip.c_str(), data_len);

    memcpy(producer_buf, headers, hdr_len);
    memcpy(producer_buf+hdr_len, data, data_len);

    topic = topicSel->getTopic(MSGBUS_TOPIC_VAR_BMP_RAW, &router_group_name, &peer_list[p_hash_str], peer.peer_as);
    if (topic != NULL) {
        SELF_DEBUG("rtr=%s: Producing bmp raw message: topic=%s key=%s, msg size = %lu", router_ip.c_str(),
                   topic->name().c_str(), r_hash_str.c_str(), data_len);

        RdKafka::ErrorCode resp = producer->produce(topic, RdKafka::Topic::PARTITION_UA,
                                                    RdKafka::Producer::RK_MSG_COPY /* Copy payload */,
                                                    producer_buf, data_len + hdr_len,
                                                    (const std::string *)&r_hash_str, NULL);

        if (resp != RdKafka::ERR_NO_ERROR)
            LOG_ERR("rtr=%s: Failed to produce bmp raw message: %s", router_ip.c_str(), RdKafka::err2str(resp).c_str());
    }
    else {
        SELF_DEBUG("rtr=%s: failed to produce bmp raw message because topic couldn't be found: topic=%s key=%s, msg size = %lu",
                   router_ip.c_str(), MSGBUS_TOPIC_VAR_BMP_RAW, r_hash_str.c_str(), data_len);
    }

    producer->poll(0);
}
Example #2
0
/**
 * produce message to Kafka
 *
 * \param [in] topic_var     Topic var to use in KafkaTopicSelector::getTopic() MSGBUS_TOPIC_VAR_*
 * \param [in] msg           message to produce
 * \param [in] msg_size      Length in bytes of the message
 * \param [in] rows          Number of rows
 * \param [in] key           Hash key
 * \param [in] peer_group    Peer group name - empty/NULL if not set or used
 * \param [in] peer_asn      Peer ASN
 */
void msgBus_kafka::produce(const char *topic_var, char *msg, size_t msg_size, int rows, string key,
                           const string *peer_group, uint32_t peer_asn) {
    size_t len;
    RdKafka::Topic *topic = NULL;

    while (isConnected == false or topicSel == NULL) {
        // Do not attempt to reconnect if this is the main process (router ip is null)
        // Changed on 10/29/15 to support docker startup delay with kafka
        /*
        if (router_ip.size() <= 0) {
            return;
        }*/

        LOG_WARN("rtr=%s: Not connected to Kafka, attempting to reconnect", router_ip.c_str());
        connect();

        sleep(1);
    }

    char headers[256];
    len = snprintf(headers, sizeof(headers), "V: %s\nC_HASH_ID: %s\nL: %lu\nR: %d\n\n",
            MSGBUS_API_VERSION, collector_hash.c_str(), msg_size, rows);

    memcpy(producer_buf, headers, len);
    memcpy(producer_buf+len, msg, msg_size);


    topic = topicSel->getTopic(topic_var, &router_group_name, peer_group, peer_asn);
    if (topic != NULL) {
        SELF_DEBUG("rtr=%s: Producing message: topic=%s key=%s, msg size = %lu", router_ip.c_str(),
                   topic->name().c_str(), key.c_str(), msg_size);

        RdKafka::ErrorCode resp = producer->produce(topic, RdKafka::Topic::PARTITION_UA,
                                                    RdKafka::Producer::RK_MSG_COPY,
                                                    producer_buf, msg_size + len,
                                                    (const std::string *) &key, NULL);
        if (resp != RdKafka::ERR_NO_ERROR)
            LOG_ERR("rtr=%s: Failed to produce message: %s", router_ip.c_str(), RdKafka::err2str(resp).c_str());
    } else {
        LOG_NOTICE("rtr=%s: failed to produce message because topic couldn't be found: topic=%s key=%s, msg size = %lu", router_ip.c_str(),
                   topic_var, key.c_str(), msg_size);
    }

    producer->poll(0);
}
int main (int argc, char **argv) {
  std::string brokers = "localhost";
  std::string errstr;
  std::vector<std::string> topics;
  std::string conf_file;
  std::string mode = "P";
  int throughput = 0;
  int32_t partition = RdKafka::Topic::PARTITION_UA;
  bool do_conf_dump = false;
  MyHashPartitionerCb hash_partitioner;

  /*
   * Create configuration objects
   */
  RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
  RdKafka::Conf *tconf = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC);

  {
    char hostname[128];
    gethostname(hostname, sizeof(hostname)-1);
    conf->set("client.id", std::string("rdkafka@") + hostname, errstr);
  }

  conf->set("debug", "cgrp,topic", errstr);

  for (int i = 1 ; i < argc ; i++) {
    const char *name = argv[i];
    const char *val = i+1 < argc ? argv[i+1] : NULL;

    if (val && !strncmp(val, "--", 2))
      val = NULL;

    std::cout << now() << ": argument: " << name << " " <<
        (val?val:"") << std::endl;
    
    if (val) {
      if (!strcmp(name, "--topic"))
	topics.push_back(val);
      else if (!strcmp(name, "--broker-list"))
	brokers = val;
      else if (!strcmp(name, "--max-messages"))
	state.maxMessages = atoi(val);
      else if (!strcmp(name, "--throughput"))
	throughput = atoi(val);
      else if (!strcmp(name, "--producer.config") ||
	       !strcmp(name, "--consumer.config"))
	read_conf_file(val);
      else if (!strcmp(name, "--group-id"))
	conf->set("group.id", val, errstr);
      else if (!strcmp(name, "--session-timeout"))
	conf->set("session.timeout.ms", val, errstr);
      else if (!strcmp(name, "--reset-policy")) {
	if (tconf->set("auto.offset.reset", val, errstr)) {
	  std::cerr << now() << ": " << errstr << std::endl;
	  exit(1);
	}
      } else if (!strcmp(name, "--debug")) {
	conf->set("debug", val, errstr);
      } else {
	std::cerr << now() << ": Unknown option " << name << std::endl;
	exit(1);
      }

      i++;

    } else {
      if (!strcmp(name, "--consumer"))
	mode = "C";
      else if (!strcmp(name, "--producer"))
	mode = "P";
      else if (!strcmp(name, "--enable-autocommit")) {
	state.consumer.useAutoCommit = true;
	conf->set("enable.auto.commit", "true", errstr);
      } else {
	std::cerr << now() << ": Unknown option or missing argument to " << name << std::endl;
	exit(1);
      }
    }
  }

  if (topics.empty() || brokers.empty()) {
    std::cerr << now() << ": Missing --topic and --broker-list" << std::endl;
    exit(1);
  }


  /*
   * Set configuration properties
   */
  conf->set("metadata.broker.list", brokers, errstr);

  ExampleEventCb ex_event_cb;
  conf->set("event_cb", &ex_event_cb, errstr);

  if (do_conf_dump) {
    int pass;

    for (pass = 0 ; pass < 2 ; pass++) {
      std::list<std::string> *dump;
      if (pass == 0) {
        dump = conf->dump();
        std::cerr << now() << ": # Global config" << std::endl;
      } else {
        dump = tconf->dump();
        std::cerr << now() << ": # Topic config" << std::endl;
      }

      for (std::list<std::string>::iterator it = dump->begin();
           it != dump->end(); ) {
        std::cerr << *it << " = ";
        it++;
        std::cerr << *it << std::endl;
        it++;
      }
      std::cerr << std::endl;
    }
    exit(0);
  }

  signal(SIGINT, sigterm);
  signal(SIGTERM, sigterm);
  signal(SIGALRM,  sigwatchdog);


  if (mode == "P") {
    /*
     * Producer mode
     */

    ExampleDeliveryReportCb ex_dr_cb;

    /* Set delivery report callback */
    conf->set("dr_cb", &ex_dr_cb, errstr);

    /*
     * Create producer using accumulated global configuration.
     */
    RdKafka::Producer *producer = RdKafka::Producer::create(conf, errstr);
    if (!producer) {
      std::cerr << now() << ": Failed to create producer: " << errstr << std::endl;
      exit(1);
    }

    std::cerr << now() << ": % Created producer " << producer->name() << std::endl;

    /*
     * Create topic handle.
     */
    RdKafka::Topic *topic = RdKafka::Topic::create(producer, topics[0],
						   tconf, errstr);
    if (!topic) {
      std::cerr << now() << ": Failed to create topic: " << errstr << std::endl;
      exit(1);
    }

    static const int delay_us = throughput ? 1000000/throughput : 0;

    if (state.maxMessages == -1)
      state.maxMessages = 1000000; /* Avoid infinite produce */

    for (int i = 0 ; run && i < state.maxMessages ; i++) {
      /*
       * Produce message
       */
      std::ostringstream msg;
      msg << i;
      RdKafka::ErrorCode resp =
	producer->produce(topic, partition,
			  RdKafka::Producer::RK_MSG_COPY /* Copy payload */,
			  const_cast<char *>(msg.str().c_str()),
			  msg.str().size(), NULL, NULL);
      if (resp != RdKafka::ERR_NO_ERROR) {
	errorString("producer_send_error",
		    RdKafka::err2str(resp), topic->name(), NULL, msg.str());
	state.producer.numErr++;
      } else {
	std::cerr << now() << ": % Produced message (" <<
	  msg.str().size() << " bytes)" << std::endl;
	state.producer.numSent++;
      }

      producer->poll(delay_us / 1000);
      watchdog_kick();
    }
    run = true;

    while (run && producer->outq_len() > 0) {
      std::cerr << now() << ": Waiting for " << producer->outq_len() << std::endl;
      producer->poll(50);
      watchdog_kick();
    }

    std::cerr << now() << ": " << state.producer.numAcked << "/" <<
      state.producer.numSent << "/" << state.maxMessages <<
      " msgs acked/sent/max, " << state.producer.numErr <<
      " errored" << std::endl;

    delete topic;
    delete producer;


  } else if (mode == "C") {
    /*
     * Consumer mode
     */

    tconf->set("auto.offset.reset", "smallest", errstr);

    /* Set default topic config */
    conf->set("default_topic_conf", tconf, errstr);

    ExampleRebalanceCb ex_rebalance_cb;
    conf->set("rebalance_cb", &ex_rebalance_cb, errstr);

    ExampleOffsetCommitCb ex_offset_commit_cb;
    conf->set("offset_commit_cb", &ex_offset_commit_cb, errstr);


    /*
     * Create consumer using accumulated global configuration.
     */
    consumer = RdKafka::KafkaConsumer::create(conf, errstr);
    if (!consumer) {
      std::cerr << now() << ": Failed to create consumer: " <<
	errstr << std::endl;
      exit(1);
    }

    std::cerr << now() << ": % Created consumer " << consumer->name() <<
      std::endl;

    /*
     * Subscribe to topic(s)
     */
    RdKafka::ErrorCode resp = consumer->subscribe(topics);
    if (resp != RdKafka::ERR_NO_ERROR) {
      std::cerr << now() << ": Failed to subscribe to " << topics.size() << " topics: "
		<< RdKafka::err2str(resp) << std::endl;
      exit(1);
    }

    /*
     * Consume messages
     */
    while (run) {
      RdKafka::Message *msg = consumer->consume(500);
      msg_consume(consumer, msg, NULL);
      delete msg;
      watchdog_kick();
    }

    /* Final commit */
    do_commit(consumer, 1);

    /*
     * Stop consumer
     */
    consumer->close();

    delete consumer;
  }


  /*
   * Wait for RdKafka to decommission.
   * This is not strictly needed (when check outq_len() above), but
   * allows RdKafka to clean up all its resources before the application
   * exits so that memory profilers such as valgrind wont complain about
   * memory leaks.
   */
  RdKafka::wait_destroyed(5000);

  std::cerr << now() << ": EXITING WITH RETURN VALUE 0" << std::endl;
  return 0;
}