Exemplo n.º 1
0
Kafka::Kafka(std::string brokers, std::string topic)
{
    std::string err_str;
    m_partition = RdKafka::Topic::PARTITION_UA;

    RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
    RdKafka::Conf *tconf = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC);

    conf->set("metadata.broker.list", brokers, err_str);

    MyEventCb my_event_cb;
    conf->set("event_cb", &my_event_cb, err_str);

    MyDeliveryReportCb my_dr_cb;
    conf->set("dr_cb", &my_dr_cb, err_str);

    k_producer = RdKafka::Producer::create(conf, err_str);
    if (!k_producer) {
        std::cerr << "Failed to create producer: " << err_str << std::endl;
        exit(1);
    }
    std::cout << "% Created producer " << k_producer->name() << std::endl;
    k_topic = RdKafka::Topic::create(k_producer, topic, tconf, err_str);
    if (!k_topic) {
        std::cerr << "Failed to create topic: " << err_str << std::endl;
        exit(1);
    }
}
Exemplo n.º 2
0
  int main_0095_all_brokers_down (int argc, char **argv) {
    RdKafka::Conf *conf;
    std::string errstr;

    Test::conf_init(&conf, NULL, 20);
    /* Two broker addresses that will quickly reject the connection */
    Test::conf_set(conf, "bootstrap.servers", "127.0.0.1:1,127.0.0.1:2");

    /*
     * First test producer
     */
    errorEventCb pEvent = errorEventCb();

    if (conf->set("event_cb", &pEvent, errstr) != RdKafka::Conf::CONF_OK)
      Test::Fail(errstr);

    Test::Say("Test Producer\n");

    RdKafka::Producer *p = RdKafka::Producer::create(conf, errstr);
    if (!p)
      Test::Fail("Failed to create Producer: " + errstr);

    /* Wait for all brokers down */
    while (!pEvent.error_seen)
      p->poll(1000);

    delete p;


    /*
     * Test high-level consumer that has a logical broker (group coord),
     * which has caused AllBrokersDown generation problems (#2259)
     */
    errorEventCb cEvent = errorEventCb();

    Test::conf_set(conf, "group.id", "test");

    if (conf->set("event_cb", &cEvent, errstr) != RdKafka::Conf::CONF_OK)
      Test::Fail(errstr);

    Test::Say("Test KafkaConsumer\n");

    RdKafka::KafkaConsumer *c = RdKafka::KafkaConsumer::create(conf, errstr);
    if (!c)
      Test::Fail("Failed to create KafkaConsumer: " + errstr);

    /* Wait for all brokers down */
    while (!cEvent.error_seen) {
      RdKafka::Message *m = c->consume(1000);
      if (m)
        delete m;
    }

    c->close();

    delete c;

    return 0;
  }
Exemplo n.º 3
0
 void KafkaLogProducer::configure(){
     //todo: modified with configmanager
     ConfigManager& configManager = ConfigManager::getInstance();
     LogConfig* logConfig = (LogConfig*)configManager.get(CONFIG_LOG);
     topicName = logConfig->kafkaTopic;
     logKey = logConfig->kafkaKey;
     std::string brokers = logConfig->kafkaBroker;
     RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
     RdKafka::Conf *tconf = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC);
     std::string errstr;
     RdKafka::Conf::ConfResult res;
     if((res=conf->set("metadata.broker.list", brokers, errstr))!=RdKafka::Conf::CONF_OK ||
             (res = conf->set("event_cb", &eventCb, errstr))!= RdKafka::Conf::CONF_OK ||
             (res=conf->set("dr_cb", &drCb, errstr))!= RdKafka::Conf::CONF_OK ||
             (res=conf->set("queue.buffering.max.messages",logConfig->kafkaMQMaxSize,errstr))!=RdKafka::Conf::CONF_OK||
             (res=conf->set("message.send.max.retries","3",errstr))!= RdKafka::Conf::CONF_OK ||
             (res=conf->set("retry.backoff.ms","500",errstr))!=RdKafka::Conf::CONF_OK){
         LOG_ERROR<<"error occured when configuring kafka log producer,"<<errstr;
         throw LogClientException(errstr,-1);
     }
     producer = RdKafka::Producer::create(conf, errstr);
     if (!producer) {
         LOG_ERROR<<"error occured when configuring kafka log producer" << errstr;
         throw LogClientException(errstr,-1);
     }
     topic = RdKafka::Topic::create(producer, topicName, tconf, errstr);
     if (!topic) {
         LOG_ERROR<<"error occured when configuring kafka log producer" << errstr;
         throw LogClientException(errstr,-1);
     }
 }
int main (int argc, char **argv) {
  std::string brokers = "localhost";
  std::string errstr;
  std::vector<std::string> topics;
  std::string conf_file;
  std::string mode = "P";
  int throughput = 0;
  int32_t partition = RdKafka::Topic::PARTITION_UA;
  bool do_conf_dump = false;
  MyHashPartitionerCb hash_partitioner;

  /*
   * Create configuration objects
   */
  RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
  RdKafka::Conf *tconf = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC);

  {
    char hostname[128];
    gethostname(hostname, sizeof(hostname)-1);
    conf->set("client.id", std::string("rdkafka@") + hostname, errstr);
  }

  conf->set("debug", "cgrp,topic", errstr);

  for (int i = 1 ; i < argc ; i++) {
    const char *name = argv[i];
    const char *val = i+1 < argc ? argv[i+1] : NULL;

    if (val && !strncmp(val, "--", 2))
      val = NULL;

    std::cout << now() << ": argument: " << name << " " <<
        (val?val:"") << std::endl;
    
    if (val) {
      if (!strcmp(name, "--topic"))
	topics.push_back(val);
      else if (!strcmp(name, "--broker-list"))
	brokers = val;
      else if (!strcmp(name, "--max-messages"))
	state.maxMessages = atoi(val);
      else if (!strcmp(name, "--throughput"))
	throughput = atoi(val);
      else if (!strcmp(name, "--producer.config") ||
	       !strcmp(name, "--consumer.config"))
	read_conf_file(val);
      else if (!strcmp(name, "--group-id"))
	conf->set("group.id", val, errstr);
      else if (!strcmp(name, "--session-timeout"))
	conf->set("session.timeout.ms", val, errstr);
      else if (!strcmp(name, "--reset-policy")) {
	if (tconf->set("auto.offset.reset", val, errstr)) {
	  std::cerr << now() << ": " << errstr << std::endl;
	  exit(1);
	}
      } else if (!strcmp(name, "--debug")) {
	conf->set("debug", val, errstr);
      } else {
	std::cerr << now() << ": Unknown option " << name << std::endl;
	exit(1);
      }

      i++;

    } else {
      if (!strcmp(name, "--consumer"))
	mode = "C";
      else if (!strcmp(name, "--producer"))
	mode = "P";
      else if (!strcmp(name, "--enable-autocommit")) {
	state.consumer.useAutoCommit = true;
	conf->set("enable.auto.commit", "true", errstr);
      } else {
	std::cerr << now() << ": Unknown option or missing argument to " << name << std::endl;
	exit(1);
      }
    }
  }

  if (topics.empty() || brokers.empty()) {
    std::cerr << now() << ": Missing --topic and --broker-list" << std::endl;
    exit(1);
  }


  /*
   * Set configuration properties
   */
  conf->set("metadata.broker.list", brokers, errstr);

  ExampleEventCb ex_event_cb;
  conf->set("event_cb", &ex_event_cb, errstr);

  if (do_conf_dump) {
    int pass;

    for (pass = 0 ; pass < 2 ; pass++) {
      std::list<std::string> *dump;
      if (pass == 0) {
        dump = conf->dump();
        std::cerr << now() << ": # Global config" << std::endl;
      } else {
        dump = tconf->dump();
        std::cerr << now() << ": # Topic config" << std::endl;
      }

      for (std::list<std::string>::iterator it = dump->begin();
           it != dump->end(); ) {
        std::cerr << *it << " = ";
        it++;
        std::cerr << *it << std::endl;
        it++;
      }
      std::cerr << std::endl;
    }
    exit(0);
  }

  signal(SIGINT, sigterm);
  signal(SIGTERM, sigterm);
  signal(SIGALRM,  sigwatchdog);


  if (mode == "P") {
    /*
     * Producer mode
     */

    ExampleDeliveryReportCb ex_dr_cb;

    /* Set delivery report callback */
    conf->set("dr_cb", &ex_dr_cb, errstr);

    /*
     * Create producer using accumulated global configuration.
     */
    RdKafka::Producer *producer = RdKafka::Producer::create(conf, errstr);
    if (!producer) {
      std::cerr << now() << ": Failed to create producer: " << errstr << std::endl;
      exit(1);
    }

    std::cerr << now() << ": % Created producer " << producer->name() << std::endl;

    /*
     * Create topic handle.
     */
    RdKafka::Topic *topic = RdKafka::Topic::create(producer, topics[0],
						   tconf, errstr);
    if (!topic) {
      std::cerr << now() << ": Failed to create topic: " << errstr << std::endl;
      exit(1);
    }

    static const int delay_us = throughput ? 1000000/throughput : 0;

    if (state.maxMessages == -1)
      state.maxMessages = 1000000; /* Avoid infinite produce */

    for (int i = 0 ; run && i < state.maxMessages ; i++) {
      /*
       * Produce message
       */
      std::ostringstream msg;
      msg << i;
      RdKafka::ErrorCode resp =
	producer->produce(topic, partition,
			  RdKafka::Producer::RK_MSG_COPY /* Copy payload */,
			  const_cast<char *>(msg.str().c_str()),
			  msg.str().size(), NULL, NULL);
      if (resp != RdKafka::ERR_NO_ERROR) {
	errorString("producer_send_error",
		    RdKafka::err2str(resp), topic->name(), NULL, msg.str());
	state.producer.numErr++;
      } else {
	std::cerr << now() << ": % Produced message (" <<
	  msg.str().size() << " bytes)" << std::endl;
	state.producer.numSent++;
      }

      producer->poll(delay_us / 1000);
      watchdog_kick();
    }
    run = true;

    while (run && producer->outq_len() > 0) {
      std::cerr << now() << ": Waiting for " << producer->outq_len() << std::endl;
      producer->poll(50);
      watchdog_kick();
    }

    std::cerr << now() << ": " << state.producer.numAcked << "/" <<
      state.producer.numSent << "/" << state.maxMessages <<
      " msgs acked/sent/max, " << state.producer.numErr <<
      " errored" << std::endl;

    delete topic;
    delete producer;


  } else if (mode == "C") {
    /*
     * Consumer mode
     */

    tconf->set("auto.offset.reset", "smallest", errstr);

    /* Set default topic config */
    conf->set("default_topic_conf", tconf, errstr);

    ExampleRebalanceCb ex_rebalance_cb;
    conf->set("rebalance_cb", &ex_rebalance_cb, errstr);

    ExampleOffsetCommitCb ex_offset_commit_cb;
    conf->set("offset_commit_cb", &ex_offset_commit_cb, errstr);


    /*
     * Create consumer using accumulated global configuration.
     */
    consumer = RdKafka::KafkaConsumer::create(conf, errstr);
    if (!consumer) {
      std::cerr << now() << ": Failed to create consumer: " <<
	errstr << std::endl;
      exit(1);
    }

    std::cerr << now() << ": % Created consumer " << consumer->name() <<
      std::endl;

    /*
     * Subscribe to topic(s)
     */
    RdKafka::ErrorCode resp = consumer->subscribe(topics);
    if (resp != RdKafka::ERR_NO_ERROR) {
      std::cerr << now() << ": Failed to subscribe to " << topics.size() << " topics: "
		<< RdKafka::err2str(resp) << std::endl;
      exit(1);
    }

    /*
     * Consume messages
     */
    while (run) {
      RdKafka::Message *msg = consumer->consume(500);
      msg_consume(consumer, msg, NULL);
      delete msg;
      watchdog_kick();
    }

    /* Final commit */
    do_commit(consumer, 1);

    /*
     * Stop consumer
     */
    consumer->close();

    delete consumer;
  }


  /*
   * Wait for RdKafka to decommission.
   * This is not strictly needed (when check outq_len() above), but
   * allows RdKafka to clean up all its resources before the application
   * exits so that memory profilers such as valgrind wont complain about
   * memory leaks.
   */
  RdKafka::wait_destroyed(5000);

  std::cerr << now() << ": EXITING WITH RETURN VALUE 0" << std::endl;
  return 0;
}
int main (int argc, char **argv) {
  std::string brokers = "localhost";
  std::string errstr;
  std::string topic_str;
  std::string mode;
  std::string debug;
  std::vector<std::string> topics;
  bool do_conf_dump = false;
  int opt;
  int use_ccb = 0;

  /*
   * Create configuration objects
   */
  RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
  RdKafka::Conf *tconf = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC);

  ExampleRebalanceCb ex_rebalance_cb;
  conf->set("rebalance_cb", &ex_rebalance_cb, errstr);

  while ((opt = getopt(argc, argv, "g:b:z:qd:eX:AM:f:qv")) != -1) {
    switch (opt) {
    case 'g':
      if (conf->set("group.id",  optarg, errstr) != RdKafka::Conf::CONF_OK) {
        std::cerr << errstr << std::endl;
        exit(1);
      }
      break;
    case 'b':
      brokers = optarg;
      break;
    case 'z':
      if (conf->set("compression.codec", optarg, errstr) !=
	  RdKafka::Conf::CONF_OK) {
	std::cerr << errstr << std::endl;
	exit(1);
      }
      break;
    case 'e':
      exit_eof = true;
      break;
    case 'd':
      debug = optarg;
      break;
    case 'M':
      if (conf->set("statistics.interval.ms", optarg, errstr) !=
          RdKafka::Conf::CONF_OK) {
        std::cerr << errstr << std::endl;
        exit(1);
      }
      break;
    case 'X':
      {
	char *name, *val;

	if (!strcmp(optarg, "dump")) {
	  do_conf_dump = true;
	  continue;
	}

	name = optarg;
	if (!(val = strchr(name, '='))) {
          std::cerr << "%% Expected -X property=value, not " <<
              name << std::endl;
	  exit(1);
	}

	*val = '\0';
	val++;

	/* Try "topic." prefixed properties on topic
	 * conf first, and then fall through to global if
	 * it didnt match a topic configuration property. */
        RdKafka::Conf::ConfResult res = RdKafka::Conf::CONF_UNKNOWN;
	if (!strncmp(name, "topic.", strlen("topic.")))
          res = tconf->set(name+strlen("topic."), val, errstr);
        if (res == RdKafka::Conf::CONF_UNKNOWN)
	  res = conf->set(name, val, errstr);

	if (res != RdKafka::Conf::CONF_OK) {
          std::cerr << errstr << std::endl;
	  exit(1);
	}
      }
      break;

      case 'f':
        if (!strcmp(optarg, "ccb"))
          use_ccb = 1;
        else {
          std::cerr << "Unknown option: " << optarg << std::endl;
          exit(1);
        }
        break;

      case 'q':
        verbosity--;
        break;

      case 'v':
        verbosity++;
        break;

    default:
      goto usage;
    }
  }

  for (; optind < argc ; optind++)
    topics.push_back(std::string(argv[optind]));

  if (topics.empty() || optind != argc) {
  usage:
    fprintf(stderr,
            "Usage: %s -g <group-id> [options] topic1 topic2..\n"
            "\n"
            "librdkafka version %s (0x%08x)\n"
            "\n"
            " Options:\n"
            "  -g <group-id>   Consumer group id\n"
            "  -b <brokers>    Broker address (localhost:9092)\n"
            "  -z <codec>      Enable compression:\n"
            "                  none|gzip|snappy\n"
            "  -e              Exit consumer when last message\n"
            "                  in partition has been received.\n"
            "  -d [facs..]     Enable debugging contexts:\n"
            "                  %s\n"
            "  -M <intervalms> Enable statistics\n"
            "  -X <prop=name>  Set arbitrary librdkafka "
            "configuration property\n"
            "                  Properties prefixed with \"topic.\" "
            "will be set on topic object.\n"
            "                  Use '-X list' to see the full list\n"
            "                  of supported properties.\n"
            "  -f <flag>       Set option:\n"
            "                     ccb - use consume_callback\n"
            "  -q              Quiet / Decrease verbosity\n"
            "  -v              Increase verbosity\n"
            "\n"
            "\n",
	    argv[0],
	    RdKafka::version_str().c_str(), RdKafka::version(),
	    RdKafka::get_debug_contexts().c_str());
	exit(1);
  }


  /*
   * Set configuration properties
   */
  conf->set("metadata.broker.list", brokers, errstr);

  if (!debug.empty()) {
    if (conf->set("debug", debug, errstr) != RdKafka::Conf::CONF_OK) {
      std::cerr << errstr << std::endl;
      exit(1);
    }
  }

  ExampleEventCb ex_event_cb;
  conf->set("event_cb", &ex_event_cb, errstr);

  conf->set("default_topic_conf", tconf, errstr);
  delete tconf;

  if (do_conf_dump) {
    int pass;

    for (pass = 0 ; pass < 2 ; pass++) {
      std::list<std::string> *dump;
      if (pass == 0) {
        dump = conf->dump();
        std::cout << "# Global config" << std::endl;
      } else {
        dump = tconf->dump();
        std::cout << "# Topic config" << std::endl;
      }

      for (std::list<std::string>::iterator it = dump->begin();
           it != dump->end(); ) {
        std::cout << *it << " = ";
        it++;
        std::cout << *it << std::endl;
        it++;
      }
      std::cout << std::endl;
    }
    exit(0);
  }

  signal(SIGINT, sigterm);
  signal(SIGTERM, sigterm);


  /*
   * Consumer mode
   */

  /*
   * Create consumer using accumulated global configuration.
   */
  RdKafka::KafkaConsumer *consumer = RdKafka::KafkaConsumer::create(conf, errstr);
  if (!consumer) {
    std::cerr << "Failed to create consumer: " << errstr << std::endl;
    exit(1);
  }

  delete conf;

  std::cout << "% Created consumer " << consumer->name() << std::endl;


  /*
   * Subscribe to topics
   */
  RdKafka::ErrorCode err = consumer->subscribe(topics);
  if (err) {
    std::cerr << "Failed to subscribe to " << topics.size() << " topics: "
              << RdKafka::err2str(err) << std::endl;
    exit(1);
  }


  /*
   * Consume messages
   */
  while (run) {
    if (use_ccb) {
      std::cerr << "Use callback: Not implemented" << std::endl;
      break;
    }
    RdKafka::Message *msg = consumer->consume(1000);
    msg_consume(msg, NULL);
    delete msg;
  }

  alarm(10);

  /*
   * Stop consumer
   */
  consumer->close();
  delete consumer;

  std::cerr << "% Consumed " << msg_cnt << " messages ("
            << msg_bytes << " bytes)" << std::endl;

  /*
   * Wait for RdKafka to decommission.
   * This is not strictly needed (with check outq_len() above), but
   * allows RdKafka to clean up all its resources before the application
   * exits so that memory profilers such as valgrind wont complain about
   * memory leaks.
   */
  RdKafka::wait_destroyed(5000);

  return 0;
}
Exemplo n.º 6
0
int main (int argc, char **argv) {
  std::string brokers = "localhost";
  std::string errstr;
  std::string topic_str;
  std::string mode;
  std::string debug;
  int32_t partition = RdKafka::Topic::PARTITION_UA;
  int64_t start_offset = RdKafka::Topic::OFFSET_BEGINNING;
  bool do_conf_dump = false;
  int opt;
  MyHashPartitionerCb hash_partitioner;
  int use_ccb = 0;

  /*
   * Create configuration objects
   */
  RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
  RdKafka::Conf *tconf = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC);


  while ((opt = getopt(argc, argv, "PCLt:p:b:z:qd:o:eX:AM:f:")) != -1) {
    switch (opt) {
    case 'P':
    case 'C':
    case 'L':
      mode = opt;
      break;
    case 't':
      topic_str = optarg;
      break;
    case 'p':
      if (!strcmp(optarg, "random"))
        /* default */;
      else if (!strcmp(optarg, "hash")) {
        if (tconf->set("partitioner_cb", &hash_partitioner, errstr) !=
            RdKafka::Conf::CONF_OK) {
          std::cerr << errstr << std::endl;
          exit(1);
        }
      } else
        partition = std::atoi(optarg);
      break;
    case 'b':
      brokers = optarg;
      break;
    case 'z':
      if (conf->set("compression.codec", optarg, errstr) !=
	  RdKafka::Conf::CONF_OK) {
	std::cerr << errstr << std::endl;
	exit(1);
      }
      break;
    case 'o':
      if (!strcmp(optarg, "end"))
	start_offset = RdKafka::Topic::OFFSET_END;
      else if (!strcmp(optarg, "beginning"))
	start_offset = RdKafka::Topic::OFFSET_BEGINNING;
      else if (!strcmp(optarg, "stored"))
	start_offset = RdKafka::Topic::OFFSET_STORED;
      else
	start_offset = strtoll(optarg, NULL, 10);
      break;
    case 'e':
      exit_eof = true;
      break;
    case 'd':
      debug = optarg;
      break;
    case 'M':
      if (conf->set("statistics.interval.ms", optarg, errstr) !=
          RdKafka::Conf::CONF_OK) {
        std::cerr << errstr << std::endl;
        exit(1);
      }
      break;
    case 'X':
      {
	char *name, *val;

	if (!strcmp(optarg, "dump")) {
	  do_conf_dump = true;
	  continue;
	}

	name = optarg;
	if (!(val = strchr(name, '='))) {
          std::cerr << "%% Expected -X property=value, not " <<
              name << std::endl;
	  exit(1);
	}

	*val = '\0';
	val++;

	/* Try "topic." prefixed properties on topic
	 * conf first, and then fall through to global if
	 * it didnt match a topic configuration property. */
        RdKafka::Conf::ConfResult res;
	if (!strncmp(name, "topic.", strlen("topic.")))
          res = tconf->set(name+strlen("topic."), val, errstr);
        else
	  res = conf->set(name, val, errstr);

	if (res != RdKafka::Conf::CONF_OK) {
          std::cerr << errstr << std::endl;
	  exit(1);
	}
      }
      break;

      case 'f':
        if (!strcmp(optarg, "ccb"))
          use_ccb = 1;
        else {
          std::cerr << "Unknown option: " << optarg << std::endl;
          exit(1);
        }
        break;

    default:
      goto usage;
    }
  }

  if (mode.empty() || (topic_str.empty() && mode != "L") || optind != argc) {
  usage:
	  std::string features;
	  conf->get("builtin.features", features);
    fprintf(stderr,
            "Usage: %s [-C|-P] -t <topic> "
            "[-p <partition>] [-b <host1:port1,host2:port2,..>]\n"
            "\n"
            "librdkafka version %s (0x%08x, builtin.features \"%s\")\n"
            "\n"
            " Options:\n"
            "  -C | -P         Consumer or Producer mode\n"
            "  -L              Metadata list mode\n"
            "  -t <topic>      Topic to fetch / produce\n"
            "  -p <num>        Partition (random partitioner)\n"
            "  -p <func>       Use partitioner:\n"
            "                  random (default), hash\n"
            "  -b <brokers>    Broker address (localhost:9092)\n"
            "  -z <codec>      Enable compression:\n"
            "                  none|gzip|snappy\n"
            "  -o <offset>     Start offset (consumer)\n"
            "  -e              Exit consumer when last message\n"
            "                  in partition has been received.\n"
            "  -d [facs..]     Enable debugging contexts:\n"
            "                  %s\n"
            "  -M <intervalms> Enable statistics\n"
            "  -X <prop=name>  Set arbitrary librdkafka "
            "configuration property\n"
            "                  Properties prefixed with \"topic.\" "
            "will be set on topic object.\n"
            "                  Use '-X list' to see the full list\n"
            "                  of supported properties.\n"
            "  -f <flag>       Set option:\n"
            "                     ccb - use consume_callback\n"
            "\n"
            " In Consumer mode:\n"
            "  writes fetched messages to stdout\n"
            " In Producer mode:\n"
            "  reads messages from stdin and sends to broker\n"
            "\n"
            "\n"
            "\n",
	    argv[0],
	    RdKafka::version_str().c_str(), RdKafka::version(),
		features.c_str(),
	    RdKafka::get_debug_contexts().c_str());
	exit(1);
  }


  /*
   * Set configuration properties
   */
  conf->set("metadata.broker.list", brokers, errstr);

  if (!debug.empty()) {
    if (conf->set("debug", debug, errstr) != RdKafka::Conf::CONF_OK) {
      std::cerr << errstr << std::endl;
      exit(1);
    }
  }

  ExampleEventCb ex_event_cb;
  conf->set("event_cb", &ex_event_cb, errstr);

  if (do_conf_dump) {
    int pass;

    for (pass = 0 ; pass < 2 ; pass++) {
      std::list<std::string> *dump;
      if (pass == 0) {
        dump = conf->dump();
        std::cout << "# Global config" << std::endl;
      } else {
        dump = tconf->dump();
        std::cout << "# Topic config" << std::endl;
      }

      for (std::list<std::string>::iterator it = dump->begin();
           it != dump->end(); ) {
        std::cout << *it << " = ";
        it++;
        std::cout << *it << std::endl;
        it++;
      }
      std::cout << std::endl;
    }
    exit(0);
  }

  signal(SIGINT, sigterm);
  signal(SIGTERM, sigterm);


  if (mode == "P") {
    /*
     * Producer mode
     */

    if(topic_str.empty())
      goto usage;

    ExampleDeliveryReportCb ex_dr_cb;

    /* Set delivery report callback */
    conf->set("dr_cb", &ex_dr_cb, errstr);

    /*
     * Create producer using accumulated global configuration.
     */
    RdKafka::Producer *producer = RdKafka::Producer::create(conf, errstr);
    if (!producer) {
      std::cerr << "Failed to create producer: " << errstr << std::endl;
      exit(1);
    }

    std::cout << "% Created producer " << producer->name() << std::endl;

    /*
     * Create topic handle.
     */
    RdKafka::Topic *topic = RdKafka::Topic::create(producer, topic_str,
						   tconf, errstr);
    if (!topic) {
      std::cerr << "Failed to create topic: " << errstr << std::endl;
      exit(1);
    }

    /*
     * Read messages from stdin and produce to broker.
     */
    for (std::string line; run && std::getline(std::cin, line);) {
      if (line.empty()) {
        producer->poll(0);
	continue;
      }

      /*
       * Produce message
       */
      RdKafka::ErrorCode resp =
	producer->produce(topic, partition,
			  RdKafka::Producer::RK_MSG_COPY /* Copy payload */,
			  const_cast<char *>(line.c_str()), line.size(),
			  NULL, NULL);
      if (resp != RdKafka::ERR_NO_ERROR)
	std::cerr << "% Produce failed: " <<
	  RdKafka::err2str(resp) << std::endl;
      else
	std::cerr << "% Produced message (" << line.size() << " bytes)" <<
	  std::endl;

      producer->poll(0);
    }
    run = true;

    while (run && producer->outq_len() > 0) {
      std::cerr << "Waiting for " << producer->outq_len() << std::endl;
      producer->poll(1000);
    }

    delete topic;
    delete producer;


  } else if (mode == "C") {
    /*
     * Consumer mode
     */

    if(topic_str.empty())
      goto usage;

    /*
     * Create consumer using accumulated global configuration.
     */
    RdKafka::Consumer *consumer = RdKafka::Consumer::create(conf, errstr);
    if (!consumer) {
      std::cerr << "Failed to create consumer: " << errstr << std::endl;
      exit(1);
    }

    std::cout << "% Created consumer " << consumer->name() << std::endl;

    /*
     * Create topic handle.
     */
    RdKafka::Topic *topic = RdKafka::Topic::create(consumer, topic_str,
						   tconf, errstr);
    if (!topic) {
      std::cerr << "Failed to create topic: " << errstr << std::endl;
      exit(1);
    }

    /*
     * Start consumer for topic+partition at start offset
     */
    RdKafka::ErrorCode resp = consumer->start(topic, partition, start_offset);
    if (resp != RdKafka::ERR_NO_ERROR) {
      std::cerr << "Failed to start consumer: " <<
	RdKafka::err2str(resp) << std::endl;
      exit(1);
    }

    ExampleConsumeCb ex_consume_cb;

    /*
     * Consume messages
     */
    while (run) {
      if (use_ccb) {
        consumer->consume_callback(topic, partition, 1000,
                                   &ex_consume_cb, &use_ccb);
      } else {
        RdKafka::Message *msg = consumer->consume(topic, partition, 1000);
        msg_consume(msg, NULL);
        delete msg;
      }
      consumer->poll(0);
    }

    /*
     * Stop consumer
     */
    consumer->stop(topic, partition);

    consumer->poll(1000);

    delete topic;
    delete consumer;
  } else {
    /* Metadata mode */

    /*
     * Create producer using accumulated global configuration.
     */
    RdKafka::Producer *producer = RdKafka::Producer::create(conf, errstr);
    if (!producer) {
      std::cerr << "Failed to create producer: " << errstr << std::endl;
      exit(1);
    }

    std::cout << "% Created producer " << producer->name() << std::endl;

    /*
     * Create topic handle.
     */
    RdKafka::Topic *topic = NULL;
    if(!topic_str.empty()) {
      topic = RdKafka::Topic::create(producer, topic_str, tconf, errstr);
      if (!topic) {
        std::cerr << "Failed to create topic: " << errstr << std::endl;
        exit(1);
      }
    }

    while (run) {
      class RdKafka::Metadata *metadata;

      /* Fetch metadata */
      RdKafka::ErrorCode err = producer->metadata(topic!=NULL, topic,
                              &metadata, 5000);
      if (err != RdKafka::ERR_NO_ERROR) {
        std::cerr << "%% Failed to acquire metadata: " 
                  << RdKafka::err2str(err) << std::endl;
              run = 0;
              break;
      }

      metadata_print(topic_str, metadata);

      delete metadata;
      run = 0;
    }

  }


  /*
   * Wait for RdKafka to decommission.
   * This is not strictly needed (when check outq_len() above), but
   * allows RdKafka to clean up all its resources before the application
   * exits so that memory profilers such as valgrind wont complain about
   * memory leaks.
   */
  RdKafka::wait_destroyed(5000);

  return 0;
}
Exemplo n.º 7
0
    void Consumer::ensureSetup()
    {
        if (!topicPtr.load(std::memory_order_acquire))
        {
            CriticalBlock block(lock);

            if (!topicPtr.load(std::memory_order_relaxed))
            {
                initFileOffsetIfNotExist();

                std::string errStr;
                RdKafka::Conf* globalConfig = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);

                if (globalConfig)
                {
                    // Set global configuration parameters, used mainly at the consumer level
                    globalConfig->set("metadata.broker.list", brokers, errStr);
                    globalConfig->set("compression.codec", "snappy", errStr);
                    globalConfig->set("queued.max.messages.kbytes", "10000000", errStr);
                    globalConfig->set("fetch.message.max.bytes", "10000000", errStr);

                    // Set any global configurations from file, allowing
                    // overrides of above settings
                    applyConfig("kafka_global.conf", globalConfig, traceLevel);

                    // Set consumer callbacks
                    globalConfig->set("event_cb", static_cast<RdKafka::EventCb*>(this), errStr);

                    // Create the consumer
                    consumerPtr = RdKafka::Consumer::create(globalConfig, errStr);

                    if (consumerPtr)
                    {
                        RdKafka::Conf* topicConfPtr = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC);

                        // Set the per-topic configuration parameters
                        topicConfPtr->set("group.id", consumerGroup, errStr);
                        topicConfPtr->set("auto.offset.reset", "smallest", errStr);

                        // Set any topic configurations from file, allowing
                        // overrides of above settings
                        std::string confName = "kafka_consumer_topic_" + topic + ".conf";
                        applyConfig(confName.c_str(), topicConfPtr, traceLevel);

                        // Ensure that some items are set a certain way
                        // by setting them after loading the external conf
                        topicConfPtr->set("auto.commit.enable", "false", errStr);

                        // Create the topic
                        topicPtr.store(RdKafka::Topic::create(consumerPtr, topic, topicConfPtr, errStr), std::memory_order_release);

                        if (!topicPtr)
                        {
                            throw MakeStringException(-1, "Kafka: Unable to create consumer topic object for topic '%s'; error: '%s'", topic.c_str(), errStr.c_str());
                        }
                    }
                    else
                    {
                        throw MakeStringException(-1, "Kafka: Unable to create consumer object for brokers '%s'; error: '%s'", brokers.c_str(), errStr.c_str());
                    }
                }
                else
                {
                    throw MakeStringException(-1, "Kafka: Unable to create consumer global configuration object for brokers '%s'; error: '%s'", brokers.c_str(), errStr.c_str());
                }
            }
        }
    }
Exemplo n.º 8
0
    void Publisher::ensureSetup()
    {
        if (!topicPtr.load(std::memory_order_acquire))
        {
            CriticalBlock   block(lock);

            if (!topicPtr.load(std::memory_order_relaxed))
            {
                std::string errStr;
                RdKafka::Conf* globalConfig = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);

                if (globalConfig)
                {
                    // Set global configuration parameters, used mainly at the producer level
                    globalConfig->set("metadata.broker.list", brokers, errStr);
                    globalConfig->set("queue.buffering.max.messages", "1000000", errStr);
                    globalConfig->set("compression.codec", "snappy", errStr);
                    globalConfig->set("message.send.max.retries", "3", errStr);
                    globalConfig->set("retry.backoff.ms", "500", errStr);

                    // Set any global configurations from file, allowing
                    // overrides of above settings
                    applyConfig("kafka_global.conf", globalConfig, traceLevel);

                    // Set producer callbacks
                    globalConfig->set("event_cb", static_cast<RdKafka::EventCb*>(this), errStr);
                    globalConfig->set("dr_cb", static_cast<RdKafka::DeliveryReportCb*>(this), errStr);

                    // Create the producer
                    producerPtr = RdKafka::Producer::create(globalConfig, errStr);

                    if (producerPtr)
                    {
                        RdKafka::Conf* topicConfPtr = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC);

                        // Set any topic configurations from file
                        std::string confName = "kafka_publisher_topic_" + topic + ".conf";
                        applyConfig(confName.c_str(), topicConfPtr, traceLevel);

                        // Create the topic
                        topicPtr.store(RdKafka::Topic::create(producerPtr, topic, topicConfPtr, errStr), std::memory_order_release);

                        if (topicPtr)
                        {
                            // Start the attached background poller
                            pollerPtr->start();
                        }
                        else
                        {
                            throw MakeStringException(-1, "Kafka: Unable to create producer topic object for topic '%s'; error: '%s'", topic.c_str(), errStr.c_str());
                        }
                    }
                    else
                    {
                        throw MakeStringException(-1, "Kafka: Unable to create producer object for brokers '%s'; error: '%s'", brokers.c_str(), errStr.c_str());
                    }
                }
                else
                {
                    throw MakeStringException(-1, "Kafka: Unable to create producer global configuration object for brokers '%s'; error: '%s'", brokers.c_str(), errStr.c_str());
                }
            }
        }
    }
Exemplo n.º 9
0
/**
 * Generate unique topic name (there is a C function for that in test.h wihch you should use)
 * Query metadata for that topic
 * Wait one second
 * Query again, it should now have isrs and everything
 */
static void test_metadata_cpp (void) {
	RdKafka::Conf *conf  = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL); /* @TODO: Do we need to merge with C test_conf_init()? */
	RdKafka::Conf *tconf = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC); /* @TODO: Same of prev */

	RdKafka::Metadata *metadata;
	RdKafka::ErrorCode err;
	int msgcnt = 10000;
	int partition_cnt = 2;
	int i;
	uint64_t testid;
	int msg_base = 0;
	std::string errstr;
	const char *topic_str = test_mk_topic_name("0013", 1);
/*	if(!topic){
		TEST_FAIL()
	}*/

	//const RdKafka::Conf::ConfResult confResult = conf->set("debug","all",errstr);
	//if(confResult != RdKafka::Conf::CONF_OK){
	//	std::stringstream errstring;
	//	errstring << "Can't set config" << errstr;
	//	TEST_FAIL(errstring.str().c_str());
	//}

	TEST_SAY("Topic %s.\n", topic_str);
	
	const RdKafka::Conf::ConfResult confBrokerResult = conf->set("metadata.broker.list", "localhost:9092", errstr);
	if(confBrokerResult != RdKafka::Conf::CONF_OK){
		std::stringstream errstring;
		errstring << "Can't set broker" << errstr;
		TEST_FAIL(errstring.str().c_str());
	}

	/* Create a producer to fetch metadata */
	RdKafka::Producer *producer = RdKafka::Producer::create(conf, errstr);
	if (!producer) {
		std::stringstream errstring;
		errstring << "Can't create producer" << errstr;
		TEST_FAIL(errstring.str().c_str());
	}

	/*
	* Create topic handle.
	*/
	RdKafka::Topic *topic = NULL;
	topic = RdKafka::Topic::create(producer, topic_str, tconf, errstr);
	if (!topic) {
		std::stringstream errstring;
		errstring << "Can't create topic" << errstr;
		exit(1);
	}

	/* First request of metadata: It have to fail */
	err = producer->metadata(topic!=NULL, topic,
	                      &metadata, 5000);
	if (err != RdKafka::ERR_NO_ERROR) {
		std::stringstream errstring;
		errstring << "Can't request first metadata: " << errstr;
		TEST_FAIL(errstring.str().c_str());
	}

	/* It's a new topic, it should have no partitions */
	if(metadata->topics()->at(0)->partitions()->size() != 0){
		TEST_FAIL("ISRS != 0");
	}

	sleep(1);

	/* Second request of metadata: It have to success */
	err = producer->metadata(topic!=NULL, topic,
	                      &metadata, 5000);
	
	/* It should have now partitions */
	if(metadata->topics()->at(0)->partitions()->size() == 0){
		TEST_FAIL("ISRS == 0");
	}


	delete topic;
	delete producer;
	delete tconf;
	delete conf;

	/* Wait for everything to be cleaned up since broker destroys are
	 * handled in its own thread. */
	test_wait_exit(10);

	/* If we havent failed at this point then
	 * there were no threads leaked */
	return;
}
Exemplo n.º 10
0
static void do_test_consumer_lag (void) {
  const int msgcnt = 10;
  std::string errstr;
  RdKafka::ErrorCode err;

  topic = Test::mk_topic_name("0061-consumer_lag", 1);

  test_produce_msgs_easy(topic.c_str(), 0, 0, msgcnt);

  /*
   * Create consumer
   */

  /* Create consumer */
  RdKafka::Conf *conf;
  Test::conf_init(&conf, NULL, 10);
  StatsCb stats;
  if (conf->set("event_cb", &stats, errstr) != RdKafka::Conf::CONF_OK)
    Test::Fail("set event_cb failed: " + errstr);
  Test::conf_set(conf, "group.id", topic);
  Test::conf_set(conf, "enable.auto.commit", "false");
  Test::conf_set(conf, "enable.partition.eof", "false");
  Test::conf_set(conf, "auto.offset.reset", "earliest");
  Test::conf_set(conf, "statistics.interval.ms", "100");

  RdKafka::KafkaConsumer *c = RdKafka::KafkaConsumer::create(conf, errstr);
  if (!c)
    Test::Fail("Failed to create KafkaConsumer: " + errstr);
  delete conf;

  /* Assign partitions */
  /* Subscribe */
  std::vector<RdKafka::TopicPartition*> parts;
  parts.push_back(RdKafka::TopicPartition::create(topic, 0));
  if ((err = c->assign(parts)))
    Test::Fail("assign failed: " + RdKafka::err2str(err));
  RdKafka::TopicPartition::destroy(parts);

  /* Start consuming */
  Test::Say("Consuming topic " + topic + "\n");
  int cnt = 0;
  while (cnt < msgcnt) {
    RdKafka::Message *msg = c->consume(tmout_multip(1000));
    switch (msg->err())
      {
      case RdKafka::ERR__TIMED_OUT:
        break;
      case RdKafka::ERR__PARTITION_EOF:
        Test::Fail(tostr() << "Consume error after " << cnt << "/" << msgcnt << " messages: " << msg->errstr());
        break;

      case RdKafka::ERR_NO_ERROR:
        /* Proper message. Update calculated lag for later
         * checking in stats callback */
        stats.calc_lag = msgcnt - (msg->offset()+1);
        cnt++;
        Test::Say(2, tostr() << "Received message #" << cnt << "/" << msgcnt <<
                  " at offset " << msg->offset() << " (calc lag " << stats.calc_lag << ")\n");
        /* Slow down message "processing" to make sure we get
         * at least one stats callback per message. */
        if (cnt < msgcnt)
          rd_sleep(1);
        break;

      default:
        Test::Fail("Consume error: " + msg->errstr());
        break;
      }

    delete msg;
  }
  Test::Say(tostr() << "Done, lag was valid " <<
            stats.lag_valid << " times\n");
  if (stats.lag_valid == 0)
    Test::Fail("No valid consumer_lag in statistics seen");

  c->close();
  delete c;
}
Exemplo n.º 11
0
void kafka_writer::set_brokers(std::string brokers) {
    std::string errstr;
    conf->set("metadata.broker.list", brokers, errstr);
}