Beispiel #1
0
static void do_test_empty_topic_consumer () {
  std::string errstr;
  std::string topic = Test::mk_topic_name("0067_empty_topic", 1);
  const int32_t partition = 0;

  RdKafka::Conf *conf;

  Test::conf_init(&conf, NULL, 0);

  /* Create simple consumer */
  RdKafka::Consumer *consumer = RdKafka::Consumer::create(conf, errstr);
  if (!consumer)
          Test::Fail("Failed to create Consumer: " + errstr);

  RdKafka::Topic *rkt = RdKafka::Topic::create(consumer, topic, NULL, errstr);
  if (!rkt)
          Test::Fail("Simple Topic failed: " + errstr);


  /* Create the topic through a metadata request. */
  Test::Say("Creating empty topic " + topic + "\n");
  RdKafka::Metadata *md;
  RdKafka::ErrorCode err = consumer->metadata(false, rkt, &md,
                                              tmout_multip(10*1000));
  if (err)
          Test::Fail("Failed to create topic " + topic + ": " + RdKafka::err2str(err));
  delete md;

  /* Start consumer */
  err = consumer->start(rkt, partition, RdKafka::Topic::OFFSET_BEGINNING);
  if (err)
          Test::Fail("Consume start() failed: " + RdKafka::err2str(err));

  /* Consume using legacy consumer, should give an EOF and nothing else. */
  Test::Say("Simple Consumer: consuming\n");
  RdKafka::Message *msg = consumer->consume(rkt, partition,
                                            tmout_multip(10 * 1000));
  if (msg->err() != RdKafka::ERR__PARTITION_EOF)
          Test::Fail("Simple consume() expected EOF, got " + RdKafka::err2str(msg->err()));
  delete msg;

  /* Nothing else should come now, just a consume() timeout */
  msg = consumer->consume(rkt, partition, 1 * 1000);
  if (msg->err() != RdKafka::ERR__TIMED_OUT)
          Test::Fail("Simple consume() expected timeout, got " + RdKafka::err2str(msg->err()));
  delete msg;

  consumer->stop(rkt, partition);

  delete rkt;
  delete consumer;


  /*
   * Now do the same thing using the high-level KafkaConsumer.
   */

  Test::conf_set(conf, "group.id", topic);

  RdKafka::KafkaConsumer *kconsumer = RdKafka::KafkaConsumer::create(conf, errstr);
  if (!kconsumer)
          Test::Fail("Failed to create KafkaConsumer: " + errstr);

  std::vector<RdKafka::TopicPartition*> part;
  part.push_back(RdKafka::TopicPartition::create(topic, partition));

  err = kconsumer->assign(part);
  if (err)
          Test::Fail("assign() failed: " + RdKafka::err2str(err));

  RdKafka::TopicPartition::destroy(part);

  Test::Say("KafkaConsumer: consuming\n");
  msg = kconsumer->consume(tmout_multip(5 * 1000));
  if (msg->err() != RdKafka::ERR__PARTITION_EOF)
          Test::Fail("KafkaConsumer consume() expected EOF, got " + RdKafka::err2str(msg->err()));
  delete msg;

  /* Nothing else should come now, just a consume() timeout */
  msg = kconsumer->consume(1 * 1000);
  if (msg->err() != RdKafka::ERR__TIMED_OUT)
          Test::Fail("KafkaConsumer consume() expected timeout, got " + RdKafka::err2str(msg->err()));
  delete msg;

  kconsumer->close();

  delete kconsumer;
  delete conf;
}
Beispiel #2
0
static void do_test_null_empty (bool api_version_request) {
  std::string topic = Test::mk_topic_name("0070_null_empty", 1);
  const int partition = 0;

  Test::Say(tostr() << "Testing with api.version.request=" << api_version_request << " on topic " << topic << " partition " << partition << "\n");

  RdKafka::Conf *conf;
  Test::conf_init(&conf, NULL, 0);
  Test::conf_set(conf, "api.version.request",
                 api_version_request ? "true" : "false");
  Test::conf_set(conf, "acks", "all");


  std::string errstr;
  RdKafka::Producer *p = RdKafka::Producer::create(conf, errstr);
  if (!p)
    Test::Fail("Failed to create Producer: " + errstr);
  delete conf;

  const int msgcnt = 8;
  static const char *msgs[msgcnt*2] = {
    NULL, NULL,
    "key2", NULL,
    "key3", "val3",
    NULL, "val4",
    "", NULL,
    NULL, "",
    "", ""
  };

  RdKafka::ErrorCode err;

  for (int i = 0 ; i < msgcnt * 2 ; i += 2) {
    Test::Say(3, tostr() << "Produce message #" << (i/2) <<
              ": key=\"" << (msgs[i] ? msgs[i] : "Null") <<
              "\", value=\"" << (msgs[i+1] ? msgs[i+1] : "Null") << "\"\n");
    err = p->produce(topic, partition, RdKafka::Producer::RK_MSG_COPY,
                     /* Value */
                     (void *)msgs[i+1], msgs[i+1] ? strlen(msgs[i+1]) : 0,
                     /* Key */
                     (void *)msgs[i], msgs[i] ? strlen(msgs[i]) : 0,
                     0, NULL);
    if (err != RdKafka::ERR_NO_ERROR)
      Test::Fail("Produce failed: " + RdKafka::err2str(err));
  }

  if (p->flush(tmout_multip(3*5000)) != 0)
    Test::Fail("Not all messages flushed");

  Test::Say(tostr() << "Produced " << msgcnt << " messages to " << topic << "\n");

  delete p;

  /*
   * Now consume messages from the beginning, making sure they match
   * what was produced.
   */

  /* Create consumer */
  Test::conf_init(&conf, NULL, 10);
  Test::conf_set(conf, "group.id", topic);
  Test::conf_set(conf, "api.version.request",
                 api_version_request ? "true" : "false");
  Test::conf_set(conf, "enable.auto.commit", "false");

  RdKafka::KafkaConsumer *c = RdKafka::KafkaConsumer::create(conf, errstr);
  if (!c)
    Test::Fail("Failed to create KafkaConsumer: " + errstr);
  delete conf;

  /* Assign the partition */
  std::vector<RdKafka::TopicPartition*> parts;
  parts.push_back(RdKafka::TopicPartition::create(topic, partition,
                                                 RdKafka::Topic::OFFSET_BEGINNING));
  err = c->assign(parts);
  if (err != RdKafka::ERR_NO_ERROR)
    Test::Fail("assign() failed: " + RdKafka::err2str(err));
  RdKafka::TopicPartition::destroy(parts);

  /* Start consuming */
  int failures = 0;
  for (int i = 0 ; i < msgcnt * 2 ; i += 2) {
    RdKafka::Message *msg = c->consume(tmout_multip(5000));
    if (msg->err())
      Test::Fail(tostr() << "consume() failed at message " << (i/2) << ": " <<
                 msg->errstr());

    /* verify key */
    failures += check_equal(msgs[i], msg->key() ? msg->key()->c_str() : NULL, msg->key_len(),
                            tostr() << "message #" << (i/2) << " (offset " << msg->offset() << ") key");
    /* verify key_pointer() API as too */
    failures += check_equal(msgs[i], (const char *)msg->key_pointer(), msg->key_len(),
                tostr() << "message #" << (i/2) << " (offset " << msg->offset() << ") key");

    /* verify value */
    failures += check_equal(msgs[i+1], (const char *)msg->payload(), msg->len(),
                tostr() << "message #" << (i/2) << " (offset " << msg->offset() << ") value");
    delete msg;
  }

  Test::Say(tostr() << "Done consuming, closing. " << failures << " test failures\n");
  if (failures)
    Test::Fail(tostr() << "See " << failures << "  previous test failure(s)");

  c->close();
  delete c;
}
static void do_test_consumer_lag (void) {
  const int msgcnt = 10;
  std::string errstr;
  RdKafka::ErrorCode err;

  topic = Test::mk_topic_name("0061-consumer_lag", 1);

  test_produce_msgs_easy(topic.c_str(), 0, 0, msgcnt);

  /*
   * Create consumer
   */

  /* Create consumer */
  RdKafka::Conf *conf;
  Test::conf_init(&conf, NULL, 10);
  StatsCb stats;
  if (conf->set("event_cb", &stats, errstr) != RdKafka::Conf::CONF_OK)
    Test::Fail("set event_cb failed: " + errstr);
  Test::conf_set(conf, "group.id", topic);
  Test::conf_set(conf, "enable.auto.commit", "false");
  Test::conf_set(conf, "enable.partition.eof", "false");
  Test::conf_set(conf, "auto.offset.reset", "earliest");
  Test::conf_set(conf, "statistics.interval.ms", "100");

  RdKafka::KafkaConsumer *c = RdKafka::KafkaConsumer::create(conf, errstr);
  if (!c)
    Test::Fail("Failed to create KafkaConsumer: " + errstr);
  delete conf;

  /* Assign partitions */
  /* Subscribe */
  std::vector<RdKafka::TopicPartition*> parts;
  parts.push_back(RdKafka::TopicPartition::create(topic, 0));
  if ((err = c->assign(parts)))
    Test::Fail("assign failed: " + RdKafka::err2str(err));
  RdKafka::TopicPartition::destroy(parts);

  /* Start consuming */
  Test::Say("Consuming topic " + topic + "\n");
  int cnt = 0;
  while (cnt < msgcnt) {
    RdKafka::Message *msg = c->consume(tmout_multip(1000));
    switch (msg->err())
      {
      case RdKafka::ERR__TIMED_OUT:
        break;
      case RdKafka::ERR__PARTITION_EOF:
        Test::Fail(tostr() << "Consume error after " << cnt << "/" << msgcnt << " messages: " << msg->errstr());
        break;

      case RdKafka::ERR_NO_ERROR:
        /* Proper message. Update calculated lag for later
         * checking in stats callback */
        stats.calc_lag = msgcnt - (msg->offset()+1);
        cnt++;
        Test::Say(2, tostr() << "Received message #" << cnt << "/" << msgcnt <<
                  " at offset " << msg->offset() << " (calc lag " << stats.calc_lag << ")\n");
        /* Slow down message "processing" to make sure we get
         * at least one stats callback per message. */
        if (cnt < msgcnt)
          rd_sleep(1);
        break;

      default:
        Test::Fail("Consume error: " + msg->errstr());
        break;
      }

    delete msg;
  }
  Test::Say(tostr() << "Done, lag was valid " <<
            stats.lag_valid << " times\n");
  if (stats.lag_valid == 0)
    Test::Fail("No valid consumer_lag in statistics seen");

  c->close();
  delete c;
}