static void do_test_empty_topic_consumer () {
std::string errstr;
std::string topic = Test::mk_topic_name("0067_empty_topic", 1);
const int32_t partition = 0;
RdKafka::Conf *conf;
Test::conf_init(&conf, NULL, 0);
/* Create simple consumer */
RdKafka::Consumer *consumer = RdKafka::Consumer::create(conf, errstr);
if (!consumer)
Test::Fail("Failed to create Consumer: " + errstr);
RdKafka::Topic *rkt = RdKafka::Topic::create(consumer, topic, NULL, errstr);
if (!rkt)
Test::Fail("Simple Topic failed: " + errstr);
/* Create the topic through a metadata request. */
Test::Say("Creating empty topic " + topic + "\n");
RdKafka::Metadata *md;
RdKafka::ErrorCode err = consumer->metadata(false, rkt, &md,
tmout_multip(10*1000));
if (err)
Test::Fail("Failed to create topic " + topic + ": " + RdKafka::err2str(err));
delete md;
/* Start consumer */
err = consumer->start(rkt, partition, RdKafka::Topic::OFFSET_BEGINNING);
if (err)
Test::Fail("Consume start() failed: " + RdKafka::err2str(err));
/* Consume using legacy consumer, should give an EOF and nothing else. */
Test::Say("Simple Consumer: consuming\n");
RdKafka::Message *msg = consumer->consume(rkt, partition,
tmout_multip(10 * 1000));
if (msg->err() != RdKafka::ERR__PARTITION_EOF)
Test::Fail("Simple consume() expected EOF, got " + RdKafka::err2str(msg->err()));
delete msg;
/* Nothing else should come now, just a consume() timeout */
msg = consumer->consume(rkt, partition, 1 * 1000);
if (msg->err() != RdKafka::ERR__TIMED_OUT)
Test::Fail("Simple consume() expected timeout, got " + RdKafka::err2str(msg->err()));
delete msg;
consumer->stop(rkt, partition);
delete rkt;
delete consumer;
/*
* Now do the same thing using the high-level KafkaConsumer.
*/
Test::conf_set(conf, "group.id", topic);
RdKafka::KafkaConsumer *kconsumer = RdKafka::KafkaConsumer::create(conf, errstr);
if (!kconsumer)
Test::Fail("Failed to create KafkaConsumer: " + errstr);
std::vector<RdKafka::TopicPartition*> part;
part.push_back(RdKafka::TopicPartition::create(topic, partition));
err = kconsumer->assign(part);
if (err)
Test::Fail("assign() failed: " + RdKafka::err2str(err));
RdKafka::TopicPartition::destroy(part);
Test::Say("KafkaConsumer: consuming\n");
msg = kconsumer->consume(tmout_multip(5 * 1000));
if (msg->err() != RdKafka::ERR__PARTITION_EOF)
Test::Fail("KafkaConsumer consume() expected EOF, got " + RdKafka::err2str(msg->err()));
delete msg;
/* Nothing else should come now, just a consume() timeout */
msg = kconsumer->consume(1 * 1000);
if (msg->err() != RdKafka::ERR__TIMED_OUT)
Test::Fail("KafkaConsumer consume() expected timeout, got " + RdKafka::err2str(msg->err()));
delete msg;
kconsumer->close();
delete kconsumer;
delete conf;
}
static void do_test_null_empty (bool api_version_request) {
std::string topic = Test::mk_topic_name("0070_null_empty", 1);
const int partition = 0;
Test::Say(tostr() << "Testing with api.version.request=" << api_version_request << " on topic " << topic << " partition " << partition << "\n");
RdKafka::Conf *conf;
Test::conf_init(&conf, NULL, 0);
Test::conf_set(conf, "api.version.request",
api_version_request ? "true" : "false");
Test::conf_set(conf, "acks", "all");
std::string errstr;
RdKafka::Producer *p = RdKafka::Producer::create(conf, errstr);
if (!p)
Test::Fail("Failed to create Producer: " + errstr);
delete conf;
const int msgcnt = 8;
static const char *msgs[msgcnt*2] = {
NULL, NULL,
"key2", NULL,
"key3", "val3",
NULL, "val4",
"", NULL,
NULL, "",
"", ""
};
RdKafka::ErrorCode err;
for (int i = 0 ; i < msgcnt * 2 ; i += 2) {
Test::Say(3, tostr() << "Produce message #" << (i/2) <<
": key=\"" << (msgs[i] ? msgs[i] : "Null") <<
"\", value=\"" << (msgs[i+1] ? msgs[i+1] : "Null") << "\"\n");
err = p->produce(topic, partition, RdKafka::Producer::RK_MSG_COPY,
/* Value */
(void *)msgs[i+1], msgs[i+1] ? strlen(msgs[i+1]) : 0,
/* Key */
(void *)msgs[i], msgs[i] ? strlen(msgs[i]) : 0,
0, NULL);
if (err != RdKafka::ERR_NO_ERROR)
Test::Fail("Produce failed: " + RdKafka::err2str(err));
}
if (p->flush(tmout_multip(3*5000)) != 0)
Test::Fail("Not all messages flushed");
Test::Say(tostr() << "Produced " << msgcnt << " messages to " << topic << "\n");
delete p;
/*
* Now consume messages from the beginning, making sure they match
* what was produced.
*/
/* Create consumer */
Test::conf_init(&conf, NULL, 10);
Test::conf_set(conf, "group.id", topic);
Test::conf_set(conf, "api.version.request",
api_version_request ? "true" : "false");
Test::conf_set(conf, "enable.auto.commit", "false");
RdKafka::KafkaConsumer *c = RdKafka::KafkaConsumer::create(conf, errstr);
if (!c)
Test::Fail("Failed to create KafkaConsumer: " + errstr);
delete conf;
/* Assign the partition */
std::vector<RdKafka::TopicPartition*> parts;
parts.push_back(RdKafka::TopicPartition::create(topic, partition,
RdKafka::Topic::OFFSET_BEGINNING));
err = c->assign(parts);
if (err != RdKafka::ERR_NO_ERROR)
Test::Fail("assign() failed: " + RdKafka::err2str(err));
RdKafka::TopicPartition::destroy(parts);
/* Start consuming */
int failures = 0;
for (int i = 0 ; i < msgcnt * 2 ; i += 2) {
RdKafka::Message *msg = c->consume(tmout_multip(5000));
if (msg->err())
Test::Fail(tostr() << "consume() failed at message " << (i/2) << ": " <<
msg->errstr());
/* verify key */
failures += check_equal(msgs[i], msg->key() ? msg->key()->c_str() : NULL, msg->key_len(),
tostr() << "message #" << (i/2) << " (offset " << msg->offset() << ") key");
/* verify key_pointer() API as too */
failures += check_equal(msgs[i], (const char *)msg->key_pointer(), msg->key_len(),
tostr() << "message #" << (i/2) << " (offset " << msg->offset() << ") key");
/* verify value */
failures += check_equal(msgs[i+1], (const char *)msg->payload(), msg->len(),
tostr() << "message #" << (i/2) << " (offset " << msg->offset() << ") value");
delete msg;
}
Test::Say(tostr() << "Done consuming, closing. " << failures << " test failures\n");
if (failures)
Test::Fail(tostr() << "See " << failures << " previous test failure(s)");
c->close();
delete c;
}
static void do_test_consumer_lag (void) {
const int msgcnt = 10;
std::string errstr;
RdKafka::ErrorCode err;
topic = Test::mk_topic_name("0061-consumer_lag", 1);
test_produce_msgs_easy(topic.c_str(), 0, 0, msgcnt);
/*
* Create consumer
*/
/* Create consumer */
RdKafka::Conf *conf;
Test::conf_init(&conf, NULL, 10);
StatsCb stats;
if (conf->set("event_cb", &stats, errstr) != RdKafka::Conf::CONF_OK)
Test::Fail("set event_cb failed: " + errstr);
Test::conf_set(conf, "group.id", topic);
Test::conf_set(conf, "enable.auto.commit", "false");
Test::conf_set(conf, "enable.partition.eof", "false");
Test::conf_set(conf, "auto.offset.reset", "earliest");
Test::conf_set(conf, "statistics.interval.ms", "100");
RdKafka::KafkaConsumer *c = RdKafka::KafkaConsumer::create(conf, errstr);
if (!c)
Test::Fail("Failed to create KafkaConsumer: " + errstr);
delete conf;
/* Assign partitions */
/* Subscribe */
std::vector<RdKafka::TopicPartition*> parts;
parts.push_back(RdKafka::TopicPartition::create(topic, 0));
if ((err = c->assign(parts)))
Test::Fail("assign failed: " + RdKafka::err2str(err));
RdKafka::TopicPartition::destroy(parts);
/* Start consuming */
Test::Say("Consuming topic " + topic + "\n");
int cnt = 0;
while (cnt < msgcnt) {
RdKafka::Message *msg = c->consume(tmout_multip(1000));
switch (msg->err())
{
case RdKafka::ERR__TIMED_OUT:
break;
case RdKafka::ERR__PARTITION_EOF:
Test::Fail(tostr() << "Consume error after " << cnt << "/" << msgcnt << " messages: " << msg->errstr());
break;
case RdKafka::ERR_NO_ERROR:
/* Proper message. Update calculated lag for later
* checking in stats callback */
stats.calc_lag = msgcnt - (msg->offset()+1);
cnt++;
Test::Say(2, tostr() << "Received message #" << cnt << "/" << msgcnt <<
" at offset " << msg->offset() << " (calc lag " << stats.calc_lag << ")\n");
/* Slow down message "processing" to make sure we get
* at least one stats callback per message. */
if (cnt < msgcnt)
rd_sleep(1);
break;
default:
Test::Fail("Consume error: " + msg->errstr());
break;
}
delete msg;
}
Test::Say(tostr() << "Done, lag was valid " <<
stats.lag_valid << " times\n");
if (stats.lag_valid == 0)
Test::Fail("No valid consumer_lag in statistics seen");
c->close();
delete c;
}