static int do_test_consume_batch (void) {
#define topic_cnt 2
const char *topics[topic_cnt] = {
test_mk_topic_name(__FUNCTION__, 1),
test_mk_topic_name(__FUNCTION__, 1)
};
const int partition_cnt = 2;
rd_kafka_t *rk;
rd_kafka_queue_t *rkq;
rd_kafka_topic_t *rkts[topic_cnt];
rd_kafka_resp_err_t err;
const int msgcnt = 10000;
uint64_t testid;
int i, p;
int batch_cnt = 0;
int remains;
testid = test_id_generate();
/* Produce messages */
for (i = 0 ; i < topic_cnt ; i++) {
for (p = 0 ; p < partition_cnt ; p++)
test_produce_msgs_easy(topics[i], testid, p,
msgcnt / topic_cnt /
partition_cnt);
}
/* Create simple consumer */
rk = test_create_consumer(NULL, NULL, NULL, NULL);
/* Create generic consume queue */
rkq = rd_kafka_queue_new(rk);
for (i = 0 ; i < topic_cnt ; i++) {
/* Create topic object */
rkts[i] = test_create_topic(rk, topics[i],
"auto.offset.reset", "smallest",
NULL);
/* Start consuming each partition and redirect
* messages to queue */
TEST_SAY("Start consuming topic %s partitions 0..%d\n",
rd_kafka_topic_name(rkts[i]), partition_cnt);
for (p = 0 ; p < partition_cnt ; p++) {
err = rd_kafka_consume_start_queue(
rkts[i], p, RD_KAFKA_OFFSET_BEGINNING, rkq);
if (err)
TEST_FAIL("Failed to start consuming: %s\n",
rd_kafka_err2str(err));
}
}
remains = msgcnt;
/* Consume messages from common queue using batch interface. */
TEST_SAY("Consume %d messages from queue\n", remains);
while (remains > 0) {
rd_kafka_message_t *rkmessage[1000];
ssize_t r;
test_timing_t t_batch;
TIMING_START(&t_batch, "CONSUME.BATCH");
r = rd_kafka_consume_batch_queue(rkq, 1000, rkmessage, 1000);
TIMING_STOP(&t_batch);
TEST_SAY("Batch consume iteration #%d: Consumed %"PRIdsz
"/1000 messages\n", batch_cnt, r);
if (r == -1)
TEST_FAIL("Failed to consume messages: %s\n",
rd_kafka_err2str(rd_kafka_errno2err(errno)));
remains -= r;
for (i = 0 ; i < r ; i++)
rd_kafka_message_destroy(rkmessage[i]);
batch_cnt++;
}
TEST_SAY("Stopping consumer\n");
for (i = 0 ; i < topic_cnt ; i++) {
for (p = 0 ; p < partition_cnt ; p++) {
err = rd_kafka_consume_stop(rkts[i], p);
if (err)
TEST_FAIL("Failed to stop consuming: %s\n",
rd_kafka_err2str(err));
}
rd_kafka_topic_destroy(rkts[i]);
}
rd_kafka_queue_destroy(rkq);
rd_kafka_destroy(rk);
return 0;
}
int main_0008_reqacks (int argc, char **argv) {
int partition = 0;
int r;
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char errstr[512];
char msg[128];
int msgcnt = 100;
int i;
int reqacks;
int idbase = 0;
const char *topic = NULL;
TEST_SAY("\033[33mNOTE! This test requires at "
"least 3 brokers!\033[0m\n");
TEST_SAY("\033[33mNOTE! This test requires "
"default.replication.factor=3 to be configured on "
"all brokers!\033[0m\n");
/* Try different request.required.acks settings (issue #75) */
for (reqacks = -1 ; reqacks <= 1 ; reqacks++) {
char tmp[10];
test_conf_init(&conf, &topic_conf, 10);
if (!topic)
topic = test_mk_topic_name("0008", 0);
rd_snprintf(tmp, sizeof(tmp), "%i", reqacks);
if (rd_kafka_topic_conf_set(topic_conf, "request.required.acks",
tmp, errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK)
TEST_FAIL("%s", errstr);
/* Set delivery report callback */
rd_kafka_conf_set_dr_cb(conf, dr_cb);
/* Create kafka instance */
rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr));
if (!rk)
TEST_FAIL("Failed to create rdkafka instance: %s\n",
errstr);
TEST_SAY("Created kafka instance %s with required acks %i\n",
rd_kafka_name(rk), reqacks);
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
rd_strerror(errno));
/* Produce messages */
for (i = 0 ; i < msgcnt ; i++) {
int *msgidp = malloc(sizeof(*msgidp));
*msgidp = idbase + i;
rd_snprintf(msg, sizeof(msg),
"%s test message #%i (acks=%i)",
argv[0], *msgidp, reqacks);
r = rd_kafka_produce(rkt, partition,
RD_KAFKA_MSG_F_COPY,
msg, strlen(msg), NULL, 0, msgidp);
if (r == -1)
TEST_FAIL("Failed to produce message #%i: %s\n",
*msgidp, rd_strerror(errno));
}
TEST_SAY("Produced %i messages, waiting for deliveries\n",
msgcnt);
/* Wait for messages to time out */
while (rd_kafka_outq_len(rk) > 0)
rd_kafka_poll(rk, 50);
if (fails)
TEST_FAIL("%i failures, see previous errors", fails);
if (msgid_next != idbase + msgcnt)
TEST_FAIL("Still waiting for messages: "
"next %i != end %i\n",
msgid_next, msgcnt);
idbase += i;
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy rdkafka instance */
TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
rd_kafka_destroy(rk);
}
return 0;
}
static void kafka_config_topic(rd_kafka_conf_t *conf, oconfig_item_t *ci) /* {{{ */
{
int status;
int i;
struct kafka_topic_context *tctx;
char *key;
char *val;
char callback_name[DATA_MAX_NAME_LEN];
char errbuf[1024];
user_data_t ud;
oconfig_item_t *child;
rd_kafka_conf_res_t ret;
if ((tctx = calloc(1, sizeof (*tctx))) == NULL) {
ERROR ("write_kafka plugin: calloc failed.");
return;
}
tctx->escape_char = '.';
tctx->store_rates = 1;
rd_kafka_conf_set_log_cb(conf, kafka_log);
if ((tctx->kafka = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errbuf, sizeof(errbuf))) == NULL) {
sfree(tctx);
ERROR("write_kafka plugin: cannot create kafka handle.");
return;
}
conf = NULL;
if ((tctx->conf = rd_kafka_topic_conf_new()) == NULL) {
rd_kafka_destroy(tctx->kafka);
sfree(tctx);
ERROR ("write_kafka plugin: cannot create topic configuration.");
return;
}
if (ci->values_num != 1) {
WARNING("kafka topic name needed.");
goto errout;
}
if (ci->values[0].type != OCONFIG_TYPE_STRING) {
WARNING("kafka topic needs a string argument.");
goto errout;
}
if ((tctx->topic_name = strdup(ci->values[0].value.string)) == NULL) {
ERROR("write_kafka plugin: cannot copy topic name.");
goto errout;
}
for (i = 0; i < ci->children_num; i++) {
/*
* The code here could be simplified but makes room
* for easy adding of new options later on.
*/
child = &ci->children[i];
status = 0;
if (strcasecmp ("Property", child->key) == 0) {
if (child->values_num != 2) {
WARNING("kafka properties need both a key and a value.");
goto errout;
}
if (child->values[0].type != OCONFIG_TYPE_STRING ||
child->values[1].type != OCONFIG_TYPE_STRING) {
WARNING("kafka properties needs string arguments.");
goto errout;
}
key = child->values[0].value.string;
val = child->values[0].value.string;
ret = rd_kafka_topic_conf_set(tctx->conf,key, val,
errbuf, sizeof(errbuf));
if (ret != RD_KAFKA_CONF_OK) {
WARNING("cannot set kafka topic property %s to %s: %s.",
key, val, errbuf);
goto errout;
}
} else if (strcasecmp ("Key", child->key) == 0) {
char *tmp_buf = NULL;
status = cf_util_get_string(child, &tmp_buf);
if (status != 0) {
WARNING("write_kafka plugin: invalid key supplied");
break;
}
if (strcasecmp(tmp_buf, "Random") != 0) {
tctx->has_key = 1;
tctx->key = crc32_buffer((u_char *)tmp_buf, strlen(tmp_buf));
}
sfree(tmp_buf);
} else if (strcasecmp ("Format", child->key) == 0) {
status = cf_util_get_string(child, &key);
if (status != 0)
goto errout;
assert(key != NULL);
if (strcasecmp(key, "Command") == 0) {
tctx->format = KAFKA_FORMAT_COMMAND;
} else if (strcasecmp(key, "Graphite") == 0) {
tctx->format = KAFKA_FORMAT_GRAPHITE;
} else if (strcasecmp(key, "Json") == 0) {
tctx->format = KAFKA_FORMAT_JSON;
} else {
WARNING ("write_kafka plugin: Invalid format string: %s",
key);
}
sfree(key);
} else if (strcasecmp ("StoreRates", child->key) == 0) {
status = cf_util_get_boolean (child, &tctx->store_rates);
(void) cf_util_get_flag (child, &tctx->graphite_flags,
GRAPHITE_STORE_RATES);
} else if (strcasecmp ("GraphiteSeparateInstances", child->key) == 0) {
status = cf_util_get_flag (child, &tctx->graphite_flags,
GRAPHITE_SEPARATE_INSTANCES);
} else if (strcasecmp ("GraphiteAlwaysAppendDS", child->key) == 0) {
status = cf_util_get_flag (child, &tctx->graphite_flags,
GRAPHITE_ALWAYS_APPEND_DS);
} else if (strcasecmp ("GraphitePrefix", child->key) == 0) {
status = cf_util_get_string (child, &tctx->prefix);
} else if (strcasecmp ("GraphitePostfix", child->key) == 0) {
status = cf_util_get_string (child, &tctx->postfix);
} else if (strcasecmp ("GraphiteEscapeChar", child->key) == 0) {
char *tmp_buff = NULL;
status = cf_util_get_string (child, &tmp_buff);
if (strlen (tmp_buff) > 1)
WARNING ("write_kafka plugin: The option \"GraphiteEscapeChar\" handles "
"only one character. Others will be ignored.");
tctx->escape_char = tmp_buff[0];
sfree (tmp_buff);
} else {
WARNING ("write_kafka plugin: Invalid directive: %s.", child->key);
}
if (status != 0)
break;
}
rd_kafka_topic_conf_set_partitioner_cb(tctx->conf, kafka_partition);
rd_kafka_topic_conf_set_opaque(tctx->conf, tctx);
if ((tctx->topic = rd_kafka_topic_new(tctx->kafka, tctx->topic_name,
tctx->conf)) == NULL) {
ERROR("write_kafka plugin: cannot create topic.");
goto errout;
}
tctx->conf = NULL;
ssnprintf(callback_name, sizeof(callback_name),
"write_kafka/%s", tctx->topic_name);
ud.data = tctx;
ud.free_func = kafka_topic_context_free;
status = plugin_register_write (callback_name, kafka_write, &ud);
if (status != 0) {
WARNING ("write_kafka plugin: plugin_register_write (\"%s\") "
"failed with status %i.",
callback_name, status);
goto errout;
}
return;
errout:
if (conf != NULL)
rd_kafka_conf_destroy(conf);
if (tctx->kafka != NULL)
rd_kafka_destroy(tctx->kafka);
if (tctx->topic != NULL)
rd_kafka_topic_destroy(tctx->topic);
if (tctx->topic_name != NULL)
free(tctx->topic_name);
if (tctx->conf != NULL)
rd_kafka_topic_conf_destroy(tctx->conf);
sfree(tctx);
} /* }}} int kafka_config_topic */
int main_0088_produce_metadata_timeout (int argc, char **argv) {
int64_t testid;
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
const char *topic = test_mk_topic_name("0088_produce_metadata_timeout",
1);
int msgcnt = 0;
rd_kafka_conf_t *conf;
testid = test_id_generate();
/* Create topic with single partition, for simplicity. */
test_create_topic(topic, 1, 1);
test_conf_init(&conf, NULL, 15*60*2); // msgcnt * 2);
rd_kafka_conf_set_dr_msg_cb(conf, dr_msg_cb);
test_conf_set(conf, "metadata.max.age.ms", "10000");
test_conf_set(conf, "topic.metadata.refresh.interval.ms", "-1");
test_conf_set(conf, "linger.ms", "5000");
test_conf_set(conf, "batch.num.messages", "5");
test_socket_enable(conf);
test_curr->connect_cb = connect_cb;
test_curr->is_fatal_cb = is_fatal_cb;
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
rkt = rd_kafka_topic_new(rk, topic, NULL);
/* Produce first set of messages and wait for delivery */
test_produce_msgs_nowait(rk, rkt, testid, RD_KAFKA_PARTITION_UA,
msgcnt, 20, NULL, 0, &msgcnt);
while (msg_dr_cnt < 5)
rd_kafka_poll(rk, 1000);
TEST_SAY(_C_YEL "Disconnecting sockets and "
"refusing future connections\n");
rd_atomic32_set(&refuse_connect, 1);
test_socket_close_all(test_curr, 1/*reinit*/);
/* Wait for metadata timeout */
TEST_SAY("Waiting for metadata timeout\n");
rd_sleep(10+5);
/* These messages will be put on the UA queue */
test_produce_msgs_nowait(rk, rkt, testid, RD_KAFKA_PARTITION_UA,
msgcnt, 20, NULL, 0, &msgcnt);
/* Restore the connection(s) when metadata has timed out. */
TEST_SAY(_C_YEL "Allowing connections\n");
rd_atomic32_set(&refuse_connect, 0);
rd_sleep(3);
test_produce_msgs_nowait(rk, rkt, testid, RD_KAFKA_PARTITION_UA,
msgcnt, 20, NULL, 0, &msgcnt);
test_flush(rk, 2*5*1000); /* linger.ms * 2 */
TEST_ASSERT(msg_dr_cnt == msgcnt,
"expected %d, got %d", msgcnt, msg_dr_cnt);
TEST_ASSERT(msg_dr_fail_cnt == 0,
"expected %d dr failures, got %d", 0, msg_dr_fail_cnt);
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
return 0;
}
rd_kafka_t *kafka_get_connection(kafka_connection_params params, const char *brokers)
{
rd_kafka_t *r = NULL;
char errstr[512];
rd_kafka_conf_t *conf = rd_kafka_conf_new();
//set error callback
rd_kafka_conf_set_error_cb(conf, kafka_err_cb);
if (params.type == RD_KAFKA_CONSUMER)
{
if (params.queue_buffer)
rd_kafka_conf_set(conf, "queued.min.messages", params.queue_buffer, NULL, 0);
r = rd_kafka_new(params.type, conf, errstr, sizeof errstr);
if (!r)
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "Failed to connect to kafka: %s", errstr);
}
//destroy config, no connection to use it...
rd_kafka_conf_destroy(conf);
return NULL;
}
if (!rd_kafka_brokers_add(r, brokers))
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "Failed to connect to brokers %s", brokers);
}
rd_kafka_destroy(r);
return NULL;
}
return r;
}
if (params.compression)
{
rd_kafka_conf_res_t result = rd_kafka_conf_set(
conf, "compression.codec",params.compression, errstr, sizeof errstr
);
if (result != RD_KAFKA_CONF_OK)
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ALERT, "Failed to set compression %s: %s", params.compression, errstr);
}
rd_kafka_conf_destroy(conf);
return NULL;
}
}
if (params.retry_count)
{
rd_kafka_conf_res_t result = rd_kafka_conf_set(
conf, "message.send.max.retries",params.retry_count, errstr, sizeof errstr
);
if (result != RD_KAFKA_CONF_OK)
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ALERT, "Failed to set compression %s: %s", params.compression, errstr);
}
rd_kafka_conf_destroy(conf);
return NULL;
}
}
if (params.retry_interval)
{
rd_kafka_conf_res_t result = rd_kafka_conf_set(
conf, "retry.backoff.ms",params.retry_interval, errstr, sizeof errstr
);
if (result != RD_KAFKA_CONF_OK)
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ALERT, "Failed to set compression %s: %s", params.compression, errstr);
}
rd_kafka_conf_destroy(conf);
return NULL;
}
}
if (params.reporting == 1)
rd_kafka_conf_set_dr_cb(conf, kafka_produce_cb_simple);
else if (params.reporting == 2)
rd_kafka_conf_set_dr_msg_cb(conf, kafka_produce_detailed_cb);
r = rd_kafka_new(params.type, conf, errstr, sizeof errstr);
if (!r)
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "Failed to connect to kafka: %s", errstr);
}
//destroy config, no connection to use it...
rd_kafka_conf_destroy(conf);
return NULL;
}
if (!rd_kafka_brokers_add(r, brokers))
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "Failed to connect to brokers %s", brokers);
}
rd_kafka_destroy(r);
return NULL;
}
return r;
}
int main_0015_offsets_seek (int argc, char **argv) {
const char *topic = test_mk_topic_name("0015", 1);
rd_kafka_t *rk_p, *rk_c;
rd_kafka_topic_t *rkt_p, *rkt_c;
int msg_cnt = 1000;
int msg_base = 0;
int32_t partition = 0;
int i;
int64_t offset_last, offset_base;
uint64_t testid;
int dance_iterations = 10;
int msgs_per_dance = 10;
testid = test_id_generate();
/* Produce messages */
rk_p = test_create_producer();
rkt_p = test_create_producer_topic(rk_p, topic, NULL);
test_produce_msgs(rk_p, rkt_p, testid, partition, msg_base, msg_cnt,
NULL, 0);
rd_kafka_topic_destroy(rkt_p);
rd_kafka_destroy(rk_p);
rk_c = test_create_consumer(NULL, NULL, NULL, NULL);
rkt_c = test_create_consumer_topic(rk_c, topic);
/* Start consumer tests */
test_consumer_start("verify.all", rkt_c, partition,
RD_KAFKA_OFFSET_BEGINNING);
/* Make sure all messages are available */
offset_last = test_consume_msgs("verify.all", rkt_c,
testid, partition, TEST_NO_SEEK,
msg_base, msg_cnt, 1/* parse format*/);
/* Rewind offset back to its base. */
offset_base = offset_last - msg_cnt + 1;
TEST_SAY("%s [%"PRId32"]: Do random seek&consume for msgs #%d+%d with "
"offsets %"PRId64"..%"PRId64"\n",
rd_kafka_topic_name(rkt_c), partition,
msg_base, msg_cnt, offset_base, offset_last);
/* Now go dancing over the entire range with offset seeks. */
for (i = 0 ; i < dance_iterations ; i++) {
int64_t offset = jitter((int)offset_base,
(int)offset_base+msg_cnt);
test_consume_msgs("dance", rkt_c,
testid, partition, offset,
msg_base + (int)(offset - offset_base),
RD_MIN(msgs_per_dance,
(int)(offset_last - offset)),
1 /* parse format */);
}
test_consumer_stop("1", rkt_c, partition);
rd_kafka_topic_destroy(rkt_c);
rd_kafka_destroy(rk_c);
return 0;
}
/* @remark This test will fail if auto topic creation is enabled on the broker
* since the client will issue a topic-creating metadata request to find
* a new leader when the topic is removed.
*
* To run with trivup, do:
* ./interactive_broker_version.py .. -conf '{"auto_create_topics":"false"}' ..
* TESTS=0045 ./run-test.sh -k ./merged
*/
static void do_test_topic_remove (void) {
char *topic_f = rd_strdup(test_mk_topic_name("topic_f", 1));
char *topic_g = rd_strdup(test_mk_topic_name("topic_g", 1));
int parts_f = 5;
int parts_g = 9;
rd_kafka_t *rk;
rd_kafka_conf_t *conf;
rd_kafka_queue_t *queue;
rd_kafka_topic_partition_list_t *topics;
rd_kafka_resp_err_t err;
/**
* Topic removal test:
* - Create topic f & g
* - Subscribe to f & g
* - Verify f & g assignment
* - Remove topic f
* - Verify g assignment
* - Remove topic g
* - Verify empty assignment
*/
TEST_SAY("Topic removal testing\n");
test_conf_init(&conf, NULL, 60);
/* Decrease metadata interval to speed up topic change discovery. */
test_conf_set(conf, "metadata.max.age.ms", "5000");
rd_kafka_conf_set_events(conf, RD_KAFKA_EVENT_REBALANCE);
rk = test_create_consumer(test_str_id_generate_tmp(),
NULL, conf, NULL, NULL);
queue = rd_kafka_queue_get_consumer(rk);
TEST_SAY("Topic removal: creating topic %s (subscribed)\n", topic_f);
test_create_topic(topic_f, parts_f, 1);
TEST_SAY("Topic removal: creating topic %s (subscribed)\n", topic_g);
test_create_topic(topic_g, parts_g, 1);
rd_sleep(1); // FIXME: do check&wait loop instead
TEST_SAY("Topic removal: Subscribing to %s & %s\n", topic_f, topic_g);
topics = rd_kafka_topic_partition_list_new(2);
rd_kafka_topic_partition_list_add(topics, topic_f, RD_KAFKA_PARTITION_UA);
rd_kafka_topic_partition_list_add(topics, topic_g, RD_KAFKA_PARTITION_UA);
err = rd_kafka_subscribe(rk, topics);
TEST_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR,
"%s", rd_kafka_err2str(err));
rd_kafka_topic_partition_list_destroy(topics);
await_assignment("Topic removal: both topics exist", rk, queue, 2,
topic_f, parts_f,
topic_g, parts_g);
TEST_SAY("Topic removal: removing %s\n", topic_f);
test_kafka_topics("--delete --topic %s", topic_f);
await_revoke("Topic removal: rebalance after topic removal", rk, queue);
await_assignment("Topic removal: one topic exists", rk, queue, 1,
topic_g, parts_g);
TEST_SAY("Topic removal: removing %s\n", topic_g);
test_kafka_topics("--delete --topic %s", topic_g);
await_revoke("Topic removal: rebalance after 2nd topic removal",
rk, queue);
/* Should not see another rebalance since all topics now removed */
await_no_rebalance("Topic removal: empty", rk, queue, 10000);
test_consumer_close(rk);
rd_kafka_queue_destroy(queue);
rd_kafka_destroy(rk);
rd_free(topic_f);
rd_free(topic_g);
}
ECL_KAFKA_API __int32 ECL_KAFKA_CALL getTopicPartitionCount(ICodeContext* ctx, const char* brokers, const char* topic)
{
// We have to use librdkafka's C API for this right now, as the C++ API
// does not expose a topic's metadata. In addition, there is no easy
// link between the exposed C++ objects and the structs used by the
// C API, so we are basically creating a brand-new connection from
// scratch.
__int32 pCount = 0;
char errstr[512];
rd_kafka_conf_t* conf = rd_kafka_conf_new();
rd_kafka_t* rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr));
if (rk)
{
if (rd_kafka_brokers_add(rk, brokers) != 0)
{
rd_kafka_topic_conf_t* topic_conf = rd_kafka_topic_conf_new();
rd_kafka_topic_t* rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (rkt)
{
const struct rd_kafka_metadata* metadata = NULL;
rd_kafka_resp_err_t err = rd_kafka_metadata(rk, 0, rkt, &metadata, 5000);
if (err == RD_KAFKA_RESP_ERR_NO_ERROR)
{
pCount = metadata->topics[0].partition_cnt;
rd_kafka_metadata_destroy(metadata);
}
else
{
if (ctx->queryContextLogger().queryTraceLevel() > 4)
{
DBGLOG("Kafka: Error retrieving metadata from topic: %s @ %s: '%s'", topic, brokers, rd_kafka_err2str(err));
}
}
rd_kafka_topic_destroy(rkt);
}
else
{
if (ctx->queryContextLogger().queryTraceLevel() > 4)
{
DBGLOG("Kafka: Could not create topic object: %s @ %s", topic, brokers);
}
}
}
else
{
if (ctx->queryContextLogger().queryTraceLevel() > 4)
{
DBGLOG("Kafka: Could not add brokers: %s @ %s", topic, brokers);
}
}
rd_kafka_destroy(rk);
}
if (pCount == 0)
{
DBGLOG("Kafka: Unable to retrieve partition count from topic: %s @ %s", topic, brokers);
}
return pCount;
}
static void do_offset_test (const char *what, int auto_commit, int auto_store,
int async) {
test_timing_t t_all;
char groupid[64];
rd_kafka_t *rk;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *tconf;
int cnt = 0;
const int extra_cnt = 5;
rd_kafka_resp_err_t err;
rd_kafka_topic_partition_list_t *parts;
rd_kafka_topic_partition_t *rktpar;
int64_t next_offset = -1;
test_conf_init(&conf, &tconf, 20);
test_conf_set(conf, "enable.auto.commit", auto_commit ? "true":"false");
test_conf_set(conf, "enable.auto.offset.store", auto_store ?"true":"false");
test_conf_set(conf, "auto.commit.interval.ms", "500");
rd_kafka_conf_set_offset_commit_cb(conf, offset_commit_cb);
test_topic_conf_set(tconf, "auto.offset.reset", "smallest");
test_str_id_generate(groupid, sizeof(groupid));
test_conf_set(conf, "group.id", groupid);
rd_kafka_conf_set_default_topic_conf(conf, tconf);
TEST_SAY(_C_MAG "[ do_offset_test: %s with group.id %s ]\n",
what, groupid);
TIMING_START(&t_all, what);
expected_offset = 0;
committed_offset = -1;
/* MO:
* - Create consumer.
* - Start consuming from beginning
* - Perform store & commits according to settings
* - Stop storing&committing when half of the messages are consumed,
* - but consume 5 more to check against.
* - Query position.
* - Destroy consumer.
* - Create new consumer with same group.id using stored offsets
* - Should consume the expected message.
*/
/* Create kafka instance */
rk = test_create_handle(RD_KAFKA_CONSUMER, rd_kafka_conf_dup(conf));
rd_kafka_poll_set_consumer(rk);
test_consumer_subscribe(rk, topic);
while (cnt - extra_cnt < msgcnt / 2) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(rk, 10*1000);
if (!rkm)
continue;
if (rkm->err == RD_KAFKA_RESP_ERR__TIMED_OUT)
TEST_FAIL("%s: Timed out waiting for message %d", what,cnt);
else if (rkm->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
rd_kafka_message_destroy(rkm);
continue;
} else if (rkm->err)
TEST_FAIL("%s: Consumer error: %s",
what, rd_kafka_message_errstr(rkm));
/* Offset of next message. */
next_offset = rkm->offset + 1;
if (cnt < msgcnt / 2) {
if (!auto_store) {
err = rd_kafka_offset_store(rkm->rkt,rkm->partition,
rkm->offset);
if (err)
TEST_FAIL("%s: offset_store failed: %s\n",
what, rd_kafka_err2str(err));
}
expected_offset = rkm->offset+1;
if (!auto_commit) {
test_timing_t t_commit;
TIMING_START(&t_commit,
async?"commit.async":"commit.sync");
err = rd_kafka_commit_message(rk, rkm, async);
TIMING_STOP(&t_commit);
if (err)
TEST_FAIL("%s: commit failed: %s\n",
what, rd_kafka_err2str(err));
}
} else if (auto_store && auto_commit)
expected_offset = rkm->offset+1;
rd_kafka_message_destroy(rkm);
cnt++;
}
TEST_SAY("%s: done consuming after %d messages, at offset %"PRId64"\n",
what, cnt, expected_offset);
if ((err = rd_kafka_assignment(rk, &parts)))
TEST_FAIL("%s: failed to get assignment(): %s\n",
what, rd_kafka_err2str(err));
/* Verify position */
if ((err = rd_kafka_position(rk, parts)))
TEST_FAIL("%s: failed to get position(): %s\n",
what, rd_kafka_err2str(err));
if (!(rktpar = rd_kafka_topic_partition_list_find(parts,
topic, partition)))
TEST_FAIL("%s: position(): topic lost\n", what);
if (rktpar->offset != next_offset)
TEST_FAIL("%s: Expected position() offset %"PRId64", got %"PRId64,
what, next_offset, rktpar->offset);
TEST_SAY("%s: Position is at %"PRId64", good!\n",
what, rktpar->offset);
/* Pause messages while waiting so we can serve callbacks
* without having more messages received. */
if ((err = rd_kafka_pause_partitions(rk, parts)))
TEST_FAIL("%s: failed to pause partitions: %s\n",
what, rd_kafka_err2str(err));
rd_kafka_topic_partition_list_destroy(parts);
/* Fire off any enqueued offset_commit_cb */
test_consumer_poll_no_msgs(what, rk, testid, 0);
TEST_SAY("%s: committed_offset %"PRId64", expected_offset %"PRId64"\n",
what, committed_offset, expected_offset);
if (!auto_commit && !async) {
/* Sync commits should be up to date at this point. */
if (committed_offset != expected_offset)
TEST_FAIL("%s: Sync commit: committed offset %"PRId64
" should be same as expected offset "
"%"PRId64,
what, committed_offset, expected_offset);
} else {
/* Wait for offset commits to catch up */
while (committed_offset < expected_offset) {
TEST_SAYL(3, "%s: Wait for committed offset %"PRId64
" to reach expected offset %"PRId64"\n",
what, committed_offset, expected_offset);
test_consumer_poll_no_msgs(what, rk, testid, 1000);
}
}
TEST_SAY("%s: phase 1 complete, %d messages consumed, "
"next expected offset is %"PRId64"\n",
what, cnt, expected_offset);
/* Issue #827: cause committed() to return prematurely by specifying
* low timeout. The bug (use after free) will only
* be catched by valgrind. */
do {
parts = rd_kafka_topic_partition_list_new(1);
rd_kafka_topic_partition_list_add(parts, topic, partition);
err = rd_kafka_committed(rk, parts, 1);
rd_kafka_topic_partition_list_destroy(parts);
TEST_SAY("Issue #827: committed() returned %s\n",
rd_kafka_err2str(err));
} while (err != RD_KAFKA_RESP_ERR__TIMED_OUT);
/* Query position */
parts = rd_kafka_topic_partition_list_new(1);
rd_kafka_topic_partition_list_add(parts, topic, partition);
err = rd_kafka_committed(rk, parts, tmout_multip(5*1000));
if (err)
TEST_FAIL("%s: committed() failed: %s", what, rd_kafka_err2str(err));
if (!(rktpar = rd_kafka_topic_partition_list_find(parts,
topic, partition)))
TEST_FAIL("%s: committed(): topic lost\n", what);
if (rktpar->offset != expected_offset)
TEST_FAIL("%s: Expected committed() offset %"PRId64", got %"PRId64,
what, expected_offset, rktpar->offset);
TEST_SAY("%s: Committed offset is at %"PRId64", good!\n",
what, rktpar->offset);
rd_kafka_topic_partition_list_destroy(parts);
test_consumer_close(rk);
rd_kafka_destroy(rk);
/* Fire up a new consumer and continue from where we left off. */
TEST_SAY("%s: phase 2: starting new consumer to resume consumption\n",what);
rk = test_create_handle(RD_KAFKA_CONSUMER, conf);
rd_kafka_poll_set_consumer(rk);
test_consumer_subscribe(rk, topic);
while (cnt < msgcnt) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(rk, 10*1000);
if (!rkm)
continue;
if (rkm->err == RD_KAFKA_RESP_ERR__TIMED_OUT)
TEST_FAIL("%s: Timed out waiting for message %d", what,cnt);
else if (rkm->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
rd_kafka_message_destroy(rkm);
continue;
} else if (rkm->err)
TEST_FAIL("%s: Consumer error: %s",
what, rd_kafka_message_errstr(rkm));
if (rkm->offset != expected_offset)
TEST_FAIL("%s: Received message offset %"PRId64
", expected %"PRId64" at msgcnt %d/%d\n",
what, rkm->offset, expected_offset,
cnt, msgcnt);
rd_kafka_message_destroy(rkm);
expected_offset++;
cnt++;
}
TEST_SAY("%s: phase 2: complete\n", what);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_all);
}
/**
* Produces 'msgcnt' messages split over 'partition_cnt' partitions.
*/
static void produce_messages (uint64_t testid, const char *topic,
int partition_cnt, int msgcnt) {
int r;
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char errstr[512];
char msg[128];
int failcnt = 0;
int i;
rd_kafka_message_t *rkmessages;
int32_t partition;
int msgid = 0;
test_conf_init(&conf, &topic_conf, 20);
rd_kafka_conf_set_dr_cb(conf, dr_cb);
/* Make sure all replicas are in-sync after producing
* so that consume test wont fail. */
rd_kafka_topic_conf_set(topic_conf, "request.required.acks", "-1",
errstr, sizeof(errstr));
/* Create kafka instance */
rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr));
if (!rk)
TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr);
TEST_SAY("Created kafka instance %s\n", rd_kafka_name(rk));
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
rd_strerror(errno));
/* Create messages. */
prod_msg_remains = msgcnt;
rkmessages = calloc(sizeof(*rkmessages), msgcnt / partition_cnt);
for (partition = 0 ; partition < partition_cnt ; partition++) {
int batch_cnt = msgcnt / partition_cnt;
for (i = 0 ; i < batch_cnt ; i++) {
rd_snprintf(msg, sizeof(msg),
"testid=%"PRIu64", partition=%i, msg=%i",
testid, (int)partition, msgid);
rkmessages[i].payload = rd_strdup(msg);
rkmessages[i].len = strlen(msg);
msgid++;
}
TEST_SAY("Start produce to partition %i: msgs #%d..%d\n",
(int)partition, msgid-batch_cnt, msgid);
/* Produce batch for this partition */
r = rd_kafka_produce_batch(rkt, partition, RD_KAFKA_MSG_F_FREE,
rkmessages, batch_cnt);
if (r == -1)
TEST_FAIL("Failed to produce "
"batch for partition %i: %s",
(int)partition,
rd_kafka_err2str(rd_kafka_errno2err(errno)));
/* Scan through messages to check for errors. */
for (i = 0 ; i < batch_cnt ; i++) {
if (rkmessages[i].err) {
failcnt++;
if (failcnt < 100)
TEST_SAY("Message #%i failed: %s\n",
i,
rd_kafka_err2str(rkmessages[i].
err));
}
}
/* All messages should've been produced. */
if (r < batch_cnt) {
TEST_SAY("Not all messages were accepted "
"by produce_batch(): %i < %i\n", r, batch_cnt);
if (batch_cnt - r != failcnt)
TEST_SAY("Discrepency between failed "
"messages (%i) "
"and return value %i (%i - %i)\n",
failcnt, batch_cnt - r, batch_cnt, r);
TEST_FAIL("%i/%i messages failed\n",
batch_cnt - r, batch_cnt);
}
TEST_SAY("Produced %i messages to partition %i, "
"waiting for deliveries\n", r, partition);
}
free(rkmessages);
/* Wait for messages to be delivered */
while (rd_kafka_outq_len(rk) > 0)
rd_kafka_poll(rk, 100);
if (fails)
TEST_FAIL("%i failures, see previous errors", fails);
if (prod_msg_remains != 0)
TEST_FAIL("Still waiting for %i messages to be produced",
prod_msg_remains);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy rdkafka instance */
TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
rd_kafka_destroy(rk);
}
int main (int argc, char **argv) {
char *topic = "rdkafkatest1";
int partition = 0;
int r;
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char errstr[512];
char msg[100000];
int msgcnt = 10;
int i;
test_conf_init(&conf, &topic_conf, 10);
/* Set a small maximum message size. */
if (rd_kafka_conf_set(conf, "message.max.bytes", "100000",
errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK)
TEST_FAIL("%s\n", errstr);
/* Set delivery report callback */
rd_kafka_conf_set_dr_cb(conf, dr_cb);
/* Create kafka instance */
rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr));
if (!rk)
TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr);
TEST_SAY("Created kafka instance %s\n", rd_kafka_name(rk));
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
strerror(errno));
memset(msg, 0, sizeof(msg));
/* Produce 'msgcnt' messages, size odd ones larger than max.bytes,
* and even ones smaller than max.bytes. */
for (i = 0 ; i < msgcnt ; i++) {
int *msgidp = malloc(sizeof(*msgidp));
size_t len;
int toobig = i & 1;
*msgidp = i;
if (toobig) {
/* Too big */
len = 200000;
} else {
/* Good size */
len = 5000;
msgs_wait |= (1 << i);
}
snprintf(msg, sizeof(msg), "%s test message #%i", argv[0], i);
r = rd_kafka_produce(rkt, partition, RD_KAFKA_MSG_F_COPY,
msg, len, NULL, 0, msgidp);
if (toobig) {
if (r != -1)
TEST_FAIL("Succeeded to produce too "
"large message #%i\n", i);
free(msgidp);
} else if (r == -1)
TEST_FAIL("Failed to produce message #%i: %s\n",
i, strerror(errno));
}
/* Wait for messages to be delivered. */
while (rd_kafka_outq_len(rk) > 0)
rd_kafka_poll(rk, 50);
if (msgs_wait != 0)
TEST_FAIL("Still waiting for messages: 0x%x\n", msgs_wait);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy rdkafka instance */
TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
rd_kafka_destroy(rk);
/* Wait for everything to be cleaned up since broker destroys are
* handled in its own thread. */
test_wait_exit(10);
/* If we havent failed at this point then
* there were no threads leaked */
return 0;
}
static void consume_messages_with_queues (uint64_t testid, const char *topic,
int partition_cnt, int msgcnt) {
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
rd_kafka_queue_t *rkqu;
char errstr[512];
int i;
int32_t partition;
int batch_cnt = msgcnt / partition_cnt;
test_conf_init(&conf, &topic_conf, 20);
/* Create kafka instance */
rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr));
if (!rk)
TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr);
TEST_SAY("Created kafka instance %s\n", rd_kafka_name(rk));
/* Create queue */
rkqu = rd_kafka_queue_new(rk);
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
rd_strerror(errno));
TEST_SAY("Consuming %i messages from one queue serving %i partitions\n",
msgcnt, partition_cnt);
/* Start consuming each partition */
for (partition = 0 ; partition < partition_cnt ; partition++) {
/* Consume messages */
TEST_SAY("Start consuming partition %i at offset -%i\n",
partition, batch_cnt);
if (rd_kafka_consume_start_queue(rkt, partition,
RD_KAFKA_OFFSET_TAIL(batch_cnt),
rkqu) == -1)
TEST_FAIL("consume_start_queue(%i) failed: %s",
(int)partition,
rd_kafka_err2str(rd_kafka_errno2err(errno)));
}
/* Consume messages from queue */
for (i = 0 ; i < msgcnt ; ) {
rd_kafka_message_t *rkmessage;
rkmessage = rd_kafka_consume_queue(rkqu, 5000);
if (!rkmessage)
TEST_FAIL("Failed to consume message %i/%i from "
"queue: %s",
i, msgcnt,
rd_kafka_err2str(rd_kafka_errno2err(errno)));
if (rkmessage->err) {
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF){
TEST_SAY("Topic %s [%"PRId32"] reached "
"EOF at offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset);
rd_kafka_message_destroy(rkmessage);
continue;
}
TEST_FAIL("Consume message %i/%i from queue "
"has error (offset %"PRId64
", partition %"PRId32"): %s",
i, msgcnt,
rkmessage->offset, rkmessage->partition,
rd_kafka_err2str(rkmessage->err));
}
verify_consumed_msg(testid, -1, -1, rkmessage);
rd_kafka_message_destroy(rkmessage);
i++;
}
/* Stop consuming each partition */
for (partition = 0 ; partition < partition_cnt ; partition++)
rd_kafka_consume_stop(rkt, partition);
/* Destroy queue */
rd_kafka_queue_destroy(rkqu);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy rdkafka instance */
TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
rd_kafka_destroy(rk);
}
int main_0049_consume_conn_close (int argc, char **argv) {
rd_kafka_t *rk;
const char *topic = test_mk_topic_name("0049_consume_conn_close", 1);
uint64_t testid;
int msgcnt = test_on_ci ? 1000 : 10000;
test_msgver_t mv;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *tconf;
rd_kafka_topic_partition_list_t *assignment;
rd_kafka_resp_err_t err;
if (!test_conf_match(NULL, "sasl.mechanisms", "GSSAPI")) {
TEST_SKIP("KNOWN ISSUE: ApiVersionRequest+SaslHandshake "
"will not play well with sudden disconnects\n");
return 0;
}
test_conf_init(&conf, &tconf, 60);
/* Want an even number so it is divisable by two without surprises */
msgcnt = (msgcnt / (int)test_timeout_multiplier) & ~1;
testid = test_id_generate();
test_produce_msgs_easy(topic, testid, RD_KAFKA_PARTITION_UA, msgcnt);
test_socket_enable(conf);
test_curr->connect_cb = connect_cb;
test_curr->is_fatal_cb = is_fatal_cb;
test_topic_conf_set(tconf, "auto.offset.reset", "smallest");
rk = test_create_consumer(topic, NULL, conf, tconf);
test_consumer_subscribe(rk, topic);
test_msgver_init(&mv, testid);
test_consumer_poll("consume.up", rk, testid, -1, 0, msgcnt/2, &mv);
err = rd_kafka_assignment(rk, &assignment);
TEST_ASSERT(!err, "assignment() failed: %s", rd_kafka_err2str(err));
TEST_ASSERT(assignment->cnt > 0, "empty assignment");
TEST_SAY("Bringing down the network\n");
TEST_LOCK();
simulate_network_down = 1;
TEST_UNLOCK();
test_socket_close_all(test_curr, 1/*reinit*/);
TEST_SAY("Waiting for session timeout to expire (6s), and then some\n");
/* Commit an offset, which should fail, to trigger the offset commit
* callback fallback (CONSUMER_ERR) */
assignment->elems[0].offset = 123456789;
TEST_SAY("Committing offsets while down, should fail eventually\n");
err = rd_kafka_commit(rk, assignment, 1/*async*/);
TEST_ASSERT(!err, "async commit failed: %s", rd_kafka_err2str(err));
rd_kafka_topic_partition_list_destroy(assignment);
rd_sleep(10);
TEST_SAY("Bringing network back up\n");
TEST_LOCK();
simulate_network_down = 0;
TEST_UNLOCK();
TEST_SAY("Continuing to consume..\n");
test_consumer_poll("consume.up2", rk, testid, -1, msgcnt/2, msgcnt/2,
&mv);
test_msgver_verify("consume", &mv, TEST_MSGVER_ORDER|TEST_MSGVER_DUP,
0, msgcnt);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
return 0;
}
int main_0002_unkpart (int argc, char **argv) {
int partition = 99; /* non-existent */
int r;
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char msg[128];
int msgcnt = 10;
int i;
int fails = 0;
const struct rd_kafka_metadata *metadata;
test_conf_init(&conf, &topic_conf, 10);
/* Set delivery report callback */
rd_kafka_conf_set_dr_cb(conf, dr_cb);
/* Create kafka instance */
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
rkt = rd_kafka_topic_new(rk, test_mk_topic_name("0002", 0),
topic_conf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
rd_strerror(errno));
/* Request metadata so that we know the cluster is up before producing
* messages, otherwise erroneous partitions will not fail immediately.*/
if ((r = rd_kafka_metadata(rk, 0, rkt, &metadata,
tmout_multip(15000))) !=
RD_KAFKA_RESP_ERR_NO_ERROR)
TEST_FAIL("Failed to acquire metadata: %s\n",
rd_kafka_err2str(r));
rd_kafka_metadata_destroy(metadata);
/* Produce a message */
for (i = 0 ; i < msgcnt ; i++) {
int *msgidp = malloc(sizeof(*msgidp));
*msgidp = i;
rd_snprintf(msg, sizeof(msg), "%s test message #%i", argv[0], i);
r = rd_kafka_produce(rkt, partition, RD_KAFKA_MSG_F_COPY,
msg, strlen(msg), NULL, 0, msgidp);
if (r == -1) {
if (errno == ESRCH)
TEST_SAY("Failed to produce message #%i: "
"unknown partition: good!\n", i);
else
TEST_FAIL("Failed to produce message #%i: %s\n",
i, rd_kafka_err2str(
rd_kafka_errno2err(errno)));
free(msgidp);
} else {
if (i > 5) {
fails++;
TEST_SAY("Message #%i produced: "
"should've failed\n", i);
}
msgs_wait |= (1 << i);
}
/* After half the messages: sleep to allow the metadata
* to be fetched from broker and update the actual partition
* count: this will make subsequent produce() calls fail
* immediately. */
if (i == 5)
rd_sleep(2);
}
/* Wait for messages to time out */
rd_kafka_flush(rk, -1);
if (msgs_wait != 0)
TEST_FAIL("Still waiting for messages: 0x%x\n", msgs_wait);
if (fails > 0)
TEST_FAIL("See previous error(s)\n");
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy rdkafka instance */
TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
rd_kafka_destroy(rk);
return 0;
}
/**
* @brief Test that Metadata requests are retried properly when
* timing out due to high broker rtt.
*/
static void do_test_low_socket_timeout (const char *topic) {
rd_kafka_t *rk;
rd_kafka_conf_t *conf;
rd_kafka_topic_t *rkt;
rd_kafka_resp_err_t err;
const struct rd_kafka_metadata *md;
int res;
mtx_init(&ctrl.lock, mtx_plain);
cnd_init(&ctrl.cnd);
TEST_SAY("Test Metadata request retries on timeout\n");
test_conf_init(&conf, NULL, 60);
test_conf_set(conf, "socket.timeout.ms", "1000");
test_conf_set(conf, "socket.max.fails", "12345");
test_conf_set(conf, "retry.backoff.ms", "5000");
/* Avoid api version requests (with their own timeout) to get in
* the way of our test */
test_conf_set(conf, "api.version.request", "false");
test_socket_enable(conf);
test_curr->connect_cb = connect_cb;
test_curr->is_fatal_cb = is_fatal_cb;
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
rkt = test_create_producer_topic(rk, topic, NULL);
TEST_SAY("Waiting for sockem connect..\n");
mtx_lock(&ctrl.lock);
while (!ctrl.skm)
cnd_wait(&ctrl.cnd, &ctrl.lock);
mtx_unlock(&ctrl.lock);
TEST_SAY("Connected, fire off a undelayed metadata() to "
"make sure connection is up\n");
err = rd_kafka_metadata(rk, 0, rkt, &md, tmout_multip(2000));
TEST_ASSERT(!err, "metadata(undelayed) failed: %s",
rd_kafka_err2str(err));
rd_kafka_metadata_destroy(md);
if (thrd_create(&ctrl.thrd, ctrl_thrd_main, NULL) != thrd_success)
TEST_FAIL("Failed to create sockem ctrl thread");
set_delay(0, 3000); /* Takes effect immediately */
/* After two retries, remove the delay, the third retry
* should kick in and work. */
set_delay(((1000 /*socket.timeout.ms*/ +
5000 /*retry.backoff.ms*/) * 2) - 2000, 0);
TEST_SAY("Calling metadata() again which should succeed after "
"3 internal retries\n");
/* Metadata should be returned after the third retry */
err = rd_kafka_metadata(rk, 0, rkt, &md,
((1000 /*socket.timeout.ms*/ +
5000 /*retry.backoff.ms*/) * 2) + 5000);
TEST_SAY("metadata() returned %s\n", rd_kafka_err2str(err));
TEST_ASSERT(!err, "metadata(undelayed) failed: %s",
rd_kafka_err2str(err));
rd_kafka_metadata_destroy(md);
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
/* Join controller thread */
mtx_lock(&ctrl.lock);
ctrl.term = 1;
mtx_unlock(&ctrl.lock);
thrd_join(ctrl.thrd, &res);
cnd_destroy(&ctrl.cnd);
mtx_destroy(&ctrl.lock);
}
int main_0017_compression(int argc, char **argv) {
rd_kafka_t *rk_p, *rk_c;
const int msg_cnt = 1000;
int msg_base = 0;
uint64_t testid;
#define CODEC_CNT 4
const char *codecs[CODEC_CNT+1] = {
"none",
#if WITH_ZLIB
"gzip",
#endif
#if WITH_SNAPPY
"snappy",
#endif
#if WITH_LZ4
"lz4",
#endif
NULL
};
const char *topics[CODEC_CNT];
const int32_t partition = 0;
int i;
testid = test_id_generate();
/* Produce messages */
rk_p = test_create_producer();
for (i = 0; codecs[i] != NULL ; i++) {
rd_kafka_topic_t *rkt_p;
topics[i] = test_mk_topic_name(codecs[i], 1);
TEST_SAY("Produce %d messages with %s compression to "
"topic %s\n",
msg_cnt, codecs[i], topics[i]);
rkt_p = test_create_producer_topic(rk_p, topics[i],
"compression.codec", codecs[i], NULL);
/* Produce small message that will not decrease with
* compression (issue #781) */
test_produce_msgs(rk_p, rkt_p, testid, partition,
msg_base + (partition*msg_cnt), 1,
NULL, 5);
/* Produce standard sized messages */
test_produce_msgs(rk_p, rkt_p, testid, partition,
msg_base + (partition*msg_cnt) + 1, msg_cnt-1,
NULL, 512);
rd_kafka_topic_destroy(rkt_p);
}
rd_kafka_destroy(rk_p);
/* Consume messages */
rk_c = test_create_consumer(NULL, NULL, NULL, NULL);
for (i = 0; codecs[i] != NULL ; i++) {
rd_kafka_topic_t *rkt_c = rd_kafka_topic_new(rk_c,
topics[i], NULL);
TEST_SAY("Consume %d messages from topic %s\n",
msg_cnt, topics[i]);
/* Start consuming */
test_consumer_start(codecs[i], rkt_c, partition,
RD_KAFKA_OFFSET_BEGINNING);
/* Consume messages */
test_consume_msgs(codecs[i], rkt_c, testid, partition,
/* Use offset 0 here, which is wrong, should
* be TEST_NO_SEEK, but it exposed a bug
* where the Offset query was postponed
* till after the seek, causing messages
* to be replayed. */
0,
msg_base, msg_cnt, 1 /* parse format */);
test_consumer_stop(codecs[i], rkt_c, partition);
rd_kafka_topic_destroy(rkt_c);
}
rd_kafka_destroy(rk_c);
return 0;
}
int main_0029_assign_offset (int argc, char **argv) {
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_topic_conf_t *tconf;
rd_kafka_topic_partition_list_t *parts;
uint64_t testid;
int i;
test_timing_t t_simple, t_hl;
test_msgver_t mv;
test_conf_init(NULL, &tconf, 20 + (test_session_timeout_ms * 3 / 1000));
test_topic_conf_set(tconf, "auto.offset.reset", "smallest");
/* Produce X messages to Y partitions so we get a
* nice seekable 0..X offset one each partition. */
/* Produce messages */
testid = test_id_generate();
rk = test_create_producer();
rkt = test_create_producer_topic(rk, topic, NULL);
parts = rd_kafka_topic_partition_list_new(partitions);
for (i = 0 ; i < partitions ; i++) {
test_produce_msgs(rk, rkt, testid, i, 0, msgcnt, NULL, 0);
rd_kafka_topic_partition_list_add(parts, topic, i)->offset =
msgcnt / 2;
}
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
/* Simple consumer */
TIMING_START(&t_simple, "SIMPLE.CONSUMER");
rk = test_create_consumer(topic, NULL, NULL, NULL);
test_msgver_init(&mv, testid);
test_consumer_assign("SIMPLE.ASSIGN", rk, parts);
test_consumer_poll("SIMPLE.CONSUME", rk, testid, -1, 0,
partitions * (msgcnt / 2), &mv);
for (i = 0 ; i < partitions ; i++)
test_msgver_verify_part("HL.MSGS", &mv, TEST_MSGVER_ALL_PART,
topic, i, msgcnt/2, msgcnt/2);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_simple);
rd_kafka_topic_partition_list_destroy(parts);
/* High-level consumer.
* Offsets are set in rebalance callback. */
TIMING_START(&t_hl, "HL.CONSUMER");
test_msgver_init(&mv, testid);
rk = test_create_consumer(topic, rebalance_cb, NULL, NULL);
test_consumer_subscribe(rk, topic);
test_consumer_poll("HL.CONSUME", rk, testid, -1, 0,
partitions * (msgcnt / 2), &mv);
for (i = 0 ; i < partitions ; i++)
test_msgver_verify_part("HL.MSGS", &mv, TEST_MSGVER_ALL_PART,
topic, i, msgcnt/2, msgcnt/2);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_hl);
rd_kafka_topic_conf_destroy(tconf);
return 0;
}
static void consume_messages_with_queues (uint64_t testid, const char *topic,
int partition_cnt, int msgcnt) {
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
rd_kafka_queue_t *rkqu;
int i;
int32_t partition;
int batch_cnt = msgcnt / partition_cnt;
test_conf_init(&conf, &topic_conf, 20);
/* Create kafka instance */
rk = test_create_handle(RD_KAFKA_CONSUMER, conf);
/* Create queue */
rkqu = rd_kafka_queue_new(rk);
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
rd_kafka_err2str(rd_kafka_last_error()));
TEST_SAY("Consuming %i messages from one queue serving %i partitions\n",
msgcnt, partition_cnt);
/* Start consuming each partition */
for (partition = 0 ; partition < partition_cnt ; partition++) {
/* Consume messages */
TEST_SAY("Start consuming partition %i at tail offset -%i\n",
partition, batch_cnt);
if (rd_kafka_consume_start_queue(rkt, partition,
RD_KAFKA_OFFSET_TAIL(batch_cnt),
rkqu) == -1)
TEST_FAIL("consume_start_queue(%i) failed: %s",
(int)partition,
rd_kafka_err2str(rd_kafka_last_error()));
}
/* Consume messages from queue */
for (i = 0 ; i < msgcnt ; i++) {
rd_kafka_message_t *rkmessage;
rkmessage = rd_kafka_consume_queue(rkqu, tmout_multip(5000));
if (!rkmessage)
TEST_FAIL("Failed to consume message %i/%i from "
"queue: %s",
i, msgcnt,
rd_kafka_err2str(rd_kafka_last_error()));
if (rkmessage->err)
TEST_FAIL("Consume message %i/%i from queue "
"has error (partition %"PRId32"): %s",
i, msgcnt,
rkmessage->partition,
rd_kafka_err2str(rkmessage->err));
verify_consumed_msg(testid, -1, -1, rkmessage);
rd_kafka_message_destroy(rkmessage);
}
/* Stop consuming each partition */
for (partition = 0 ; partition < partition_cnt ; partition++)
rd_kafka_consume_stop(rkt, partition);
/* Destroy queue */
rd_kafka_queue_destroy(rkqu);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy rdkafka instance */
TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
rd_kafka_destroy(rk);
}
static void do_test_regex (void) {
char *base_topic = rd_strdup(test_mk_topic_name("topic", 1));
char *topic_b = rd_strdup(tsprintf("%s_b", base_topic));
char *topic_c = rd_strdup(tsprintf("%s_c", base_topic));
char *topic_d = rd_strdup(tsprintf("%s_d", base_topic));
char *topic_e = rd_strdup(tsprintf("%s_e", base_topic));
rd_kafka_t *rk;
rd_kafka_conf_t *conf;
rd_kafka_queue_t *queue;
/**
* Regex test:
* - Create topic b
* - Subscribe to b & d & e
* - Verify b assignment
* - Create topic c
* - Verify no rebalance
* - Create topic d
* - Verify b & d assignment
*/
TEST_SAY("Regex testing\n");
test_conf_init(&conf, NULL, 60);
/* Decrease metadata interval to speed up topic change discovery. */
test_conf_set(conf, "metadata.max.age.ms", "5000");
rd_kafka_conf_set_events(conf, RD_KAFKA_EVENT_REBALANCE);
rk = test_create_consumer(test_str_id_generate_tmp(),
NULL, conf, NULL, NULL);
queue = rd_kafka_queue_get_consumer(rk);
TEST_SAY("Regex: creating topic %s (subscribed)\n", topic_b);
test_create_topic(topic_b, 2, 1);
rd_sleep(1); // FIXME: do check&wait loop instead
TEST_SAY("Regex: Subscribing to %s & %s & %s\n",
topic_b, topic_d, topic_e);
test_consumer_subscribe(rk, tsprintf("^%s_[bde]$", base_topic));
await_assignment("Regex: just one topic exists", rk, queue, 1,
topic_b, 2);
TEST_SAY("Regex: creating topic %s (not subscribed)\n", topic_c);
test_create_topic(topic_c, 4, 1);
/* Should not see a rebalance since no topics are matched. */
await_no_rebalance("Regex: empty", rk, queue, 10000);
TEST_SAY("Regex: creating topic %s (subscribed)\n", topic_d);
test_create_topic(topic_d, 1, 1);
await_revoke("Regex: rebalance after topic creation", rk, queue);
await_assignment("Regex: two topics exist", rk, queue, 2,
topic_b, 2,
topic_d, 1);
test_consumer_close(rk);
rd_kafka_queue_destroy(queue);
rd_kafka_destroy(rk);
rd_free(base_topic);
rd_free(topic_b);
rd_free(topic_c);
rd_free(topic_d);
rd_free(topic_e);
}
/**
* Produces 'msgcnt' messages split over 'partition_cnt' partitions.
*/
static void produce_null_messages (uint64_t testid, const char *topic,
int partition_cnt, int msgcnt) {
int r;
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char errstr[512];
int i;
int32_t partition;
int msgid = 0;
test_conf_init(&conf, &topic_conf, 20);
rd_kafka_conf_set_dr_cb(conf, dr_cb);
/* Make sure all replicas are in-sync after producing
* so that consume test wont fail. */
rd_kafka_topic_conf_set(topic_conf, "request.required.acks", "-1",
errstr, sizeof(errstr));
/* Create kafka instance */
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
rd_kafka_err2str(rd_kafka_last_error()));
/* Produce messages */
prod_msg_remains = msgcnt;
for (partition = 0 ; partition < partition_cnt ; partition++) {
int batch_cnt = msgcnt / partition_cnt;
for (i = 0 ; i < batch_cnt ; i++) {
char key[128];
rd_snprintf(key, sizeof(key),
"testid=%"PRIu64", partition=%i, msg=%i",
testid, (int)partition, msgid);
r = rd_kafka_produce(rkt, partition, 0,
NULL, 0,
key, strlen(key),
NULL);
if (r == -1)
TEST_FAIL("Failed to produce message %i "
"to partition %i: %s",
msgid, (int)partition,
rd_kafka_err2str(rd_kafka_last_error()));
msgid++;
}
}
TEST_SAY("Produced %d messages to %d partition(s), "
"waiting for deliveries\n", msgcnt, partition_cnt);
/* Wait for messages to be delivered */
while (rd_kafka_outq_len(rk) > 0)
rd_kafka_poll(rk, 100);
if (fails)
TEST_FAIL("%i failures, see previous errors", fails);
if (prod_msg_remains != 0)
TEST_FAIL("Still waiting for %i messages to be produced",
prod_msg_remains);
else
TEST_SAY("All messages delivered\n");
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy rdkafka instance */
TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
rd_kafka_destroy(rk);
}
kafka_consumer_::~kafka_consumer_() {
rd_kafka_consume_stop(rkt_, partition_);
rd_kafka_topic_destroy(rkt_);
rd_kafka_destroy(rk_);
}
int main (int argc, char **argv) {
rd_kafka_t *rk;
char *broker = NULL;
char mode = 'C';
char *topic = NULL;
int partition = 0;
int opt;
int msgsize = 1024;
int msgcnt = -1;
int sendflags = 0;
int dispintvl = 1000;
struct {
rd_ts_t t_start;
rd_ts_t t_end;
rd_ts_t t_end_send;
uint64_t msgs;
uint64_t bytes;
rd_ts_t t_latency;
rd_ts_t t_last;
rd_ts_t t_total;
} cnt = {};
rd_ts_t now;
char *dirstr = "";
while ((opt = getopt(argc, argv, "PCt:p:b:s:c:fi:D")) != -1) {
switch (opt) {
case 'P':
case 'C':
mode = opt;
break;
case 't':
topic = optarg;
break;
case 'p':
partition = atoi(optarg);
break;
case 'b':
broker = optarg;
break;
case 's':
msgsize = atoi(optarg);
break;
case 'c':
msgcnt = atoi(optarg);
break;
case 'D':
sendflags |= RD_KAFKA_OP_F_FREE;
break;
case 'i':
dispintvl = atoi(optarg);
break;
default:
goto usage;
}
}
if (!topic || optind != argc) {
usage:
fprintf(stderr,
"Usage: %s [-C|-P] -t <topic> "
"[-p <partition>] [-b <broker>] [options..]\n"
"\n"
" Options:\n"
" -C | -P Consumer or Producer mode\n"
" -t <topic> Topic to fetch / produce\n"
" -p <num> Partition (defaults to 0)\n"
" -b <broker> Broker address (localhost:9092)\n"
" -s <size> Message size (producer)\n"
" -c <cnt> Messages to transmit/receive\n"
" -D Copy/Duplicate data buffer (producer)\n"
" -i <ms> Display interval\n"
"\n"
" In Consumer mode:\n"
" consumes messages and prints thruput\n"
" In Producer mode:\n"
" writes messages of size -s <..> and prints thruput\n"
"\n",
argv[0]);
exit(1);
}
dispintvl *= 1000; /* us */
signal(SIGINT, stop);
/* Socket hangups are gracefully handled in librdkafka on socket error
* without the use of signals, so SIGPIPE should be ignored by the
* calling program. */
signal(SIGPIPE, SIG_IGN);
if (mode == 'P') {
/*
* Producer
*/
char *sbuf = malloc(msgsize);
int endwait;
int outq;
int i;
memset(sbuf, 'R', msgsize);
if (msgcnt == -1)
printf("%% Sending messages of size %i bytes\n",
msgsize);
else
printf("%% Sending %i messages of size %i bytes\n",
msgcnt ,msgsize);
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, broker, NULL))) {
perror("kafka_new producer");
exit(1);
}
cnt.t_start = rd_clock();
while (run && (msgcnt == -1 || cnt.msgs < msgcnt)) {
char *pbuf = sbuf;
/* Send/Produce message. */
if (sendflags & RD_KAFKA_OP_F_FREE) {
/* Duplicate memory */
pbuf = malloc(msgsize);
memcpy(pbuf, sbuf, msgsize);
}
rd_kafka_produce(rk, topic, partition,
sendflags, pbuf, msgsize);
cnt.msgs++;
cnt.bytes += msgsize;
now = rd_clock();
if (cnt.t_last + dispintvl <= now) {
printf("%% %"PRIu64" messages and %"PRIu64
"bytes: %"PRIu64" msgs/s and "
"%.2f Mb/s\n",
cnt.msgs, cnt.bytes,
(cnt.msgs / (now - cnt.t_start)) *
1000000,
(float)(cnt.bytes /
(now - cnt.t_start)));
cnt.t_last = now;
}
}
/* Wait for messaging to finish. */
i = 0;
while (run && rd_kafka_outq_len(rk) > 0) {
if (!(i++ % (dispintvl/1000)))
printf("%% Waiting for %i messages in outq "
"to be sent. Abort with Ctrl-c\n",
rd_kafka_outq_len(rk));
usleep(1000);
}
cnt.t_end_send = rd_clock();
outq = rd_kafka_outq_len(rk);
cnt.msgs -= outq;
cnt.bytes -= msgsize * outq;
cnt.t_end = rd_clock();
/* Since there is no ack for produce messages in 0.7
* we wait some more for any packets in the socket buffers
* to be sent.
* This is fixed in protocol version 0.8 */
endwait = cnt.msgs * 10;
printf("%% Test timers stopped, but waiting %ims more "
"for the %"PRIu64 " messages to be transmitted from "
"socket buffers.\n"
"%% End with Ctrl-c\n",
endwait / 1000,
cnt.msgs);
run = 1;
while (run && endwait > 0) {
usleep(10000);
endwait -= 10000;
}
/* Destroy the handle */
rd_kafka_destroy(rk);
dirstr = "sent";
} else if (mode == 'C') {
/*
* Consumer
*/
rd_kafka_op_t *rko;
/* Base our configuration on the default config. */
rd_kafka_conf_t conf = rd_kafka_defaultconf;
/* The offset storage file is optional but its presence
* avoids starting all over from offset 0 again when
* the program restarts.
* ZooKeeper functionality will be implemented in future
* versions and then the offset will be stored there instead. */
conf.consumer.offset_file = "."; /* current directory */
/* Indicate to rdkafka that the application is responsible
* for storing the offset. This allows the application to
* succesfully handle a message before storing the offset.
* If this flag is not set rdkafka will store the offset
* just prior to returning the message from rd_kafka_consume().
*/
conf.flags |= RD_KAFKA_CONF_F_APP_OFFSET_STORE;
/* Tell rdkafka to (try to) maintain 10000 messages
* in its internal receive buffers. This is to avoid
* application -> rdkafka -> broker per-message ping-pong
* latency. */
conf.consumer.replyq_low_thres = 100000;
/* Use the consumer convenience function
* to create a Kafka handle. */
if (!(rk = rd_kafka_new_consumer(broker, topic,
(uint32_t)partition,
0, &conf))) {
perror("kafka_new_consumer");
exit(1);
}
cnt.t_start = rd_clock();
while (run && (msgcnt == -1 || msgcnt > cnt.msgs)) {
/* Fetch an "op" which is one of:
* - a kafka message (if rko_len>0 && rko_err==0)
* - an error (if rko_err)
*/
uint64_t latency;
latency = rd_clock();
if (!(rko = rd_kafka_consume(rk, 1000/*timeout ms*/)))
continue;
cnt.t_latency += rd_clock() - latency;
if (rko->rko_err)
fprintf(stderr, "%% Error: %.*s\n",
rko->rko_len, rko->rko_payload);
else if (rko->rko_len) {
cnt.msgs++;
cnt.bytes += rko->rko_len;
}
/* rko_offset contains the offset of the _next_
* message. We store it when we're done processing
* the current message. */
if (rko->rko_offset)
rd_kafka_offset_store(rk, rko->rko_offset);
/* Destroy the op */
rd_kafka_op_destroy(rk, rko);
now = rd_clock();
if (cnt.t_last + dispintvl <= now &&
cnt.t_start + 1000000 < now) {
printf("%% %"PRIu64" messages and %"PRIu64
" bytes: %"PRIu64" msgs/s and "
"%.2f Mb/s\n",
cnt.msgs, cnt.bytes,
(cnt.msgs / ((now - cnt.t_start)/1000))
* 1000,
(float)(cnt.bytes /
((now - cnt.t_start) / 1000)));
cnt.t_last = now;
}
}
cnt.t_end = rd_clock();
/* Destroy the handle */
rd_kafka_destroy(rk);
dirstr = "received";
}
if (cnt.t_end_send)
cnt.t_total = cnt.t_end_send - cnt.t_start;
else
cnt.t_total = cnt.t_end - cnt.t_start;
printf("%% %"PRIu64" messages and %"PRIu64" bytes "
"%s in %"PRIu64"ms: %"PRIu64" msgs/s and %.02f Mb/s\n",
cnt.msgs, cnt.bytes,
dirstr,
cnt.t_total / 1000,
(cnt.msgs / (cnt.t_total / 1000)) * 1000,
(float)(cnt.bytes / (cnt.t_total / 1000)));
if (cnt.t_latency)
printf("%% Average application fetch latency: %"PRIu64"us\n",
cnt.t_latency / cnt.msgs);
return 0;
}
int main_0038_performance (int argc, char **argv) {
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
const int partition = 0;
const int msgsize = 100;
uint64_t testid;
rd_kafka_conf_t *conf;
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
test_timing_t t_create, t_produce, t_consume;
int totsize = 1024*1024*128;
int msgcnt;
if (!strcmp(test_mode, "helgrind"))
totsize = 1024*1024*8; /* 8 meg, helgrind is slow. */
msgcnt = totsize / msgsize;
TEST_SAY("Producing %d messages of size %d to %s [%d]\n",
msgcnt, (int)msgsize, topic, partition);
testid = test_id_generate();
test_conf_init(&conf, NULL, 120);
rd_kafka_conf_set_dr_cb(conf, test_dr_cb);
test_conf_set(conf, "queue.buffering.max.messages", "10000000");
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
rkt = test_create_producer_topic(rk, topic, "acks", "1", NULL);
/* First produce one message to create the topic, etc, this might take
* a while and we dont want this to affect the throughput timing. */
TIMING_START(&t_create, "CREATE TOPIC");
test_produce_msgs(rk, rkt, testid, partition, 0, 1, NULL, msgsize);
TIMING_STOP(&t_create);
TIMING_START(&t_produce, "PRODUCE");
test_produce_msgs(rk, rkt, testid, partition, 1, msgcnt-1, NULL, msgsize);
TIMING_STOP(&t_produce);
TEST_SAY("Destroying producer\n");
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
TEST_SAY("Creating consumer\n");
test_conf_init(&conf, NULL, 120);
rk = test_create_consumer(NULL, NULL, conf, NULL, NULL);
rkt = rd_kafka_topic_new(rk, topic, NULL);
test_consumer_start("CONSUME", rkt, partition,
RD_KAFKA_OFFSET_BEGINNING);
TIMING_START(&t_consume, "CONSUME");
test_consume_msgs("CONSUME", rkt, testid, partition, TEST_NO_SEEK,
0, msgcnt, 1);
TIMING_STOP(&t_consume);
test_consumer_stop("CONSUME", rkt, partition);
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
TEST_REPORT("{ \"producer\": "
" { \"mb_per_sec\": %.2f, \"records_per_sec\": %.2f },"
" \"consumer\": "
"{ \"mb_per_sec\": %.2f, \"records_per_sec\": %.2f } "
"}",
(double)
(totsize/((double)TIMING_DURATION(&t_produce)/1000000.0f)) /
1000000.0f,
(float)
(msgcnt/((double)TIMING_DURATION(&t_produce)/1000000.0f)),
(double)
(totsize/((double)TIMING_DURATION(&t_consume)/1000000.0f)) /
1000000.0f,
(float)
(msgcnt/((double)TIMING_DURATION(&t_consume)/1000000.0f)));
return 0;
}
int main (int argc, char **argv) {
if (argc < 0 /* always false */) {
rd_kafka_version();
rd_kafka_version_str();
rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR);
rd_kafka_errno2err(EINVAL);
rd_kafka_conf_new();
rd_kafka_conf_destroy(NULL);
rd_kafka_conf_dup(NULL);
rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_conf_set_dr_cb(NULL, NULL);
rd_kafka_conf_set_dr_msg_cb(NULL, NULL);
rd_kafka_conf_set_error_cb(NULL, NULL);
rd_kafka_conf_set_stats_cb(NULL, NULL);
rd_kafka_conf_set_log_cb(NULL, NULL);
rd_kafka_conf_set_socket_cb(NULL, NULL);
rd_kafka_conf_set_open_cb(NULL, NULL);
rd_kafka_conf_set_opaque(NULL, NULL);
rd_kafka_opaque(NULL);
rd_kafka_conf_dump(NULL, NULL);
rd_kafka_topic_conf_dump(NULL, NULL);
rd_kafka_conf_dump_free(NULL, 0);
rd_kafka_conf_properties_show(NULL);
rd_kafka_topic_conf_new();
rd_kafka_topic_conf_dup(NULL);
rd_kafka_topic_conf_destroy(NULL);
rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_topic_conf_set_opaque(NULL, NULL);
rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL);
rd_kafka_topic_partition_available(NULL, 0);
rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_new(0, NULL, NULL, 0);
rd_kafka_destroy(NULL);
rd_kafka_name(NULL);
rd_kafka_topic_new(NULL, NULL, NULL);
rd_kafka_topic_destroy(NULL);
rd_kafka_topic_name(NULL);
rd_kafka_message_destroy(NULL);
rd_kafka_message_errstr(NULL);
rd_kafka_consume_start(NULL, 0, 0);
rd_kafka_consume_stop(NULL, 0);
rd_kafka_consume(NULL, 0, 0);
rd_kafka_consume_batch(NULL, 0, 0, NULL, 0);
rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL);
rd_kafka_offset_store(NULL, 0, 0);
rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL);
rd_kafka_poll(NULL, 0);
rd_kafka_brokers_add(NULL, NULL);
rd_kafka_set_logger(NULL, NULL);
rd_kafka_set_log_level(NULL, 0);
rd_kafka_log_print(NULL, 0, NULL, NULL);
rd_kafka_log_syslog(NULL, 0, NULL, NULL);
rd_kafka_outq_len(NULL);
rd_kafka_dump(NULL, NULL);
rd_kafka_thread_cnt();
rd_kafka_wait_destroyed(0);
rd_kafka_metadata(NULL, 0, NULL, NULL, 0);
rd_kafka_metadata_destroy(NULL);
}
return 0;
}
/* Produce a batch of messages to a single partition. */
static void test_single_partition (void) {
char *topic = "rdkafkatest1";
int partition = 0;
int r;
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char errstr[512];
char msg[128];
int msgcnt = 100000;
int failcnt;
int i;
rd_kafka_message_t *rkmessages;
msgid_next = 0;
test_conf_init(&conf, &topic_conf, 20);
/* Set delivery report callback */
rd_kafka_conf_set_dr_cb(conf, dr_single_partition_cb);
/* Create kafka instance */
rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr));
if (!rk)
TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr);
TEST_SAY("Created kafka instance %s\n", rd_kafka_name(rk));
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
strerror(errno));
/* Create messages */
rkmessages = calloc(sizeof(*rkmessages), msgcnt);
for (i = 0 ; i < msgcnt ; i++) {
int *msgidp = malloc(sizeof(*msgidp));
*msgidp = i;
snprintf(msg, sizeof(msg), "%s:%s test message #%i",
__FILE__, __FUNCTION__, i);
rkmessages[i].payload = strdup(msg);
rkmessages[i].len = strlen(msg);
rkmessages[i]._private = msgidp;
}
r = rd_kafka_produce_batch(rkt, partition, RD_KAFKA_MSG_F_FREE,
rkmessages, msgcnt);
/* Scan through messages to check for errors. */
for (i = 0 ; i < msgcnt ; i++) {
if (rkmessages[i].err) {
failcnt++;
if (failcnt < 100)
TEST_SAY("Message #%i failed: %s\n",
i,
rd_kafka_err2str(rkmessages[i].err));
}
}
/* All messages should've been produced. */
if (r < msgcnt) {
TEST_SAY("Not all messages were accepted "
"by produce_batch(): %i < %i\n", r, msgcnt);
if (msgcnt - r != failcnt)
TEST_SAY("Discrepency between failed messages (%i) "
"and return value %i (%i - %i)\n",
failcnt, msgcnt - r, msgcnt, r);
TEST_FAIL("%i/%i messages failed\n", msgcnt - r, msgcnt);
}
free(rkmessages);
TEST_SAY("Single partition: "
"Produced %i messages, waiting for deliveries\n", r);
/* Wait for messages to be delivered */
while (rd_kafka_outq_len(rk) > 0)
rd_kafka_poll(rk, 50);
if (fails)
TEST_FAIL("%i failures, see previous errors", fails);
if (msgid_next != msgcnt)
TEST_FAIL("Still waiting for messages: next %i != end %i\n",
msgid_next, msgcnt);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy rdkafka instance */
TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
rd_kafka_destroy(rk);
/* Wait for everything to be cleaned up since broker destroys are
* handled in its own thread. */
test_wait_exit(10);
/* If we havent failed at this point then
* there were no threads leaked */
return;
}
int main_0040_io_event (int argc, char **argv) {
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *tconf;
rd_kafka_t *rk_p, *rk_c;
const char *topic;
rd_kafka_topic_t *rkt_p;
rd_kafka_queue_t *queue;
uint64_t testid;
int msgcnt = 100;
int recvd = 0;
int fds[2];
int wait_multiplier = 1;
struct pollfd pfd;
int r;
enum {
_NOPE,
_YEP,
_REBALANCE
} expecting_io = _REBALANCE;
testid = test_id_generate();
topic = test_mk_topic_name(__FUNCTION__, 1);
rk_p = test_create_producer();
rkt_p = test_create_producer_topic(rk_p, topic, NULL);
test_auto_create_topic_rkt(rk_p, rkt_p);
test_conf_init(&conf, &tconf, 0);
rd_kafka_conf_set_events(conf, RD_KAFKA_EVENT_REBALANCE);
test_conf_set(conf, "session.timeout.ms", "6000");
test_conf_set(conf, "enable.partition.eof", "false");
/* Speed up propagation of new topics */
test_conf_set(conf, "metadata.max.age.ms", "5000");
test_topic_conf_set(tconf, "auto.offset.reset", "earliest");
rk_c = test_create_consumer(topic, NULL, conf, tconf);
queue = rd_kafka_queue_get_consumer(rk_c);
test_consumer_subscribe(rk_c, topic);
#ifndef _MSC_VER
r = pipe(fds);
#else
r = _pipe(fds, 2, _O_BINARY);
#endif
if (r == -1)
TEST_FAIL("pipe() failed: %s\n", strerror(errno));
rd_kafka_queue_io_event_enable(queue, fds[1], "1", 1);
pfd.fd = fds[0];
pfd.events = POLLIN;
pfd.revents = 0;
/**
* 1) Wait for rebalance event
* 2) Wait 1 interval (1s) expecting no IO (nothing produced).
* 3) Produce half the messages
* 4) Expect IO
* 5) Consume the available messages
* 6) Wait 1 interval expecting no IO.
* 7) Produce remaing half
* 8) Expect IO
* 9) Done.
*/
while (recvd < msgcnt) {
int r;
#ifndef _MSC_VER
r = poll(&pfd, 1, 1000 * wait_multiplier);
#else
r = WSAPoll(&pfd, 1, 1000 * wait_multiplier);
#endif
if (r == -1) {
TEST_FAIL("poll() failed: %s", strerror(errno));
} else if (r == 1) {
rd_kafka_event_t *rkev;
char b;
int eventcnt = 0;
if (pfd.events & POLLERR)
TEST_FAIL("Poll error\n");
if (!(pfd.events & POLLIN)) {
TEST_SAY("Stray event 0x%x\n", (int)pfd.events);
continue;
}
TEST_SAY("POLLIN\n");
/* Read signaling token to purge socket queue and
* eventually silence POLLIN */
#ifndef _MSC_VER
r = read(pfd.fd, &b, 1);
#else
r = _read((int)pfd.fd, &b, 1);
#endif
if (r == -1)
TEST_FAIL("read failed: %s\n", strerror(errno));
if (!expecting_io)
TEST_WARN("Got unexpected IO after %d/%d msgs\n",
recvd, msgcnt);
while ((rkev = rd_kafka_queue_poll(queue, 0))) {
eventcnt++;
switch (rd_kafka_event_type(rkev))
{
case RD_KAFKA_EVENT_REBALANCE:
TEST_SAY("Got %s: %s\n", rd_kafka_event_name(rkev),
rd_kafka_err2str(rd_kafka_event_error(rkev)));
if (expecting_io != _REBALANCE)
TEST_FAIL("Got Rebalance when expecting message\n");
if (rd_kafka_event_error(rkev) == RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS) {
rd_kafka_assign(rk_c, rd_kafka_event_topic_partition_list(rkev));
expecting_io = _NOPE;
} else
rd_kafka_assign(rk_c, NULL);
break;
case RD_KAFKA_EVENT_FETCH:
if (expecting_io != _YEP)
TEST_FAIL("Did not expect more messages at %d/%d\n",
recvd, msgcnt);
recvd++;
if (recvd == (msgcnt / 2) || recvd == msgcnt)
expecting_io = _NOPE;
break;
case RD_KAFKA_EVENT_ERROR:
TEST_FAIL("Error: %s\n", rd_kafka_event_error_string(rkev));
break;
default:
TEST_SAY("Ignoring event %s\n", rd_kafka_event_name(rkev));
}
rd_kafka_event_destroy(rkev);
}
TEST_SAY("%d events, Consumed %d/%d messages\n", eventcnt, recvd, msgcnt);
wait_multiplier = 1;
} else {
if (expecting_io == _REBALANCE) {
continue;
} else if (expecting_io == _YEP) {
TEST_FAIL("Did not see expected IO after %d/%d msgs\n",
recvd, msgcnt);
}
TEST_SAY("IO poll timeout (good)\n");
TEST_SAY("Got idle period, producing\n");
test_produce_msgs(rk_p, rkt_p, testid, 0, recvd, msgcnt/2,
NULL, 10);
expecting_io = _YEP;
/* When running slowly (e.g., valgrind) it might take
* some time before the first message is received
* after producing. */
wait_multiplier = 3;
}
}
TEST_SAY("Done\n");
rd_kafka_topic_destroy(rkt_p);
rd_kafka_destroy(rk_p);
rd_kafka_queue_destroy(queue);
rd_kafka_consumer_close(rk_c);
rd_kafka_destroy(rk_c);
#ifndef _MSC_VER
close(fds[0]);
close(fds[1]);
#else
_close(fds[0]);
_close(fds[1]);
#endif
return 0;
}
int main (int argc, char **argv) {
char *brokers = "localhost";
char mode = 'C';
char *topic = NULL;
const char *key = NULL;
int partition = RD_KAFKA_PARTITION_UA; /* random */
int opt;
int msgcnt = -1;
int sendflags = 0;
char *msgpattern = "librdkafka_performance testing!";
int msgsize = strlen(msgpattern);
const char *debug = NULL;
rd_ts_t now;
char errstr[512];
uint64_t seq = 0;
int seed = time(NULL);
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
const char *compression = "no";
int64_t start_offset = 0;
int batch_size = 0;
/* Kafka configuration */
conf = rd_kafka_conf_new();
rd_kafka_conf_set_error_cb(conf, err_cb);
rd_kafka_conf_set_dr_cb(conf, msg_delivered);
/* Producer config */
rd_kafka_conf_set(conf, "queue.buffering.max.messages", "500000",
NULL, 0);
rd_kafka_conf_set(conf, "message.send.max.retries", "3", NULL, 0);
rd_kafka_conf_set(conf, "retry.backoff.ms", "500", NULL, 0);
/* Consumer config */
/* Tell rdkafka to (try to) maintain 1M messages
* in its internal receive buffers. This is to avoid
* application -> rdkafka -> broker per-message ping-pong
* latency.
* The larger the local queue, the higher the performance.
* Try other values with: ... -X queued.min.messages=1000
*/
rd_kafka_conf_set(conf, "queued.min.messages", "1000000", NULL, 0);
/* Kafka topic configuration */
topic_conf = rd_kafka_topic_conf_new();
rd_kafka_topic_conf_set(topic_conf, "message.timeout.ms", "5000",
NULL, 0);
while ((opt =
getopt(argc, argv,
"PCt:p:b:s:k:c:fi:Dd:m:S:x:R:a:z:o:X:B:eT:q")) != -1) {
switch (opt) {
case 'P':
case 'C':
mode = opt;
break;
case 't':
topic = optarg;
break;
case 'p':
partition = atoi(optarg);
break;
case 'b':
brokers = optarg;
break;
case 's':
msgsize = atoi(optarg);
break;
case 'k':
key = optarg;
break;
case 'c':
msgcnt = atoi(optarg);
break;
case 'D':
sendflags |= RD_KAFKA_MSG_F_FREE;
break;
case 'i':
dispintvl = atoi(optarg);
break;
case 'm':
msgpattern = optarg;
break;
case 'S':
seq = strtoull(optarg, NULL, 10);
do_seq = 1;
break;
case 'x':
exit_after = atoi(optarg);
break;
case 'R':
seed = atoi(optarg);
break;
case 'a':
if (rd_kafka_topic_conf_set(topic_conf,
"request.required.acks",
optarg,
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
break;
case 'B':
batch_size = atoi(optarg);
break;
case 'z':
if (rd_kafka_conf_set(conf, "compression.codec",
optarg,
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
compression = optarg;
break;
case 'o':
start_offset = strtoll(optarg, NULL, 10);
break;
case 'e':
exit_eof = 1;
break;
case 'd':
debug = optarg;
break;
case 'X':
{
char *name, *val;
rd_kafka_conf_res_t res;
if (!strcmp(optarg, "list") ||
!strcmp(optarg, "help")) {
rd_kafka_conf_properties_show(stdout);
exit(0);
}
name = optarg;
if (!(val = strchr(name, '='))) {
fprintf(stderr, "%% Expected "
"-X property=value, not %s\n", name);
exit(1);
}
*val = '\0';
val++;
res = RD_KAFKA_CONF_UNKNOWN;
/* Try "topic." prefixed properties on topic
* conf first, and then fall through to global if
* it didnt match a topic configuration property. */
if (!strncmp(name, "topic.", strlen("topic.")))
res = rd_kafka_topic_conf_set(topic_conf,
name+
strlen("topic"),
val,
errstr,
sizeof(errstr));
if (res == RD_KAFKA_CONF_UNKNOWN)
res = rd_kafka_conf_set(conf, name, val,
errstr, sizeof(errstr));
if (res != RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
}
break;
case 'T':
if (rd_kafka_conf_set(conf, "statistics.interval.ms",
optarg, errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
rd_kafka_conf_set_stats_cb(conf, stats_cb);
break;
case 'q':
quiet = 1;
break;
default:
goto usage;
}
}
if (!topic || optind != argc) {
usage:
fprintf(stderr,
"Usage: %s [-C|-P] -t <topic> "
"[-p <partition>] [-b <broker,broker..>] [options..]\n"
"\n"
" Options:\n"
" -C | -P Consumer or Producer mode\n"
" -t <topic> Topic to fetch / produce\n"
" -p <num> Partition (defaults to random)\n"
" -b <brokers> Broker address list (host[:port],..)\n"
" -s <size> Message size (producer)\n"
" -k <key> Message key (producer)\n"
" -c <cnt> Messages to transmit/receive\n"
" -D Copy/Duplicate data buffer (producer)\n"
" -i <ms> Display interval\n"
" -m <msg> Message payload pattern\n"
" -S <start> Send a sequence number starting at "
"<start> as payload\n"
" -R <seed> Random seed value (defaults to time)\n"
" -a <acks> Required acks (producer): "
"-1, 0, 1, >1\n"
" -B <size> Consume batch size (# of msgs)\n"
" -z <codec> Enable compression:\n"
" none|gzip|snappy\n"
" -o <offset> Start offset (consumer)\n"
" -d [facs..] Enable debugging contexts:\n"
" %s\n"
" -X <prop=name> Set arbitrary librdkafka "
"configuration property\n"
" Properties prefixed with \"topic.\" "
"will be set on topic object.\n"
" Use '-X list' to see the full list\n"
" of supported properties.\n"
" -T <intvl> Enable statistics from librdkafka at "
"specified interval (ms)\n"
" -q Be more quiet\n"
"\n"
" In Consumer mode:\n"
" consumes messages and prints thruput\n"
" If -B <..> is supplied the batch consumer\n"
" mode is used, else the callback mode is used.\n"
"\n"
" In Producer mode:\n"
" writes messages of size -s <..> and prints thruput\n"
"\n",
argv[0],
RD_KAFKA_DEBUG_CONTEXTS);
exit(1);
}
dispintvl *= 1000; /* us */
printf("%% Using random seed %i\n", seed);
srand(seed);
signal(SIGINT, stop);
signal(SIGUSR1, sig_usr1);
if (debug &&
rd_kafka_conf_set(conf, "debug", debug, errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
printf("%% Debug configuration failed: %s: %s\n",
errstr, debug);
exit(1);
}
/* Socket hangups are gracefully handled in librdkafka on socket error
* without the use of signals, so SIGPIPE should be ignored by the
* calling program. */
signal(SIGPIPE, SIG_IGN);
if (msgcnt != -1)
forever = 0;
if (mode == 'P') {
/*
* Producer
*/
char *sbuf;
char *pbuf;
int outq;
int i;
int keylen = key ? strlen(key) : 0;
off_t rof = 0;
size_t plen = strlen(msgpattern);
if (do_seq) {
if (msgsize < strlen("18446744073709551615: ")+1)
msgsize = strlen("18446744073709551615: ")+1;
/* Force duplication of payload */
sendflags |= RD_KAFKA_MSG_F_FREE;
}
sbuf = malloc(msgsize);
/* Copy payload content to new buffer */
while (rof < msgsize) {
size_t xlen = RD_MIN(msgsize-rof, plen);
memcpy(sbuf+rof, msgpattern, xlen);
rof += xlen;
}
if (msgcnt == -1)
printf("%% Sending messages of size %i bytes\n",
msgsize);
else
printf("%% Sending %i messages of size %i bytes\n",
msgcnt, msgsize);
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create Kafka producer: %s\n",
errstr);
exit(1);
}
if (debug)
rd_kafka_set_log_level(rk, 7);
/* Add broker(s) */
if (rd_kafka_brokers_add(rk, brokers) < 1) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Explicitly create topic to avoid per-msg lookups. */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
cnt.t_start = rd_clock();
while (run && (msgcnt == -1 || cnt.msgs < msgcnt)) {
/* Send/Produce message. */
if (do_seq) {
snprintf(sbuf, msgsize-1, "%"PRIu64": ", seq);
seq++;
}
if (sendflags & RD_KAFKA_MSG_F_FREE) {
/* Duplicate memory */
pbuf = malloc(msgsize);
memcpy(pbuf, sbuf, msgsize);
} else
pbuf = sbuf;
cnt.tx++;
while (run &&
rd_kafka_produce(rkt, partition,
sendflags, pbuf, msgsize,
key, keylen, NULL) == -1) {
if (!quiet || errno != ENOBUFS)
printf("produce error: %s%s\n",
strerror(errno),
errno == ENOBUFS ?
" (backpressure)":"");
cnt.tx_err++;
if (errno != ENOBUFS) {
run = 0;
break;
}
now = rd_clock();
if (cnt.t_last + dispintvl <= now) {
printf("%% Backpressure %i "
"(tx %"PRIu64", "
"txerr %"PRIu64")\n",
rd_kafka_outq_len(rk),
cnt.tx, cnt.tx_err);
cnt.t_last = now;
}
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 10);
}
msgs_wait_cnt++;
cnt.msgs++;
cnt.bytes += msgsize;
print_stats(mode, 0, compression);
/* Must poll to handle delivery reports */
rd_kafka_poll(rk, 0);
}
forever = 0;
printf("All messages produced, "
"now waiting for %li deliveries\n",
msgs_wait_cnt);
rd_kafka_dump(stdout, rk);
/* Wait for messages to be delivered */
i = 0;
while (run && rd_kafka_poll(rk, 1000) != -1) {
if (!(i++ % (dispintvl/1000)))
printf("%% Waiting for %li, "
"%i messages in outq "
"to be sent. Abort with Ctrl-c\n",
msgs_wait_cnt,
rd_kafka_outq_len(rk));
}
outq = rd_kafka_outq_len(rk);
printf("%% %i messages in outq\n", outq);
cnt.msgs -= outq;
cnt.bytes -= msgsize * outq;
cnt.t_end = t_end;
if (cnt.tx_err > 0)
printf("%% %"PRIu64" backpressures for %"PRIu64
" produce calls: %.3f%% backpressure rate\n",
cnt.tx_err, cnt.tx,
((double)cnt.tx_err / (double)cnt.tx) * 100.0);
rd_kafka_dump(stdout, rk);
/* Destroy the handle */
rd_kafka_destroy(rk);
} else if (mode == 'C') {
/*
* Consumer
*/
rd_kafka_message_t **rkmessages = NULL;
#if 0 /* Future API */
/* The offset storage file is optional but its presence
* avoids starting all over from offset 0 again when
* the program restarts.
* ZooKeeper functionality will be implemented in future
* versions and then the offset will be stored there instead. */
conf.consumer.offset_file = "."; /* current directory */
/* Indicate to rdkafka that the application is responsible
* for storing the offset. This allows the application to
* successfully handle a message before storing the offset.
* If this flag is not set rdkafka will store the offset
* just prior to returning the message from rd_kafka_consume().
*/
conf.flags |= RD_KAFKA_CONF_F_APP_OFFSET_STORE;
#endif
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create Kafka producer: %s\n",
errstr);
exit(1);
}
if (debug)
rd_kafka_set_log_level(rk, 7);
/* Add broker(s) */
if (rd_kafka_brokers_add(rk, brokers) < 1) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Create topic to consume from */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
/* Batch consumer */
if (batch_size)
rkmessages = malloc(sizeof(*rkmessages) * batch_size);
/* Start consuming */
if (rd_kafka_consume_start(rkt, partition, start_offset) == -1){
fprintf(stderr, "%% Failed to start consuming: %s\n",
strerror(errno));
exit(1);
}
cnt.t_start = rd_clock();
while (run && (msgcnt == -1 || msgcnt > cnt.msgs)) {
/* Consume messages.
* A message may either be a real message, or
* an error signaling (if rkmessage->err is set).
*/
uint64_t latency;
int r;
latency = rd_clock();
if (batch_size) {
int i;
/* Batch fetch mode */
r = rd_kafka_consume_batch(rkt, partition,
1000,
rkmessages,
batch_size);
if (r != -1) {
for (i = 0 ; i < r ; i++) {
msg_consume(rkmessages[i],NULL);
rd_kafka_message_destroy(
rkmessages[i]);
}
}
} else {
/* Callback mode */
r = rd_kafka_consume_callback(rkt, partition,
1000/*timeout*/,
msg_consume,
NULL);
}
cnt.t_latency += rd_clock() - latency;
if (r == -1)
fprintf(stderr, "%% Error: %s\n",
strerror(errno));
print_stats(mode, 0, compression);
/* Poll to handle stats callbacks */
rd_kafka_poll(rk, 0);
}
cnt.t_end = rd_clock();
/* Stop consuming */
rd_kafka_consume_stop(rkt, partition);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
if (batch_size)
free(rkmessages);
/* Destroy the handle */
rd_kafka_destroy(rk);
}
print_stats(mode, 1, compression);
if (cnt.t_latency && cnt.msgs)
printf("%% Average application fetch latency: %"PRIu64"us\n",
cnt.t_latency / cnt.msgs);
/* Let background threads clean up and terminate cleanly. */
rd_kafka_wait_destroyed(2000);
return 0;
}
int main (int argc, char **argv) {
rd_kafka_topic_t *rkt;
char *brokers = "localhost:9092";
char mode = 'C';
char *topic = NULL;
int partition = RD_KAFKA_PARTITION_UA;
int opt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char errstr[512];
const char *debug = NULL;
int64_t start_offset = 0;
int report_offsets = 0;
int do_conf_dump = 0;
quiet = !isatty(STDIN_FILENO);
/* Kafka configuration */
conf = rd_kafka_conf_new();
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
while ((opt = getopt(argc, argv, "PCLt:p:b:z:qd:o:eX:A")) != -1) {
switch (opt) {
case 'P':
case 'C':
case 'L':
mode = opt;
break;
case 't':
topic = optarg;
break;
case 'p':
partition = atoi(optarg);
break;
case 'b':
brokers = optarg;
break;
case 'z':
if (rd_kafka_conf_set(conf, "compression.codec",
optarg,
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
break;
case 'o':
if (!strcmp(optarg, "end"))
start_offset = RD_KAFKA_OFFSET_END;
else if (!strcmp(optarg, "beginning"))
start_offset = RD_KAFKA_OFFSET_BEGINNING;
else if (!strcmp(optarg, "stored"))
start_offset = RD_KAFKA_OFFSET_STORED;
else if (!strcmp(optarg, "report"))
report_offsets = 1;
else {
start_offset = strtoll(optarg, NULL, 10);
if (start_offset < 0)
start_offset = RD_KAFKA_OFFSET_TAIL(-start_offset);
}
break;
case 'e':
exit_eof = 1;
break;
case 'd':
debug = optarg;
break;
case 'q':
quiet = 1;
break;
case 'A':
output = OUTPUT_RAW;
break;
case 'X':
{
char *name, *val;
rd_kafka_conf_res_t res;
if (!strcmp(optarg, "list") ||
!strcmp(optarg, "help")) {
rd_kafka_conf_properties_show(stdout);
exit(0);
}
if (!strcmp(optarg, "dump")) {
do_conf_dump = 1;
continue;
}
name = optarg;
if (!(val = strchr(name, '='))) {
fprintf(stderr, "%% Expected "
"-X property=value, not %s\n", name);
exit(1);
}
*val = '\0';
val++;
res = RD_KAFKA_CONF_UNKNOWN;
/* Try "topic." prefixed properties on topic
* conf first, and then fall through to global if
* it didnt match a topic configuration property. */
if (!strncmp(name, "topic.", strlen("topic.")))
res = rd_kafka_topic_conf_set(topic_conf,
name+
strlen("topic."),
val,
errstr,
sizeof(errstr));
if (res == RD_KAFKA_CONF_UNKNOWN)
res = rd_kafka_conf_set(conf, name, val,
errstr, sizeof(errstr));
if (res != RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
}
break;
default:
goto usage;
}
}
if (do_conf_dump) {
const char **arr;
size_t cnt;
int pass;
for (pass = 0 ; pass < 2 ; pass++) {
int i;
if (pass == 0) {
arr = rd_kafka_conf_dump(conf, &cnt);
printf("# Global config\n");
} else {
printf("# Topic config\n");
arr = rd_kafka_topic_conf_dump(topic_conf,
&cnt);
}
for (i = 0 ; i < cnt ; i += 2)
printf("%s = %s\n",
arr[i], arr[i+1]);
printf("\n");
rd_kafka_conf_dump_free(arr, cnt);
}
exit(0);
}
if (optind != argc || (mode != 'L' && !topic)) {
usage:
fprintf(stderr,
"Usage: %s -C|-P|-L -t <topic> "
"[-p <partition>] [-b <host1:port1,host2:port2,..>]\n"
"\n"
"librdkafka version %s (0x%08x)\n"
"\n"
" Options:\n"
" -C | -P Consumer or Producer mode\n"
" -L Metadata list mode\n"
" -t <topic> Topic to fetch / produce\n"
" -p <num> Partition (random partitioner)\n"
" -b <brokers> Broker address (localhost:9092)\n"
" -z <codec> Enable compression:\n"
" none|gzip|snappy\n"
" -o <offset> Start offset (consumer):\n"
" beginning, end, NNNNN or -NNNNN\n"
" -o report Report message offsets (producer)\n"
" -e Exit consumer when last message\n"
" in partition has been received.\n"
" -d [facs..] Enable debugging contexts:\n"
" -q Be quiet\n"
" -A Raw payload output (consumer)\n"
" %s\n"
" -X <prop=name> Set arbitrary librdkafka "
"configuration property\n"
" Properties prefixed with \"topic.\" "
"will be set on topic object.\n"
" Use '-X list' to see the full list\n"
" of supported properties.\n"
"\n"
" In Consumer mode:\n"
" writes fetched messages to stdout\n"
" In Producer mode:\n"
" reads messages from stdin and sends to broker\n"
" In List mode:\n"
" queries broker for metadata information, "
"topic is optional.\n"
"\n"
"\n"
"\n",
argv[0],
rd_kafka_version_str(), rd_kafka_version(),
RD_KAFKA_DEBUG_CONTEXTS);
exit(1);
}
signal(SIGINT, stop);
signal(SIGUSR1, sig_usr1);
if (debug &&
rd_kafka_conf_set(conf, "debug", debug, errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% Debug configuration failed: %s: %s\n",
errstr, debug);
exit(1);
}
if (mode == 'P') {
/*
* Producer
*/
char buf[2048];
int sendcnt = 0;
/* Set up a message delivery report callback.
* It will be called once for each message, either on successful
* delivery to broker, or upon failure to deliver to broker. */
/* If offset reporting (-o report) is enabled, use the
* richer dr_msg_cb instead. */
if (report_offsets) {
rd_kafka_topic_conf_set(topic_conf,
"produce.offset.report",
"true", errstr, sizeof(errstr));
rd_kafka_conf_set_dr_msg_cb(conf, msg_delivered2);
} else
rd_kafka_conf_set_dr_cb(conf, msg_delivered);
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new producer: %s\n",
errstr);
exit(1);
}
/* Set logger */
rd_kafka_set_logger(rk, logger);
rd_kafka_set_log_level(rk, LOG_DEBUG);
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Create topic */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (!quiet)
fprintf(stderr,
"%% Type stuff and hit enter to send\n");
while (run && fgets(buf, sizeof(buf), stdin)) {
size_t len = strlen(buf);
if (buf[len-1] == '\n')
buf[--len] = '\0';
/* Send/Produce message. */
if (rd_kafka_produce(rkt, partition,
RD_KAFKA_MSG_F_COPY,
/* Payload and length */
buf, len,
/* Optional key and its length */
NULL, 0,
/* Message opaque, provided in
* delivery report callback as
* msg_opaque. */
NULL) == -1) {
fprintf(stderr,
"%% Failed to produce to topic %s "
"partition %i: %s\n",
rd_kafka_topic_name(rkt), partition,
rd_kafka_err2str(
rd_kafka_errno2err(errno)));
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
continue;
}
if (!quiet)
fprintf(stderr, "%% Sent %zd bytes to topic "
"%s partition %i\n",
len, rd_kafka_topic_name(rkt), partition);
sendcnt++;
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
}
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
/* Wait for messages to be delivered */
while (run && rd_kafka_outq_len(rk) > 0)
rd_kafka_poll(rk, 100);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy the handle */
rd_kafka_destroy(rk);
} else if (mode == 'C') {
/*
* Consumer
*/
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new consumer: %s\n",
errstr);
exit(1);
}
/* Set logger */
rd_kafka_set_logger(rk, logger);
rd_kafka_set_log_level(rk, LOG_DEBUG);
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Create topic */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
/* Start consuming */
if (rd_kafka_consume_start(rkt, partition, start_offset) == -1){
fprintf(stderr, "%% Failed to start consuming: %s\n",
rd_kafka_err2str(rd_kafka_errno2err(errno)));
exit(1);
}
while (run) {
rd_kafka_message_t *rkmessage;
/* Consume single message.
* See rdkafka_performance.c for high speed
* consuming of messages. */
rkmessage = rd_kafka_consume(rkt, partition, 1000);
if (!rkmessage) /* timeout */
continue;
msg_consume(rkmessage, NULL);
/* Return message to rdkafka */
rd_kafka_message_destroy(rkmessage);
}
/* Stop consuming */
rd_kafka_consume_stop(rkt, partition);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy handle */
rd_kafka_destroy(rk);
} else if (mode == 'L') {
rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new producer: %s\n",
errstr);
exit(1);
}
/* Set logger */
rd_kafka_set_logger(rk, logger);
rd_kafka_set_log_level(rk, LOG_DEBUG);
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Create topic */
if (topic)
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
else
rkt = NULL;
while (run) {
const struct rd_kafka_metadata *metadata;
/* Fetch metadata */
err = rd_kafka_metadata(rk, rkt ? 0 : 1, rkt,
&metadata, 5000);
if (err != RD_KAFKA_RESP_ERR_NO_ERROR) {
fprintf(stderr,
"%% Failed to acquire metadata: %s\n",
rd_kafka_err2str(err));
run = 0;
break;
}
metadata_print(topic, metadata);
rd_kafka_metadata_destroy(metadata);
run = 0;
}
/* Destroy topic */
if (rkt)
rd_kafka_topic_destroy(rkt);
/* Destroy the handle */
rd_kafka_destroy(rk);
/* Exit right away, dont wait for background cleanup, we haven't
* done anything important anyway. */
exit(err ? 2 : 0);
}
/* Let background threads clean up and terminate cleanly. */
rd_kafka_wait_destroyed(2000);
return 0;
}
static void consume_messages (uint64_t testid, const char *topic,
int32_t partition, int msg_base, int batch_cnt,
int msgcnt) {
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
int i;
test_conf_init(&conf, &topic_conf, 20);
/* Create kafka instance */
rk = test_create_handle(RD_KAFKA_CONSUMER, conf);
TEST_SAY("Created kafka instance %s\n", rd_kafka_name(rk));
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
rd_strerror(errno));
TEST_SAY("Consuming %i messages from partition %i\n",
batch_cnt, partition);
/* Consume messages */
if (rd_kafka_consume_start(rkt, partition,
RD_KAFKA_OFFSET_TAIL(batch_cnt)) == -1)
TEST_FAIL("consume_start(%i, -%i) failed: %s",
(int)partition, batch_cnt,
rd_kafka_err2str(rd_kafka_errno2err(errno)));
for (i = 0 ; i < batch_cnt ; ) {
rd_kafka_message_t *rkmessage;
rkmessage = rd_kafka_consume(rkt, partition,
tmout_multip(5000));
if (!rkmessage)
TEST_FAIL("Failed to consume message %i/%i from "
"partition %i: %s",
i, batch_cnt, (int)partition,
rd_kafka_err2str(rd_kafka_errno2err(errno)));
if (rkmessage->err) {
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF){
rd_kafka_message_destroy(rkmessage);
continue;
}
TEST_FAIL("Consume message %i/%i from partition %i "
"has error: %s",
i, batch_cnt, (int)partition,
rd_kafka_err2str(rkmessage->err));
}
verify_consumed_msg(testid, partition, msg_base+i, rkmessage);
rd_kafka_message_destroy(rkmessage);
i++;
}
rd_kafka_consume_stop(rkt, partition);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy rdkafka instance */
TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
rd_kafka_destroy(rk);
}
/* Produce a batch of messages to a single partition. */
static void test_single_partition (void) {
int partition = 0;
int r;
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char msg[128];
int msgcnt = 100000;
int failcnt = 0;
int i;
rd_kafka_message_t *rkmessages;
int msgcounter = 0;
msgid_next = 0;
test_conf_init(&conf, &topic_conf, 20);
/* Set delivery report callback */
rd_kafka_conf_set_dr_cb(conf, dr_single_partition_cb);
rd_kafka_conf_set_opaque(conf, &msgcounter);
/* Create kafka instance */
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
TEST_SAY("test_single_partition: Created kafka instance %s\n",
rd_kafka_name(rk));
rkt = rd_kafka_topic_new(rk, test_mk_topic_name("0011", 0),
topic_conf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
rd_strerror(errno));
/* Create messages */
rkmessages = calloc(sizeof(*rkmessages), msgcnt);
for (i = 0 ; i < msgcnt ; i++) {
int *msgidp = malloc(sizeof(*msgidp));
*msgidp = i;
rd_snprintf(msg, sizeof(msg), "%s:%s test message #%i",
__FILE__, __FUNCTION__, i);
rkmessages[i].payload = rd_strdup(msg);
rkmessages[i].len = strlen(msg);
rkmessages[i]._private = msgidp;
}
r = rd_kafka_produce_batch(rkt, partition, RD_KAFKA_MSG_F_FREE,
rkmessages, msgcnt);
/* Scan through messages to check for errors. */
for (i = 0 ; i < msgcnt ; i++) {
if (rkmessages[i].err) {
failcnt++;
if (failcnt < 100)
TEST_SAY("Message #%i failed: %s\n",
i,
rd_kafka_err2str(rkmessages[i].err));
}
}
/* All messages should've been produced. */
if (r < msgcnt) {
TEST_SAY("Not all messages were accepted "
"by produce_batch(): %i < %i\n", r, msgcnt);
if (msgcnt - r != failcnt)
TEST_SAY("Discrepency between failed messages (%i) "
"and return value %i (%i - %i)\n",
failcnt, msgcnt - r, msgcnt, r);
TEST_FAIL("%i/%i messages failed\n", msgcnt - r, msgcnt);
}
free(rkmessages);
TEST_SAY("Single partition: "
"Produced %i messages, waiting for deliveries\n", r);
msgcounter = msgcnt;
/* Wait for messages to be delivered */
test_wait_delivery(rk, &msgcounter);
if (fails)
TEST_FAIL("%i failures, see previous errors", fails);
if (msgid_next != msgcnt)
TEST_FAIL("Still waiting for messages: next %i != end %i\n",
msgid_next, msgcnt);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy rdkafka instance */
TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
rd_kafka_destroy(rk);
return;
}
