static void legacy_consume_many (char **topics, int topic_cnt, uint64_t testid){
rd_kafka_t *rk;
test_timing_t t_rkt_create;
int i;
rd_kafka_topic_t **rkts;
int msg_base = 0;
TEST_SAY(_C_MAG "%s\n" _C_CLR, __FUNCTION__);
test_conf_init(NULL, NULL, 60);
rk = test_create_consumer(NULL, NULL, NULL, NULL);
TEST_SAY("Creating %d topic objects\n", topic_cnt);
rkts = malloc(sizeof(*rkts) * topic_cnt);
TIMING_START(&t_rkt_create, "Topic object create");
for (i = 0 ; i < topic_cnt ; i++)
rkts[i] = test_create_topic_object(rk, topics[i], NULL);
TIMING_STOP(&t_rkt_create);
TEST_SAY("Start consumer for %d topics\n", topic_cnt);
for (i = 0 ; i < topic_cnt ; i++)
test_consumer_start("legacy", rkts[i], 0,
RD_KAFKA_OFFSET_BEGINNING);
TEST_SAY("Consuming from %d messages from each %d topics\n",
msgs_per_topic, topic_cnt);
for (i = 0 ; i < topic_cnt ; i++) {
test_consume_msgs("legacy", rkts[i], testid, 0, TEST_NO_SEEK,
msg_base, msgs_per_topic, 1);
msg_base += msgs_per_topic;
}
TEST_SAY("Stopping consumers\n");
for (i = 0 ; i < topic_cnt ; i++)
test_consumer_stop("legacy", rkts[i], 0);
TEST_SAY("Destroying %d topic objects\n", topic_cnt);
for (i = 0 ; i < topic_cnt ; i++)
rd_kafka_topic_destroy(rkts[i]);
free(rkts);
rd_kafka_destroy(rk);
}
int main_0041_fetch_max_bytes (int argc, char **argv) {
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
const int partition = 0;
const int msgcnt = 2*1000;
const int MAX_BYTES = 100000;
uint64_t testid;
rd_kafka_conf_t *conf;
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
test_conf_init(NULL, NULL, 60);
testid = test_id_generate();
rk = test_create_producer();
rkt = test_create_producer_topic(rk, topic, NULL);
test_produce_msgs(rk, rkt, testid, partition, 0, msgcnt/2, NULL, MAX_BYTES/10);
test_produce_msgs(rk, rkt, testid, partition, msgcnt/2, msgcnt/2, NULL, MAX_BYTES*5);
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
TEST_SAY("Creating consumer\n");
test_conf_init(&conf, NULL, 0);
test_conf_set(conf, "fetch.message.max.bytes", tsprintf("%d", MAX_BYTES));
rk = test_create_consumer(NULL, NULL, conf, NULL);
rkt = rd_kafka_topic_new(rk, topic, NULL);
test_consumer_start("CONSUME", rkt, partition,
RD_KAFKA_OFFSET_BEGINNING);
test_consume_msgs("CONSUME", rkt, testid, partition, TEST_NO_SEEK,
0, msgcnt, 1);
test_consumer_stop("CONSUME", rkt, partition);
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
return 0;
}
int main_0036_partial_fetch (int argc, char **argv) {
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
const int partition = 0;
const int msgcnt = 100;
const int msgsize = 1000;
uint64_t testid;
rd_kafka_conf_t *conf;
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
TEST_SAY("Producing %d messages of size %d to %s [%d]\n",
msgcnt, (int)msgsize, topic, partition);
testid = test_id_generate();
rk = test_create_producer();
rkt = test_create_producer_topic(rk, topic, NULL);
test_produce_msgs(rk, rkt, testid, partition, 0, msgcnt, NULL, msgsize);
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
TEST_SAY("Creating consumer\n");
test_conf_init(&conf, NULL, 0);
/* This should fetch 1.5 messages per fetch, thus resulting in
* partial fetches, hopefully. */
test_conf_set(conf, "fetch.message.max.bytes", "1500");
rk = test_create_consumer(NULL, NULL, conf, NULL, NULL);
rkt = rd_kafka_topic_new(rk, topic, NULL);
test_consumer_start("CONSUME", rkt, partition,
RD_KAFKA_OFFSET_BEGINNING);
test_consume_msgs("CONSUME", rkt, testid, partition, TEST_NO_SEEK,
0, msgcnt, 1);
test_consumer_stop("CONSUME", rkt, partition);
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
return 0;
}
int main_0038_performance (int argc, char **argv) {
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
const int partition = 0;
const int msgsize = 100;
uint64_t testid;
rd_kafka_conf_t *conf;
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
test_timing_t t_create, t_produce, t_consume;
int totsize = 1024*1024*128;
int msgcnt;
if (!strcmp(test_mode, "valgrind") || !strcmp(test_mode, "helgrind") ||
!strcmp(test_mode, "drd"))
totsize = 1024*1024*8; /* 8 meg, valgrind is slow. */
msgcnt = totsize / msgsize;
TEST_SAY("Producing %d messages of size %d to %s [%d]\n",
msgcnt, (int)msgsize, topic, partition);
testid = test_id_generate();
test_conf_init(&conf, NULL, 120);
rd_kafka_conf_set_dr_cb(conf, test_dr_cb);
test_conf_set(conf, "queue.buffering.max.messages", "10000000");
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
rkt = test_create_producer_topic(rk, topic, "acks", "1", NULL);
/* First produce one message to create the topic, etc, this might take
* a while and we dont want this to affect the throughput timing. */
TIMING_START(&t_create, "CREATE TOPIC");
test_produce_msgs(rk, rkt, testid, partition, 0, 1, NULL, msgsize);
TIMING_STOP(&t_create);
TIMING_START(&t_produce, "PRODUCE");
test_produce_msgs(rk, rkt, testid, partition, 1, msgcnt-1, NULL, msgsize);
TIMING_STOP(&t_produce);
TEST_SAY("Destroying producer\n");
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
TEST_SAY("Creating consumer\n");
test_conf_init(&conf, NULL, 120);
rk = test_create_consumer(NULL, NULL, conf, NULL, NULL);
rkt = rd_kafka_topic_new(rk, topic, NULL);
test_consumer_start("CONSUME", rkt, partition,
RD_KAFKA_OFFSET_BEGINNING);
TIMING_START(&t_consume, "CONSUME");
test_consume_msgs("CONSUME", rkt, testid, partition, TEST_NO_SEEK,
0, msgcnt, 1);
TIMING_STOP(&t_consume);
test_consumer_stop("CONSUME", rkt, partition);
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
TEST_REPORT("{ \"producer\": "
" { \"mb_per_sec\": %.2f, \"records_per_sec\": %.2f },"
" \"consumer\": "
"{ \"mb_per_sec\": %.2f, \"records_per_sec\": %.2f } "
"}",
(double)
(totsize/((double)TIMING_DURATION(&t_produce)/1000000.0f)) /
1000000.0f,
(float)
(msgcnt/((double)TIMING_DURATION(&t_produce)/1000000.0f)),
(double)
(totsize/((double)TIMING_DURATION(&t_consume)/1000000.0f)) /
1000000.0f,
(float)
(msgcnt/((double)TIMING_DURATION(&t_consume)/1000000.0f)));
return 0;
}
int main_0015_offsets_seek (int argc, char **argv) {
const char *topic = test_mk_topic_name("0015", 1);
rd_kafka_t *rk_p, *rk_c;
rd_kafka_topic_t *rkt_p, *rkt_c;
int msg_cnt = 1000;
int msg_base = 0;
int32_t partition = 0;
int i;
int64_t offset_last, offset_base;
uint64_t testid;
int dance_iterations = 10;
int msgs_per_dance = 10;
testid = test_id_generate();
/* Produce messages */
rk_p = test_create_producer();
rkt_p = test_create_producer_topic(rk_p, topic, NULL);
test_produce_msgs(rk_p, rkt_p, testid, partition, msg_base, msg_cnt,
NULL, 0);
rd_kafka_topic_destroy(rkt_p);
rd_kafka_destroy(rk_p);
rk_c = test_create_consumer(NULL, NULL, NULL, NULL);
rkt_c = test_create_consumer_topic(rk_c, topic);
/* Start consumer tests */
test_consumer_start("verify.all", rkt_c, partition,
RD_KAFKA_OFFSET_BEGINNING);
/* Make sure all messages are available */
offset_last = test_consume_msgs("verify.all", rkt_c,
testid, partition, TEST_NO_SEEK,
msg_base, msg_cnt, 1/* parse format*/);
/* Rewind offset back to its base. */
offset_base = offset_last - msg_cnt + 1;
TEST_SAY("%s [%"PRId32"]: Do random seek&consume for msgs #%d+%d with "
"offsets %"PRId64"..%"PRId64"\n",
rd_kafka_topic_name(rkt_c), partition,
msg_base, msg_cnt, offset_base, offset_last);
/* Now go dancing over the entire range with offset seeks. */
for (i = 0 ; i < dance_iterations ; i++) {
int64_t offset = jitter((int)offset_base,
(int)offset_base+msg_cnt);
test_consume_msgs("dance", rkt_c,
testid, partition, offset,
msg_base + (int)(offset - offset_base),
RD_MIN(msgs_per_dance,
(int)(offset_last - offset)),
1 /* parse format */);
}
test_consumer_stop("1", rkt_c, partition);
rd_kafka_topic_destroy(rkt_c);
rd_kafka_destroy(rk_c);
return 0;
}
int main_0017_compression(int argc, char **argv) {
rd_kafka_t *rk_p, *rk_c;
const int msg_cnt = 1000;
int msg_base = 0;
uint64_t testid;
#define CODEC_CNT 4
const char *codecs[CODEC_CNT+1] = {
"none",
#if WITH_ZLIB
"gzip",
#endif
#if WITH_SNAPPY
"snappy",
#endif
#if WITH_LZ4
"lz4",
#endif
NULL
};
const char *topics[CODEC_CNT];
const int32_t partition = 0;
int i;
testid = test_id_generate();
/* Produce messages */
rk_p = test_create_producer();
for (i = 0; codecs[i] != NULL ; i++) {
rd_kafka_topic_t *rkt_p;
topics[i] = test_mk_topic_name(codecs[i], 1);
TEST_SAY("Produce %d messages with %s compression to "
"topic %s\n",
msg_cnt, codecs[i], topics[i]);
rkt_p = test_create_producer_topic(rk_p, topics[i],
"compression.codec", codecs[i], NULL);
/* Produce small message that will not decrease with
* compression (issue #781) */
test_produce_msgs(rk_p, rkt_p, testid, partition,
msg_base + (partition*msg_cnt), 1,
NULL, 5);
/* Produce standard sized messages */
test_produce_msgs(rk_p, rkt_p, testid, partition,
msg_base + (partition*msg_cnt) + 1, msg_cnt-1,
NULL, 512);
rd_kafka_topic_destroy(rkt_p);
}
rd_kafka_destroy(rk_p);
/* Consume messages */
rk_c = test_create_consumer(NULL, NULL, NULL, NULL);
for (i = 0; codecs[i] != NULL ; i++) {
rd_kafka_topic_t *rkt_c = rd_kafka_topic_new(rk_c,
topics[i], NULL);
TEST_SAY("Consume %d messages from topic %s\n",
msg_cnt, topics[i]);
/* Start consuming */
test_consumer_start(codecs[i], rkt_c, partition,
RD_KAFKA_OFFSET_BEGINNING);
/* Consume messages */
test_consume_msgs(codecs[i], rkt_c, testid, partition,
/* Use offset 0 here, which is wrong, should
* be TEST_NO_SEEK, but it exposed a bug
* where the Offset query was postponed
* till after the seek, causing messages
* to be replayed. */
0,
msg_base, msg_cnt, 1 /* parse format */);
test_consumer_stop(codecs[i], rkt_c, partition);
rd_kafka_topic_destroy(rkt_c);
}
rd_kafka_destroy(rk_c);
return 0;
}
int main_0031_get_offsets (int argc, char **argv) {
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
const int msgcnt = 100;
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
int64_t qry_low = -1234, qry_high = -1235;
int64_t get_low = -1234, get_high = -1235;
rd_kafka_resp_err_t err;
test_timing_t t_qry, t_get;
uint64_t testid;
/* Produce messages */
testid = test_produce_msgs_easy(topic, 0, 0, msgcnt);
/* Get offsets */
rk = test_create_consumer(NULL, NULL, NULL, NULL
);
TIMING_START(&t_qry, "query_watermark_offsets");
err = rd_kafka_query_watermark_offsets(rk, topic, 0,
&qry_low, &qry_high, 10*1000);
TIMING_STOP(&t_qry);
if (err)
TEST_FAIL("query_watermark_offsets failed: %s\n",
rd_kafka_err2str(err));
if (qry_low != 0 && qry_high != msgcnt)
TEST_FAIL("Expected low,high %d,%d, but got "
"%"PRId64",%"PRId64,
0, msgcnt, qry_low, qry_high);
TEST_SAY("query_watermark_offsets: "
"offsets %"PRId64", %"PRId64"\n", qry_low, qry_high);
/* Now start consuming to update the offset cache, then query it
* with the get_ API. */
rkt = test_create_topic_object(rk, topic, NULL);
test_consumer_start("get", rkt, 0, RD_KAFKA_OFFSET_BEGINNING);
test_consume_msgs("get", rkt, testid, 0, TEST_NO_SEEK,
0, msgcnt, 0);
/* After at least one message has been consumed the
* watermarks are cached. */
TIMING_START(&t_get, "get_watermark_offsets");
err = rd_kafka_get_watermark_offsets(rk, topic, 0,
&get_low, &get_high);
TIMING_STOP(&t_get);
if (err)
TEST_FAIL("get_watermark_offsets failed: %s\n",
rd_kafka_err2str(err));
TEST_SAY("get_watermark_offsets: "
"offsets %"PRId64", %"PRId64"\n", get_low, get_high);
if (get_high != qry_high)
TEST_FAIL("query/get discrepancies: "
"low: %"PRId64"/%"PRId64", high: %"PRId64"/%"PRId64,
qry_low, get_low, qry_high, get_high);
if (get_low >= get_high)
TEST_FAIL("get_watermark_offsets: "
"low %"PRId64" >= high %"PRId64,
get_low, get_high);
/* FIXME: We currently dont bother checking the get_low offset
* since it requires stats to be enabled. */
test_consumer_stop("get", rkt, 0);
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
return 0;
}
