static void rebalance_cb (rd_kafka_t *rk, rd_kafka_resp_err_t err,
rd_kafka_topic_partition_list_t *parts, void *opaque) {
int i;
TEST_SAY("rebalance_cb: %s:\n", rd_kafka_err2str(err));
test_print_partition_list(parts);
if (parts->cnt < partitions)
TEST_FAIL("rebalance_cb: Expected %d partitions, not %d",
partitions, parts->cnt);
switch (err)
{
case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS:
for (i = 0 ; i < parts->cnt ; i++) {
if (i < partitions)
parts->elems[i].offset = msgcnt / 2;
else
parts->elems[i].offset = RD_KAFKA_OFFSET_END;
}
TEST_SAY("Use these offsets:\n");
test_print_partition_list(parts);
test_consumer_assign("HL.REBALANCE", rk, parts);
break;
case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS:
test_consumer_unassign("HL.REBALANCE", rk);
break;
default:
TEST_FAIL("rebalance_cb: error: %s", rd_kafka_err2str(err));
}
}
static void assign_consume_many (char **topics, int topic_cnt, uint64_t testid){
rd_kafka_t *rk;
rd_kafka_topic_partition_list_t *parts;
int i;
test_msgver_t mv;
TEST_SAY(_C_MAG "%s\n" _C_CLR, __FUNCTION__);
test_conf_init(NULL, NULL, 60);
rk = test_create_consumer(__FUNCTION__, NULL, NULL, NULL);
parts = rd_kafka_topic_partition_list_new(topic_cnt);
for (i = 0 ; i < topic_cnt ; i++)
rd_kafka_topic_partition_list_add(parts, topics[i], 0)->
offset = RD_KAFKA_OFFSET_TAIL(msgs_per_topic);
test_consumer_assign("consume.assign", rk, parts);
rd_kafka_topic_partition_list_destroy(parts);
test_msgver_init(&mv, testid);
test_consumer_poll("consume.assign", rk, testid,
-1, 0, msgs_per_topic * topic_cnt, &mv);
for (i = 0 ; i < topic_cnt ; i++)
test_msgver_verify_part("assign", &mv, TEST_MSGVER_ALL_PART,
topics[i], 0, i * msgs_per_topic,
msgs_per_topic);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
}
static void do_consume (const char *topic, int msgcnt) {
rd_kafka_t *rk;
rd_kafka_topic_partition_list_t *parts;
rk = test_create_consumer(topic, NULL, NULL, NULL);
parts = rd_kafka_topic_partition_list_new(1);
rd_kafka_topic_partition_list_add(parts, topic, 0)->offset =
RD_KAFKA_OFFSET_BEGINNING;
test_consumer_assign("assign", rk, parts);
rd_kafka_topic_partition_list_destroy(parts);
exp_msgid = 0;
while (exp_msgid < msgcnt) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(rk, 1000);
if (!rkm)
continue;
if (rkm->err)
TEST_FAIL("consume error while expecting msgid %d/%d: "
"%s",
exp_msgid, msgcnt,
rd_kafka_message_errstr(rkm));
handle_consumed_msg(rkm);
rd_kafka_message_destroy(rkm);
}
test_consumer_close(rk);
rd_kafka_destroy(rk);
}
int main_0029_assign_offset (int argc, char **argv) {
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_topic_conf_t *tconf;
rd_kafka_topic_partition_list_t *parts;
uint64_t testid;
int i;
test_timing_t t_simple, t_hl;
test_msgver_t mv;
test_conf_init(NULL, &tconf, 20 + (test_session_timeout_ms * 3 / 1000));
test_topic_conf_set(tconf, "auto.offset.reset", "smallest");
/* Produce X messages to Y partitions so we get a
* nice seekable 0..X offset one each partition. */
/* Produce messages */
testid = test_id_generate();
rk = test_create_producer();
rkt = test_create_producer_topic(rk, topic, NULL);
parts = rd_kafka_topic_partition_list_new(partitions);
for (i = 0 ; i < partitions ; i++) {
test_produce_msgs(rk, rkt, testid, i, 0, msgcnt, NULL, 0);
rd_kafka_topic_partition_list_add(parts, topic, i)->offset =
msgcnt / 2;
}
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
/* Simple consumer */
TIMING_START(&t_simple, "SIMPLE.CONSUMER");
rk = test_create_consumer(topic, NULL, NULL, NULL);
test_msgver_init(&mv, testid);
test_consumer_assign("SIMPLE.ASSIGN", rk, parts);
test_consumer_poll("SIMPLE.CONSUME", rk, testid, -1, 0,
partitions * (msgcnt / 2), &mv);
for (i = 0 ; i < partitions ; i++)
test_msgver_verify_part("HL.MSGS", &mv, TEST_MSGVER_ALL_PART,
topic, i, msgcnt/2, msgcnt/2);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_simple);
rd_kafka_topic_partition_list_destroy(parts);
/* High-level consumer.
* Offsets are set in rebalance callback. */
TIMING_START(&t_hl, "HL.CONSUMER");
test_msgver_init(&mv, testid);
rk = test_create_consumer(topic, rebalance_cb, NULL, NULL);
test_consumer_subscribe(rk, topic);
test_consumer_poll("HL.CONSUME", rk, testid, -1, 0,
partitions * (msgcnt / 2), &mv);
for (i = 0 ; i < partitions ; i++)
test_msgver_verify_part("HL.MSGS", &mv, TEST_MSGVER_ALL_PART,
topic, i, msgcnt/2, msgcnt/2);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_hl);
rd_kafka_topic_conf_destroy(tconf);
return 0;
}
int main_0029_assign_offset (int argc, char **argv) {
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_topic_partition_list_t *parts;
uint64_t testid;
int i;
test_timing_t t_simple, t_hl;
test_msgver_t mv;
test_conf_init(NULL, NULL, 20 + (test_session_timeout_ms * 3 / 1000));
/* Produce X messages to Y partitions so we get a
* nice seekable 0..X offset one each partition. */
/* Produce messages */
testid = test_id_generate();
rk = test_create_producer();
rkt = test_create_producer_topic(rk, topic, NULL);
parts = rd_kafka_topic_partition_list_new(partitions);
for (i = 0 ; i < partitions ; i++) {
test_produce_msgs(rk, rkt, testid, i, 0, msgcnt, NULL, 0);
/* Set start offset */
rd_kafka_topic_partition_list_add(parts, topic, i)->offset =
msgcnt / 2;
}
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
/* Simple consumer */
TIMING_START(&t_simple, "SIMPLE.CONSUMER");
rk = test_create_consumer(topic, NULL, NULL, NULL);
test_msgver_init(&mv, testid);
test_consumer_assign("SIMPLE.ASSIGN", rk, parts);
test_consumer_poll("SIMPLE.CONSUME", rk, testid, -1, 0,
partitions * (msgcnt / 2), &mv);
for (i = 0 ; i < partitions ; i++)
test_msgver_verify_part("HL.MSGS", &mv, TEST_MSGVER_ALL_PART,
topic, i, msgcnt/2, msgcnt/2);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_simple);
rd_kafka_topic_partition_list_destroy(parts);
/* High-level consumer: method 1
* Offsets are set in rebalance callback. */
if (test_broker_version >= TEST_BRKVER(0,9,0,0)) {
reb_method = REB_METHOD_1;
TIMING_START(&t_hl, "HL.CONSUMER");
test_msgver_init(&mv, testid);
rk = test_create_consumer(topic, rebalance_cb, NULL, NULL);
test_consumer_subscribe(rk, topic);
test_consumer_poll("HL.CONSUME", rk, testid, -1, 0,
partitions * (msgcnt / 2), &mv);
for (i = 0 ; i < partitions ; i++)
test_msgver_verify_part("HL.MSGS", &mv,
TEST_MSGVER_ALL_PART,
topic, i, msgcnt/2, msgcnt/2);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_hl);
/* High-level consumer: method 2:
* first two partitions are with fixed absolute offset, rest are
* auto offset (stored, which is now at end).
* Offsets are set in rebalance callback. */
reb_method = REB_METHOD_2;
TIMING_START(&t_hl, "HL.CONSUMER2");
test_msgver_init(&mv, testid);
rk = test_create_consumer(topic, rebalance_cb, NULL, NULL);
test_consumer_subscribe(rk, topic);
test_consumer_poll("HL.CONSUME2", rk, testid, partitions, 0,
2 * (msgcnt / 2), &mv);
for (i = 0 ; i < partitions ; i++) {
if (i < 2)
test_msgver_verify_part("HL.MSGS2.A", &mv,
TEST_MSGVER_ALL_PART,
topic, i, msgcnt/2,
msgcnt/2);
}
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_hl);
}
return 0;
}
