static PyObject *Consumer_offsets_for_times (Handle *self, PyObject *args,
PyObject *kwargs) {
#if RD_KAFKA_VERSION < 0x000b0000
PyErr_Format(PyExc_NotImplementedError,
"Consumer offsets_for_times require "
"confluent-kafka-python built for librdkafka "
"version >=v0.11.0 (librdkafka runtime 0x%x, "
"buildtime 0x%x)",
rd_kafka_version(), RD_KAFKA_VERSION);
return NULL;
#else
PyObject *plist;
double tmout = -1.0f;
rd_kafka_topic_partition_list_t *c_parts;
rd_kafka_resp_err_t err;
static char *kws[] = { "partitions", "timeout", NULL };
if (!self->rk) {
PyErr_SetString(PyExc_RuntimeError,
"Consumer closed");
return NULL;
}
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|d", kws,
&plist, &tmout))
return NULL;
if (!(c_parts = py_to_c_parts(plist)))
return NULL;
Py_BEGIN_ALLOW_THREADS;
err = rd_kafka_offsets_for_times(self->rk,
c_parts,
tmout >= 0 ? (int)(tmout * 1000.0f) : -1);
Py_END_ALLOW_THREADS;
if (err) {
rd_kafka_topic_partition_list_destroy(c_parts);
cfl_PyErr_Format(err,
"Failed to get offsets: %s",
rd_kafka_err2str(err));
return NULL;
}
plist = c_parts_to_py(c_parts);
rd_kafka_topic_partition_list_destroy(c_parts);
return plist;
#endif
}
static PyObject *Consumer_assignment (Handle *self, PyObject *args,
PyObject *kwargs) {
PyObject *plist;
rd_kafka_topic_partition_list_t *c_parts;
rd_kafka_resp_err_t err;
if (!self->rk) {
PyErr_SetString(PyExc_RuntimeError,
"Consumer closed");
return NULL;
}
err = rd_kafka_assignment(self->rk, &c_parts);
if (err) {
cfl_PyErr_Format(err,
"Failed to get assignment: %s",
rd_kafka_err2str(err));
return NULL;
}
plist = c_parts_to_py(c_parts);
rd_kafka_topic_partition_list_destroy(c_parts);
return plist;
}
static void do_nonexist_commit (void) {
rd_kafka_t *rk;
char group_id[64];
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *tconf;
rd_kafka_topic_partition_list_t *offsets;
const char *unk_topic = test_mk_topic_name(__FUNCTION__, 1);
rd_kafka_resp_err_t err;
test_conf_init(&conf, &tconf, 20);
test_str_id_generate(group_id, sizeof(group_id));
TEST_SAY(_C_MAG "[ do_nonexist_commit group.id %s ]\n", group_id);
rk = test_create_consumer(group_id, NULL, conf, tconf, NULL);
TEST_SAY("Try nonexist commit\n");
offsets = rd_kafka_topic_partition_list_new(2);
rd_kafka_topic_partition_list_add(offsets, unk_topic, 0)->offset = 123;
rd_kafka_topic_partition_list_add(offsets, unk_topic, 1)->offset = 456;
err = rd_kafka_commit_queue(rk, offsets, NULL,
nonexist_offset_commit_cb, NULL);
TEST_SAY("nonexist commit returned %s\n", rd_kafka_err2str(err));
if (err != RD_KAFKA_RESP_ERR_NO_ERROR)
TEST_FAIL("commit() should succeed, not: %s",
rd_kafka_err2str(err));
rd_kafka_topic_partition_list_destroy(offsets);
test_consumer_close(rk);
rd_kafka_destroy(rk);
}
static rd_kafka_resp_err_t
rd_kafka_offset_broker_commit (rd_kafka_toppar_t *rktp) {
rd_kafka_topic_partition_list_t *offsets;
rd_kafka_topic_partition_t *rktpar;
rd_kafka_assert(rktp->rktp_rkt->rkt_rk, rktp->rktp_cgrp != NULL);
rd_kafka_assert(rktp->rktp_rkt->rkt_rk,
rktp->rktp_flags & RD_KAFKA_TOPPAR_F_OFFSET_STORE);
rktp->rktp_committing_offset = rktp->rktp_stored_offset;
offsets = rd_kafka_topic_partition_list_new(1);
rktpar = rd_kafka_topic_partition_list_add(
offsets, rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition);
rktpar->offset = rktp->rktp_committing_offset;
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSETCMT",
"%.*s [%"PRId32"]: committing offset %"PRId64,
RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
rktp->rktp_partition, rktp->rktp_committing_offset);
rd_kafka_commit0(rktp->rktp_rkt->rkt_rk, offsets, rktp,
RD_KAFKA_REPLYQ(rktp->rktp_ops, 0),
rd_kafka_offset_broker_commit_cb, NULL);
rd_kafka_topic_partition_list_destroy(offsets);
return RD_KAFKA_RESP_ERR__IN_PROGRESS;
}
static void assign_consume_many (char **topics, int topic_cnt, uint64_t testid){
rd_kafka_t *rk;
rd_kafka_topic_partition_list_t *parts;
int i;
test_msgver_t mv;
TEST_SAY(_C_MAG "%s\n" _C_CLR, __FUNCTION__);
test_conf_init(NULL, NULL, 60);
rk = test_create_consumer(__FUNCTION__, NULL, NULL, NULL);
parts = rd_kafka_topic_partition_list_new(topic_cnt);
for (i = 0 ; i < topic_cnt ; i++)
rd_kafka_topic_partition_list_add(parts, topics[i], 0)->
offset = RD_KAFKA_OFFSET_TAIL(msgs_per_topic);
test_consumer_assign("consume.assign", rk, parts);
rd_kafka_topic_partition_list_destroy(parts);
test_msgver_init(&mv, testid);
test_consumer_poll("consume.assign", rk, testid,
-1, 0, msgs_per_topic * topic_cnt, &mv);
for (i = 0 ; i < topic_cnt ; i++)
test_msgver_verify_part("assign", &mv, TEST_MSGVER_ALL_PART,
topics[i], 0, i * msgs_per_topic,
msgs_per_topic);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
}
static PyObject *Consumer_assign (Handle *self, PyObject *tlist) {
rd_kafka_topic_partition_list_t *c_parts;
rd_kafka_resp_err_t err;
if (!self->rk) {
PyErr_SetString(PyExc_RuntimeError,
"Consumer closed");
return NULL;
}
if (!(c_parts = py_to_c_parts(tlist)))
return NULL;
self->u.Consumer.rebalance_assigned++;
err = rd_kafka_assign(self->rk, c_parts);
rd_kafka_topic_partition_list_destroy(c_parts);
if (err) {
cfl_PyErr_Format(err,
"Failed to set assignment: %s",
rd_kafka_err2str(err));
return NULL;
}
Py_RETURN_NONE;
}
static PyObject *Consumer_committed (Handle *self, PyObject *args,
PyObject *kwargs) {
PyObject *plist;
rd_kafka_topic_partition_list_t *c_parts;
rd_kafka_resp_err_t err;
double tmout = -1.0f;
static char *kws[] = { "partitions", "timeout", NULL };
if (!self->rk) {
PyErr_SetString(PyExc_RuntimeError,
"Consumer closed");
return NULL;
}
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|d", kws,
&plist, &tmout))
return NULL;
if (!(c_parts = py_to_c_parts(plist)))
return NULL;
Py_BEGIN_ALLOW_THREADS;
err = rd_kafka_committed(self->rk, c_parts,
tmout >= 0 ? (int)(tmout * 1000.0f) : -1);
Py_END_ALLOW_THREADS;
if (err) {
rd_kafka_topic_partition_list_destroy(c_parts);
cfl_PyErr_Format(err,
"Failed to get committed offsets: %s",
rd_kafka_err2str(err));
return NULL;
}
plist = c_parts_to_py(c_parts);
rd_kafka_topic_partition_list_destroy(c_parts);
return plist;
}
RdKafka::ErrorCode
RdKafka::KafkaConsumerImpl::assign (const std::vector<TopicPartition*> &partitions) {
rd_kafka_topic_partition_list_t *c_parts;
rd_kafka_resp_err_t err;
c_parts = partitions_to_c_parts(partitions);
err = rd_kafka_assign(rk_, c_parts);
rd_kafka_topic_partition_list_destroy(c_parts);
return static_cast<RdKafka::ErrorCode>(err);
}
static PyObject *Consumer_position (Handle *self, PyObject *args,
PyObject *kwargs) {
PyObject *plist;
rd_kafka_topic_partition_list_t *c_parts;
rd_kafka_resp_err_t err;
static char *kws[] = { "partitions", NULL };
if (!self->rk) {
PyErr_SetString(PyExc_RuntimeError,
"Consumer closed");
return NULL;
}
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", kws,
&plist))
return NULL;
if (!(c_parts = py_to_c_parts(plist)))
return NULL;
err = rd_kafka_position(self->rk, c_parts);
if (err) {
rd_kafka_topic_partition_list_destroy(c_parts);
cfl_PyErr_Format(err,
"Failed to get position: %s",
rd_kafka_err2str(err));
return NULL;
}
plist = c_parts_to_py(c_parts);
rd_kafka_topic_partition_list_destroy(c_parts);
return plist;
}
RdKafka::ErrorCode
RdKafka::KafkaConsumerImpl::subscription (std::vector<std::string> &topics) {
rd_kafka_topic_partition_list_t *c_topics;
rd_kafka_resp_err_t err;
if ((err = rd_kafka_subscription(rk_, &c_topics)))
return static_cast<RdKafka::ErrorCode>(err);
topics.resize(c_topics->cnt);
for (int i = 0 ; i < c_topics->cnt ; i++)
topics[i] = std::string(c_topics->elems[i].topic);
rd_kafka_topic_partition_list_destroy(c_topics);
return RdKafka::ERR_NO_ERROR;
}
RdKafka::ErrorCode
RdKafka::HandleImpl::resume (std::vector<RdKafka::TopicPartition*> &partitions) {
rd_kafka_topic_partition_list_t *c_parts;
rd_kafka_resp_err_t err;
c_parts = partitions_to_c_parts(partitions);
err = rd_kafka_resume_partitions(rk_, c_parts);
if (!err)
update_partitions_from_c_parts(partitions, c_parts);
rd_kafka_topic_partition_list_destroy(c_parts);
return static_cast<RdKafka::ErrorCode>(err);
}
/**
* Trigger offset_commit_cb op, if configured.
* Takes ownership of `offsets`
*/
void rd_kafka_offset_commit_cb_op (rd_kafka_t *rk,
rd_kafka_resp_err_t err,
rd_kafka_topic_partition_list_t *offsets) {
rd_kafka_op_t *rko;
if (!rk->rk_conf.offset_commit_cb) {
rd_kafka_topic_partition_list_destroy(offsets);
return;
}
rko = rd_kafka_op_new(RD_KAFKA_OP_OFFSET_COMMIT|RD_KAFKA_OP_REPLY);
rko->rko_err = err;
rd_kafka_assert(NULL, offsets->cnt > 0);
rd_kafka_op_payload_set(rko, offsets,
(void *)rd_kafka_topic_partition_list_destroy);
rd_kafka_q_enq(&rk->rk_rep, rko);
}
RdKafka::ErrorCode
RdKafka::KafkaConsumerImpl::subscribe (const std::vector<std::string> &topics) {
rd_kafka_topic_partition_list_t *c_topics;
rd_kafka_resp_err_t err;
c_topics = rd_kafka_topic_partition_list_new((int)topics.size());
for (unsigned int i = 0 ; i < topics.size() ; i++)
rd_kafka_topic_partition_list_add(c_topics, topics[i].c_str(),
RD_KAFKA_PARTITION_UA);
err = rd_kafka_subscribe(rk_, c_topics);
rd_kafka_topic_partition_list_destroy(c_topics);
return static_cast<RdKafka::ErrorCode>(err);
}
RdKafka::ErrorCode
RdKafka::KafkaConsumerImpl::assignment (std::vector<RdKafka::TopicPartition*> &partitions) {
rd_kafka_topic_partition_list_t *c_parts;
rd_kafka_resp_err_t err;
if ((err = rd_kafka_assignment(rk_, &c_parts)))
return static_cast<RdKafka::ErrorCode>(err);
partitions.resize(c_parts->cnt);
for (int i = 0 ; i < c_parts->cnt ; i++)
partitions[i] = new RdKafka::TopicPartitionImpl(&c_parts->elems[i]);
rd_kafka_topic_partition_list_destroy(c_parts);
return RdKafka::ERR_NO_ERROR;
}
RdKafka::ErrorCode
RdKafka::KafkaConsumerImpl::committed (std::vector<RdKafka::TopicPartition*> &partitions, int timeout_ms) {
rd_kafka_topic_partition_list_t *c_parts;
rd_kafka_resp_err_t err;
c_parts = partitions_to_c_parts(partitions);
err = rd_kafka_committed(rk_, c_parts, timeout_ms);
if (!err) {
update_partitions_from_c_parts(partitions, c_parts);
}
rd_kafka_topic_partition_list_destroy(c_parts);
return static_cast<RdKafka::ErrorCode>(err);
}
static void subscribe_consume_many (char **topics, int topic_cnt,
uint64_t testid) {
rd_kafka_t *rk;
int i;
rd_kafka_topic_conf_t *tconf;
rd_kafka_topic_partition_list_t *parts;
rd_kafka_resp_err_t err;
test_msgver_t mv;
TEST_SAY(_C_MAG "%s\n" _C_CLR, __FUNCTION__);
test_conf_init(NULL, &tconf, 60);
test_topic_conf_set(tconf, "auto.offset.reset", "earliest");
rk = test_create_consumer(__FUNCTION__, NULL, NULL, tconf);
parts = rd_kafka_topic_partition_list_new(topic_cnt);
for (i = 0 ; i < topic_cnt ; i++)
rd_kafka_topic_partition_list_add(parts, topics[i],
RD_KAFKA_PARTITION_UA);
TEST_SAY("Subscribing to %d topics\n", topic_cnt);
err = rd_kafka_subscribe(rk, parts);
if (err)
TEST_FAIL("subscribe() failed: %s\n", rd_kafka_err2str(err));
rd_kafka_topic_partition_list_destroy(parts);
test_msgver_init(&mv, testid);
test_consumer_poll("consume.subscribe", rk, testid,
-1, 0, msgs_per_topic * topic_cnt, &mv);
for (i = 0 ; i < topic_cnt ; i++)
test_msgver_verify_part("subscribe", &mv, TEST_MSGVER_ALL_PART,
topics[i], 0, i * msgs_per_topic,
msgs_per_topic);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
}
/**
* Create high-level consumer subscribing to \p topic from BEGINNING
* and expects \d exp_msgcnt with matching \p testid
* Destroys consumer when done.
*
* If \p group_id is NULL a new unique group is generated
*/
void
test_consume_msgs_easy (const char *group_id, const char *topic,
uint64_t testid, int exp_msgcnt) {
rd_kafka_t *rk;
rd_kafka_topic_conf_t *tconf;
rd_kafka_resp_err_t err;
rd_kafka_topic_partition_list_t *topics;
char grpid0[64];
test_conf_init(NULL, &tconf, 0);
if (!group_id)
group_id = test_str_id_generate(grpid0, sizeof(grpid0));
test_topic_conf_set(tconf, "auto.offset.reset", "smallest");
rk = test_create_consumer(group_id, NULL, tconf, NULL);
rd_kafka_poll_set_consumer(rk);
topics = rd_kafka_topic_partition_list_new(1);
rd_kafka_topic_partition_list_add(topics, topic, RD_KAFKA_PARTITION_UA);
TEST_SAY("Subscribing to topic %s in group %s "
"(expecting %d msgs with testid %"PRIu64")\n",
topic, group_id, exp_msgcnt, testid);
err = rd_kafka_subscribe(rk, topics);
if (err)
TEST_FAIL("Failed to subscribe to %s: %s\n",
topic, rd_kafka_err2str(err));
rd_kafka_topic_partition_list_destroy(topics);
/* Consume messages */
test_consumer_poll("consume.easy", rk, testid, -1, -1, exp_msgcnt);
test_consumer_close(rk);
rd_kafka_destroy(rk);
}
rd_kafka_resp_err_t
rd_kafka_commit_message (rd_kafka_t *rk, const rd_kafka_message_t *rkmessage,
int async) {
rd_kafka_topic_partition_list_t *offsets;
rd_kafka_topic_partition_t *rktpar;
rd_kafka_resp_err_t err;
if (rkmessage->err)
return RD_KAFKA_RESP_ERR__INVALID_ARG;
offsets = rd_kafka_topic_partition_list_new(1);
rktpar = rd_kafka_topic_partition_list_add(
offsets, rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition);
rktpar->offset = rkmessage->offset+1;
err = rd_kafka_commit(rk, offsets, async);
rd_kafka_topic_partition_list_destroy(offsets);
return err;
}
static void do_consume (const char *topic, int msgcnt) {
rd_kafka_t *rk;
rd_kafka_topic_partition_list_t *parts;
rk = test_create_consumer(topic, NULL, NULL, NULL);
parts = rd_kafka_topic_partition_list_new(1);
rd_kafka_topic_partition_list_add(parts, topic, 0)->offset =
RD_KAFKA_OFFSET_BEGINNING;
test_consumer_assign("assign", rk, parts);
rd_kafka_topic_partition_list_destroy(parts);
exp_msgid = 0;
while (exp_msgid < msgcnt) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(rk, 1000);
if (!rkm)
continue;
if (rkm->err)
TEST_FAIL("consume error while expecting msgid %d/%d: "
"%s",
exp_msgid, msgcnt,
rd_kafka_message_errstr(rkm));
handle_consumed_msg(rkm);
rd_kafka_message_destroy(rkm);
}
test_consumer_close(rk);
rd_kafka_destroy(rk);
}
static PyObject *Consumer_resume (Handle *self, PyObject *args,
PyObject *kwargs) {
PyObject *plist;
rd_kafka_topic_partition_list_t *c_parts;
rd_kafka_resp_err_t err;
static char *kws[] = {"partitions", NULL};
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", kws, &plist))
return NULL;
if (!(c_parts = py_to_c_parts(plist)))
return NULL;
err = rd_kafka_resume_partitions(self->rk, c_parts);
rd_kafka_topic_partition_list_destroy(c_parts);
if (err) {
cfl_PyErr_Format(err,
"Failed to resume partitions: %s",
rd_kafka_err2str(err));
return NULL;
}
Py_RETURN_NONE;
}
int main_0093_holb_consumer (int argc, char **argv) {
const char *topic = test_mk_topic_name("0093_holb_consumer", 1);
int64_t testid;
const int msgcnt = 100;
struct _consumer c[_CONSUMER_CNT] = RD_ZERO_INIT;
rd_kafka_conf_t *conf;
testid = test_id_generate();
test_conf_init(&conf, NULL, 60);
test_create_topic(topic, 1, 1);
test_produce_msgs_easy(topic, testid, 0, msgcnt);
test_conf_set(conf, "session.timeout.ms", "6000");
test_conf_set(conf, "max.poll.interval.ms", "20000");
test_conf_set(conf, "socket.timeout.ms", "3000");
test_conf_set(conf, "auto.offset.reset", "earliest");
/* Trigger other requests often */
test_conf_set(conf, "topic.metadata.refresh.interval.ms", "500");
rd_kafka_conf_set_rebalance_cb(conf, rebalance_cb);
rd_kafka_conf_set_opaque(conf, &c[0]);
c[0].rk = test_create_consumer(topic, NULL,
rd_kafka_conf_dup(conf), NULL);
rd_kafka_conf_set_opaque(conf, &c[1]);
c[1].rk = test_create_consumer(topic, NULL, conf, NULL);
test_consumer_subscribe(c[0].rk, topic);
/* c0: assign */
c[0].max_rebalance_cnt = 1;
/* c1: none, hasn't joined yet */
c[1].max_rebalance_cnt = 0;
TEST_SAY("Waiting for c[0] assignment\n");
while (1) {
rd_kafka_topic_partition_list_t *parts = NULL;
do_consume(&c[0], 1/*1s*/);
if (rd_kafka_assignment(c[0].rk, &parts) !=
RD_KAFKA_RESP_ERR_NO_ERROR ||
!parts || parts->cnt == 0) {
if (parts)
rd_kafka_topic_partition_list_destroy(parts);
continue;
}
TEST_SAY("%s got assignment of %d partition(s)\n",
rd_kafka_name(c[0].rk), parts->cnt);
rd_kafka_topic_partition_list_destroy(parts);
break;
}
TEST_SAY("c[0] got assignment, consuming..\n");
do_consume(&c[0], 5/*5s*/);
TEST_SAY("Joining second consumer\n");
test_consumer_subscribe(c[1].rk, topic);
/* Just poll second consumer for 10s, the rebalance will not
* finish until the first consumer polls */
do_consume(&c[1], 10/*10s*/);
/* c0: the next call to do_consume/poll will trigger
* its rebalance callback, first revoke then assign. */
c[0].max_rebalance_cnt += 2;
/* c1: first rebalance */
c[1].max_rebalance_cnt++;
TEST_SAY("Expected rebalances: c[0]: %d/%d, c[1]: %d/%d\n",
c[0].rebalance_cnt, c[0].max_rebalance_cnt,
c[1].rebalance_cnt, c[1].max_rebalance_cnt);
/* Let rebalances kick in, then consume messages. */
while (c[0].cnt + c[1].cnt < msgcnt) {
do_consume(&c[0], 0);
do_consume(&c[1], 0);
}
/* Allow the extra revoke rebalance on close() */
c[0].max_rebalance_cnt++;
c[1].max_rebalance_cnt++;
test_consumer_close(c[0].rk);
test_consumer_close(c[1].rk);
rd_kafka_destroy(c[0].rk);
rd_kafka_destroy(c[1].rk);
return 0;
}
void rd_kafka_op_destroy (rd_kafka_op_t *rko) {
switch (rko->rko_type & ~RD_KAFKA_OP_FLAGMASK)
{
case RD_KAFKA_OP_FETCH:
rd_kafka_msg_destroy(NULL, &rko->rko_u.fetch.rkm);
/* Decrease refcount on rkbuf to eventually rd_free shared buf*/
if (rko->rko_u.fetch.rkbuf)
rd_kafka_buf_handle_op(rko, RD_KAFKA_RESP_ERR__DESTROY);
break;
case RD_KAFKA_OP_OFFSET_FETCH:
if (rko->rko_u.offset_fetch.partitions &&
rko->rko_u.offset_fetch.do_free)
rd_kafka_topic_partition_list_destroy(
rko->rko_u.offset_fetch.partitions);
break;
case RD_KAFKA_OP_OFFSET_COMMIT:
RD_IF_FREE(rko->rko_u.offset_commit.partitions,
rd_kafka_topic_partition_list_destroy);
RD_IF_FREE(rko->rko_u.offset_commit.reason, rd_free);
break;
case RD_KAFKA_OP_SUBSCRIBE:
case RD_KAFKA_OP_GET_SUBSCRIPTION:
RD_IF_FREE(rko->rko_u.subscribe.topics,
rd_kafka_topic_partition_list_destroy);
break;
case RD_KAFKA_OP_ASSIGN:
case RD_KAFKA_OP_GET_ASSIGNMENT:
RD_IF_FREE(rko->rko_u.assign.partitions,
rd_kafka_topic_partition_list_destroy);
break;
case RD_KAFKA_OP_REBALANCE:
RD_IF_FREE(rko->rko_u.rebalance.partitions,
rd_kafka_topic_partition_list_destroy);
break;
case RD_KAFKA_OP_NAME:
RD_IF_FREE(rko->rko_u.name.str, rd_free);
break;
case RD_KAFKA_OP_ERR:
case RD_KAFKA_OP_CONSUMER_ERR:
RD_IF_FREE(rko->rko_u.err.errstr, rd_free);
rd_kafka_msg_destroy(NULL, &rko->rko_u.err.rkm);
break;
break;
case RD_KAFKA_OP_THROTTLE:
RD_IF_FREE(rko->rko_u.throttle.nodename, rd_free);
break;
case RD_KAFKA_OP_STATS:
RD_IF_FREE(rko->rko_u.stats.json, rd_free);
break;
case RD_KAFKA_OP_XMIT_RETRY:
case RD_KAFKA_OP_XMIT_BUF:
case RD_KAFKA_OP_RECV_BUF:
if (rko->rko_u.xbuf.rkbuf)
rd_kafka_buf_handle_op(rko, RD_KAFKA_RESP_ERR__DESTROY);
RD_IF_FREE(rko->rko_u.xbuf.rkbuf, rd_kafka_buf_destroy);
break;
case RD_KAFKA_OP_DR:
rd_kafka_msgq_purge(rko->rko_rk, &rko->rko_u.dr.msgq);
if (rko->rko_u.dr.do_purge2)
rd_kafka_msgq_purge(rko->rko_rk, &rko->rko_u.dr.msgq2);
if (rko->rko_u.dr.s_rkt)
rd_kafka_topic_destroy0(rko->rko_u.dr.s_rkt);
break;
case RD_KAFKA_OP_OFFSET_RESET:
RD_IF_FREE(rko->rko_u.offset_reset.reason, rd_free);
break;
case RD_KAFKA_OP_METADATA:
RD_IF_FREE(rko->rko_u.metadata.md, rd_kafka_metadata_destroy);
break;
case RD_KAFKA_OP_LOG:
rd_free(rko->rko_u.log.str);
break;
default:
break;
}
if (rko->rko_type & RD_KAFKA_OP_CB && rko->rko_op_cb) {
rd_kafka_op_res_t res;
/* Let callback clean up */
rko->rko_err = RD_KAFKA_RESP_ERR__DESTROY;
res = rko->rko_op_cb(rko->rko_rk, NULL, rko);
assert(res != RD_KAFKA_OP_RES_YIELD);
}
RD_IF_FREE(rko->rko_rktp, rd_kafka_toppar_destroy);
rd_kafka_replyq_destroy(&rko->rko_replyq);
#if ENABLE_DEVEL
if (rd_atomic32_sub(&rd_kafka_op_cnt, 1) < 0)
rd_kafka_assert(NULL, !*"rd_kafka_op_cnt < 0");
#endif
rd_free(rko);
}
/* @remark This test will fail if auto topic creation is enabled on the broker
* since the client will issue a topic-creating metadata request to find
* a new leader when the topic is removed.
*
* To run with trivup, do:
* ./interactive_broker_version.py .. -conf '{"auto_create_topics":"false"}' ..
* TESTS=0045 ./run-test.sh -k ./merged
*/
static void do_test_topic_remove (void) {
char *topic_f = rd_strdup(test_mk_topic_name("topic_f", 1));
char *topic_g = rd_strdup(test_mk_topic_name("topic_g", 1));
int parts_f = 5;
int parts_g = 9;
rd_kafka_t *rk;
rd_kafka_conf_t *conf;
rd_kafka_queue_t *queue;
rd_kafka_topic_partition_list_t *topics;
rd_kafka_resp_err_t err;
/**
* Topic removal test:
* - Create topic f & g
* - Subscribe to f & g
* - Verify f & g assignment
* - Remove topic f
* - Verify g assignment
* - Remove topic g
* - Verify empty assignment
*/
TEST_SAY("Topic removal testing\n");
test_conf_init(&conf, NULL, 60);
/* Decrease metadata interval to speed up topic change discovery. */
test_conf_set(conf, "metadata.max.age.ms", "5000");
rd_kafka_conf_set_events(conf, RD_KAFKA_EVENT_REBALANCE);
rk = test_create_consumer(test_str_id_generate_tmp(),
NULL, conf, NULL, NULL);
queue = rd_kafka_queue_get_consumer(rk);
TEST_SAY("Topic removal: creating topic %s (subscribed)\n", topic_f);
test_create_topic(topic_f, parts_f, 1);
TEST_SAY("Topic removal: creating topic %s (subscribed)\n", topic_g);
test_create_topic(topic_g, parts_g, 1);
rd_sleep(1); // FIXME: do check&wait loop instead
TEST_SAY("Topic removal: Subscribing to %s & %s\n", topic_f, topic_g);
topics = rd_kafka_topic_partition_list_new(2);
rd_kafka_topic_partition_list_add(topics, topic_f, RD_KAFKA_PARTITION_UA);
rd_kafka_topic_partition_list_add(topics, topic_g, RD_KAFKA_PARTITION_UA);
err = rd_kafka_subscribe(rk, topics);
TEST_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR,
"%s", rd_kafka_err2str(err));
rd_kafka_topic_partition_list_destroy(topics);
await_assignment("Topic removal: both topics exist", rk, queue, 2,
topic_f, parts_f,
topic_g, parts_g);
TEST_SAY("Topic removal: removing %s\n", topic_f);
test_kafka_topics("--delete --topic %s", topic_f);
await_revoke("Topic removal: rebalance after topic removal", rk, queue);
await_assignment("Topic removal: one topic exists", rk, queue, 1,
topic_g, parts_g);
TEST_SAY("Topic removal: removing %s\n", topic_g);
test_kafka_topics("--delete --topic %s", topic_g);
await_revoke("Topic removal: rebalance after 2nd topic removal",
rk, queue);
/* Should not see another rebalance since all topics now removed */
await_no_rebalance("Topic removal: empty", rk, queue, 10000);
test_consumer_close(rk);
rd_kafka_queue_destroy(queue);
rd_kafka_destroy(rk);
rd_free(topic_f);
rd_free(topic_g);
}
int main_0029_assign_offset (int argc, char **argv) {
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_topic_conf_t *tconf;
rd_kafka_topic_partition_list_t *parts;
uint64_t testid;
int i;
test_timing_t t_simple, t_hl;
test_msgver_t mv;
test_conf_init(NULL, &tconf, 20 + (test_session_timeout_ms * 3 / 1000));
test_topic_conf_set(tconf, "auto.offset.reset", "smallest");
/* Produce X messages to Y partitions so we get a
* nice seekable 0..X offset one each partition. */
/* Produce messages */
testid = test_id_generate();
rk = test_create_producer();
rkt = test_create_producer_topic(rk, topic, NULL);
parts = rd_kafka_topic_partition_list_new(partitions);
for (i = 0 ; i < partitions ; i++) {
test_produce_msgs(rk, rkt, testid, i, 0, msgcnt, NULL, 0);
rd_kafka_topic_partition_list_add(parts, topic, i)->offset =
msgcnt / 2;
}
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
/* Simple consumer */
TIMING_START(&t_simple, "SIMPLE.CONSUMER");
rk = test_create_consumer(topic, NULL, NULL, NULL);
test_msgver_init(&mv, testid);
test_consumer_assign("SIMPLE.ASSIGN", rk, parts);
test_consumer_poll("SIMPLE.CONSUME", rk, testid, -1, 0,
partitions * (msgcnt / 2), &mv);
for (i = 0 ; i < partitions ; i++)
test_msgver_verify_part("HL.MSGS", &mv, TEST_MSGVER_ALL_PART,
topic, i, msgcnt/2, msgcnt/2);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_simple);
rd_kafka_topic_partition_list_destroy(parts);
/* High-level consumer.
* Offsets are set in rebalance callback. */
TIMING_START(&t_hl, "HL.CONSUMER");
test_msgver_init(&mv, testid);
rk = test_create_consumer(topic, rebalance_cb, NULL, NULL);
test_consumer_subscribe(rk, topic);
test_consumer_poll("HL.CONSUME", rk, testid, -1, 0,
partitions * (msgcnt / 2), &mv);
for (i = 0 ; i < partitions ; i++)
test_msgver_verify_part("HL.MSGS", &mv, TEST_MSGVER_ALL_PART,
topic, i, msgcnt/2, msgcnt/2);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_hl);
rd_kafka_topic_conf_destroy(tconf);
return 0;
}
int main_0029_assign_offset (int argc, char **argv) {
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_topic_partition_list_t *parts;
uint64_t testid;
int i;
test_timing_t t_simple, t_hl;
test_msgver_t mv;
test_conf_init(NULL, NULL, 20 + (test_session_timeout_ms * 3 / 1000));
/* Produce X messages to Y partitions so we get a
* nice seekable 0..X offset one each partition. */
/* Produce messages */
testid = test_id_generate();
rk = test_create_producer();
rkt = test_create_producer_topic(rk, topic, NULL);
parts = rd_kafka_topic_partition_list_new(partitions);
for (i = 0 ; i < partitions ; i++) {
test_produce_msgs(rk, rkt, testid, i, 0, msgcnt, NULL, 0);
/* Set start offset */
rd_kafka_topic_partition_list_add(parts, topic, i)->offset =
msgcnt / 2;
}
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
/* Simple consumer */
TIMING_START(&t_simple, "SIMPLE.CONSUMER");
rk = test_create_consumer(topic, NULL, NULL, NULL);
test_msgver_init(&mv, testid);
test_consumer_assign("SIMPLE.ASSIGN", rk, parts);
test_consumer_poll("SIMPLE.CONSUME", rk, testid, -1, 0,
partitions * (msgcnt / 2), &mv);
for (i = 0 ; i < partitions ; i++)
test_msgver_verify_part("HL.MSGS", &mv, TEST_MSGVER_ALL_PART,
topic, i, msgcnt/2, msgcnt/2);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_simple);
rd_kafka_topic_partition_list_destroy(parts);
/* High-level consumer: method 1
* Offsets are set in rebalance callback. */
if (test_broker_version >= TEST_BRKVER(0,9,0,0)) {
reb_method = REB_METHOD_1;
TIMING_START(&t_hl, "HL.CONSUMER");
test_msgver_init(&mv, testid);
rk = test_create_consumer(topic, rebalance_cb, NULL, NULL);
test_consumer_subscribe(rk, topic);
test_consumer_poll("HL.CONSUME", rk, testid, -1, 0,
partitions * (msgcnt / 2), &mv);
for (i = 0 ; i < partitions ; i++)
test_msgver_verify_part("HL.MSGS", &mv,
TEST_MSGVER_ALL_PART,
topic, i, msgcnt/2, msgcnt/2);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_hl);
/* High-level consumer: method 2:
* first two partitions are with fixed absolute offset, rest are
* auto offset (stored, which is now at end).
* Offsets are set in rebalance callback. */
reb_method = REB_METHOD_2;
TIMING_START(&t_hl, "HL.CONSUMER2");
test_msgver_init(&mv, testid);
rk = test_create_consumer(topic, rebalance_cb, NULL, NULL);
test_consumer_subscribe(rk, topic);
test_consumer_poll("HL.CONSUME2", rk, testid, partitions, 0,
2 * (msgcnt / 2), &mv);
for (i = 0 ; i < partitions ; i++) {
if (i < 2)
test_msgver_verify_part("HL.MSGS2.A", &mv,
TEST_MSGVER_ALL_PART,
topic, i, msgcnt/2,
msgcnt/2);
}
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_hl);
}
return 0;
}
int main_0019_list_groups (int argc, char **argv) {
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
#define _CONS_CNT 2
char *groups[_CONS_CNT];
rd_kafka_t *rk, *rk_c[_CONS_CNT];
rd_kafka_topic_partition_list_t *topics;
rd_kafka_resp_err_t err;
test_timing_t t_grps;
int i;
int groups_seen;
rd_kafka_topic_t *rkt;
/* Handle for group listings */
rk = test_create_producer();
/* Produce messages so that topic is auto created */
rkt = test_create_topic_object(rk, topic, NULL);
test_produce_msgs(rk, rkt, 0, 0, 0, 10, NULL, 64);
rd_kafka_topic_destroy(rkt);
/* Query groups before creation, should not list our groups. */
groups_seen = list_groups(rk, NULL, 0, "should be none");
if (groups_seen != 0)
TEST_FAIL("Saw %d groups when there wasn't "
"supposed to be any\n", groups_seen);
/* Fill in topic subscription set */
topics = rd_kafka_topic_partition_list_new(1);
rd_kafka_topic_partition_list_add(topics, topic, -1);
/* Create consumers and start subscription */
for (i = 0 ; i < _CONS_CNT ; i++) {
groups[i] = malloc(32);
test_str_id_generate(groups[i], 32);
rk_c[i] = test_create_consumer(groups[i],
NULL, NULL, NULL);
err = rd_kafka_poll_set_consumer(rk_c[i]);
if (err)
TEST_FAIL("poll_set_consumer: %s\n",
rd_kafka_err2str(err));
err = rd_kafka_subscribe(rk_c[i], topics);
if (err)
TEST_FAIL("subscribe: %s\n", rd_kafka_err2str(err));
}
rd_kafka_topic_partition_list_destroy(topics);
TIMING_START(&t_grps, "WAIT.GROUPS");
/* Query groups again until both groups are seen. */
while (1) {
int groups_seen = list_groups(rk, (char **)groups, _CONS_CNT,
"should see my groups");
if (groups_seen == _CONS_CNT)
break;
rd_sleep(1);
}
TIMING_STOP(&t_grps);
TEST_SAY("Closing remaining consumers\n");
for (i = 0 ; i < _CONS_CNT ; i++) {
test_timing_t t_close;
if (!rk_c[i])
continue;
TEST_SAY("Closing %s\n", rd_kafka_name(rk_c[i]));
TIMING_START(&t_close, "CONSUMER.CLOSE");
err = rd_kafka_consumer_close(rk_c[i]);
TIMING_STOP(&t_close);
if (err)
TEST_FAIL("consumer_close failed: %s\n",
rd_kafka_err2str(err));
rd_kafka_destroy(rk_c[i]);
rk_c[i] = NULL;
free(groups[i]);
}
rd_kafka_destroy(rk);
return 0;
}
int main (int argc, char **argv) {
char mode = 'C';
char *brokers = "localhost:9092";
int opt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char errstr[512];
const char *debug = NULL;
int do_conf_dump = 0;
char tmp[16];
rd_kafka_resp_err_t err;
char *group = NULL;
rd_kafka_topic_partition_list_t *topics;
int i;
quiet = !isatty(STDIN_FILENO);
/* Kafka configuration */
conf = rd_kafka_conf_new();
/* Set logger */
rd_kafka_conf_set_log_cb(conf, logger);
/* Quick termination */
snprintf(tmp, sizeof(tmp), "%i", SIGIO);
rd_kafka_conf_set(conf, "internal.termination.signal", tmp, NULL, 0);
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
while ((opt = getopt(argc, argv, "g:b:qd:eX:As:DO")) != -1) {
switch (opt) {
case 'b':
brokers = optarg;
break;
case 'g':
group = optarg;
break;
case 'e':
exit_eof = 1;
break;
case 'd':
debug = optarg;
break;
case 'q':
quiet = 1;
break;
case 'A':
output = OUTPUT_RAW;
break;
case 'X':
{
char *name, *val;
rd_kafka_conf_res_t res;
if (!strcmp(optarg, "list") ||
!strcmp(optarg, "help")) {
rd_kafka_conf_properties_show(stdout);
exit(0);
}
if (!strcmp(optarg, "dump")) {
do_conf_dump = 1;
continue;
}
name = optarg;
if (!(val = strchr(name, '='))) {
fprintf(stderr, "%% Expected "
"-X property=value, not %s\n", name);
exit(1);
}
*val = '\0';
val++;
res = RD_KAFKA_CONF_UNKNOWN;
/* Try "topic." prefixed properties on topic
* conf first, and then fall through to global if
* it didnt match a topic configuration property. */
if (!strncmp(name, "topic.", strlen("topic.")))
res = rd_kafka_topic_conf_set(topic_conf,
name+
strlen("topic."),
val,
errstr,
sizeof(errstr));
if (res == RD_KAFKA_CONF_UNKNOWN)
res = rd_kafka_conf_set(conf, name, val,
errstr, sizeof(errstr));
if (res != RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
}
break;
case 'D':
case 'O':
mode = opt;
break;
default:
goto usage;
}
}
if (do_conf_dump) {
const char **arr;
size_t cnt;
int pass;
for (pass = 0 ; pass < 2 ; pass++) {
if (pass == 0) {
arr = rd_kafka_conf_dump(conf, &cnt);
printf("# Global config\n");
} else {
printf("# Topic config\n");
arr = rd_kafka_topic_conf_dump(topic_conf,
&cnt);
}
for (i = 0 ; i < (int)cnt ; i += 2)
printf("%s = %s\n",
arr[i], arr[i+1]);
printf("\n");
rd_kafka_conf_dump_free(arr, cnt);
}
exit(0);
}
if (strchr("OC", mode) && optind == argc) {
usage:
fprintf(stderr,
"Usage: %s [options] <topic[:part]> <topic[:part]>..\n"
"\n"
"librdkafka version %s (0x%08x)\n"
"\n"
" Options:\n"
" -g <group> Consumer group (%s)\n"
" -b <brokers> Broker address (%s)\n"
" -e Exit consumer when last message\n"
" in partition has been received.\n"
" -D Describe group.\n"
" -O Get commmitted offset(s)\n"
" -d [facs..] Enable debugging contexts:\n"
" %s\n"
" -q Be quiet\n"
" -A Raw payload output (consumer)\n"
" -X <prop=name> Set arbitrary librdkafka "
"configuration property\n"
" Properties prefixed with \"topic.\" "
"will be set on topic object.\n"
" Use '-X list' to see the full list\n"
" of supported properties.\n"
"\n"
"\n",
argv[0],
rd_kafka_version_str(), rd_kafka_version(),
group, brokers,
RD_KAFKA_DEBUG_CONTEXTS);
exit(1);
}
signal(SIGINT, stop);
signal(SIGUSR1, sig_usr1);
if (debug &&
rd_kafka_conf_set(conf, "debug", debug, errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% Debug configuration failed: %s: %s\n",
errstr, debug);
exit(1);
}
/*
* Client/Consumer group
*/
if (strchr("CO", mode)) {
/* Consumer groups require a group id */
if (!group)
group = "rdkafka_consumer_example";
if (rd_kafka_conf_set(conf, "group.id", group,
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
/* Consumer groups always use broker based offset storage */
if (rd_kafka_topic_conf_set(topic_conf, "offset.store.method",
"broker",
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
/* Set default topic config for pattern-matched topics. */
rd_kafka_conf_set_default_topic_conf(conf, topic_conf);
/* Callback called on partition assignment changes */
rd_kafka_conf_set_rebalance_cb(conf, rebalance_cb);
}
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new consumer: %s\n",
errstr);
exit(1);
}
rd_kafka_set_log_level(rk, LOG_DEBUG);
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
if (mode == 'D') {
int r;
/* Describe groups */
r = describe_groups(rk, group);
rd_kafka_destroy(rk);
exit(r == -1 ? 1 : 0);
}
/* Redirect rd_kafka_poll() to consumer_poll() */
rd_kafka_poll_set_consumer(rk);
topics = rd_kafka_topic_partition_list_new(argc - optind);
for (i = optind ; i < argc ; i++) {
/* Parse "topic[:part] */
char *topic = argv[i];
char *t;
int32_t partition = -1;
if ((t = strstr(topic, ":"))) {
*t = '\0';
partition = atoi(t+1);
}
rd_kafka_topic_partition_list_add(topics, topic, partition);
}
if (mode == 'O') {
/* Offset query */
err = rd_kafka_position(rk, topics, 5000);
if (err) {
fprintf(stderr, "%% Failed to fetch offsets: %s\n",
rd_kafka_err2str(err));
exit(1);
}
for (i = 0 ; i < topics->cnt ; i++) {
rd_kafka_topic_partition_t *p = &topics->elems[i];
printf("Topic \"%s\" partition %"PRId32,
p->topic, p->partition);
if (p->err)
printf(" error %s",
rd_kafka_err2str(p->err));
else {
printf(" offset %"PRId64"",
p->offset);
if (p->metadata_size)
printf(" (%d bytes of metadata)",
(int)p->metadata_size);
}
printf("\n");
}
goto done;
}
if ((err = rd_kafka_subscribe(rk, topics))) {
fprintf(stderr, "%% Failed to start consuming topics: %s\n",
rd_kafka_err2str(err));
exit(1);
}
while (run) {
rd_kafka_message_t *rkmessage;
rkmessage = rd_kafka_consumer_poll(rk, 1000);
if (rkmessage) {
msg_consume(rkmessage, NULL);
rd_kafka_message_destroy(rkmessage);
}
}
done:
err = rd_kafka_consumer_close(rk);
if (err)
fprintf(stderr, "%% Failed to close consumer: %s\n",
rd_kafka_err2str(err));
else
fprintf(stderr, "%% Consumer closed\n");
rd_kafka_topic_partition_list_destroy(topics);
/* Destroy handle */
rd_kafka_destroy(rk);
/* Let background threads clean up and terminate cleanly. */
run = 5;
while (run-- > 0 && rd_kafka_wait_destroyed(1000) == -1)
printf("Waiting for librdkafka to decommission\n");
if (run <= 0)
rd_kafka_dump(stdout, rk);
return 0;
}
static void do_offset_test (const char *what, int auto_commit, int auto_store,
int async) {
test_timing_t t_all;
char groupid[64];
rd_kafka_t *rk;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *tconf;
int cnt = 0;
const int extra_cnt = 5;
rd_kafka_resp_err_t err;
rd_kafka_topic_partition_list_t *parts;
rd_kafka_topic_partition_t *rktpar;
int64_t next_offset = -1;
test_conf_init(&conf, &tconf, 20);
test_conf_set(conf, "enable.auto.commit", auto_commit ? "true":"false");
test_conf_set(conf, "enable.auto.offset.store", auto_store ?"true":"false");
test_conf_set(conf, "auto.commit.interval.ms", "500");
rd_kafka_conf_set_offset_commit_cb(conf, offset_commit_cb);
test_topic_conf_set(tconf, "auto.offset.reset", "smallest");
test_str_id_generate(groupid, sizeof(groupid));
test_conf_set(conf, "group.id", groupid);
rd_kafka_conf_set_default_topic_conf(conf, tconf);
TEST_SAY(_C_MAG "[ do_offset_test: %s with group.id %s ]\n",
what, groupid);
TIMING_START(&t_all, what);
expected_offset = 0;
committed_offset = -1;
/* MO:
* - Create consumer.
* - Start consuming from beginning
* - Perform store & commits according to settings
* - Stop storing&committing when half of the messages are consumed,
* - but consume 5 more to check against.
* - Query position.
* - Destroy consumer.
* - Create new consumer with same group.id using stored offsets
* - Should consume the expected message.
*/
/* Create kafka instance */
rk = test_create_handle(RD_KAFKA_CONSUMER, rd_kafka_conf_dup(conf));
rd_kafka_poll_set_consumer(rk);
test_consumer_subscribe(rk, topic);
while (cnt - extra_cnt < msgcnt / 2) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(rk, 10*1000);
if (!rkm)
continue;
if (rkm->err == RD_KAFKA_RESP_ERR__TIMED_OUT)
TEST_FAIL("%s: Timed out waiting for message %d", what,cnt);
else if (rkm->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
rd_kafka_message_destroy(rkm);
continue;
} else if (rkm->err)
TEST_FAIL("%s: Consumer error: %s",
what, rd_kafka_message_errstr(rkm));
/* Offset of next message. */
next_offset = rkm->offset + 1;
if (cnt < msgcnt / 2) {
if (!auto_store) {
err = rd_kafka_offset_store(rkm->rkt,rkm->partition,
rkm->offset);
if (err)
TEST_FAIL("%s: offset_store failed: %s\n",
what, rd_kafka_err2str(err));
}
expected_offset = rkm->offset+1;
if (!auto_commit) {
test_timing_t t_commit;
TIMING_START(&t_commit,
async?"commit.async":"commit.sync");
err = rd_kafka_commit_message(rk, rkm, async);
TIMING_STOP(&t_commit);
if (err)
TEST_FAIL("%s: commit failed: %s\n",
what, rd_kafka_err2str(err));
}
} else if (auto_store && auto_commit)
expected_offset = rkm->offset+1;
rd_kafka_message_destroy(rkm);
cnt++;
}
TEST_SAY("%s: done consuming after %d messages, at offset %"PRId64"\n",
what, cnt, expected_offset);
if ((err = rd_kafka_assignment(rk, &parts)))
TEST_FAIL("%s: failed to get assignment(): %s\n",
what, rd_kafka_err2str(err));
/* Verify position */
if ((err = rd_kafka_position(rk, parts)))
TEST_FAIL("%s: failed to get position(): %s\n",
what, rd_kafka_err2str(err));
if (!(rktpar = rd_kafka_topic_partition_list_find(parts,
topic, partition)))
TEST_FAIL("%s: position(): topic lost\n", what);
if (rktpar->offset != next_offset)
TEST_FAIL("%s: Expected position() offset %"PRId64", got %"PRId64,
what, next_offset, rktpar->offset);
TEST_SAY("%s: Position is at %"PRId64", good!\n",
what, rktpar->offset);
/* Pause messages while waiting so we can serve callbacks
* without having more messages received. */
if ((err = rd_kafka_pause_partitions(rk, parts)))
TEST_FAIL("%s: failed to pause partitions: %s\n",
what, rd_kafka_err2str(err));
rd_kafka_topic_partition_list_destroy(parts);
/* Fire off any enqueued offset_commit_cb */
test_consumer_poll_no_msgs(what, rk, testid, 0);
TEST_SAY("%s: committed_offset %"PRId64", expected_offset %"PRId64"\n",
what, committed_offset, expected_offset);
if (!auto_commit && !async) {
/* Sync commits should be up to date at this point. */
if (committed_offset != expected_offset)
TEST_FAIL("%s: Sync commit: committed offset %"PRId64
" should be same as expected offset "
"%"PRId64,
what, committed_offset, expected_offset);
} else {
/* Wait for offset commits to catch up */
while (committed_offset < expected_offset) {
TEST_SAYL(3, "%s: Wait for committed offset %"PRId64
" to reach expected offset %"PRId64"\n",
what, committed_offset, expected_offset);
test_consumer_poll_no_msgs(what, rk, testid, 1000);
}
}
TEST_SAY("%s: phase 1 complete, %d messages consumed, "
"next expected offset is %"PRId64"\n",
what, cnt, expected_offset);
/* Issue #827: cause committed() to return prematurely by specifying
* low timeout. The bug (use after free) will only
* be catched by valgrind. */
do {
parts = rd_kafka_topic_partition_list_new(1);
rd_kafka_topic_partition_list_add(parts, topic, partition);
err = rd_kafka_committed(rk, parts, 1);
rd_kafka_topic_partition_list_destroy(parts);
TEST_SAY("Issue #827: committed() returned %s\n",
rd_kafka_err2str(err));
} while (err != RD_KAFKA_RESP_ERR__TIMED_OUT);
/* Query position */
parts = rd_kafka_topic_partition_list_new(1);
rd_kafka_topic_partition_list_add(parts, topic, partition);
err = rd_kafka_committed(rk, parts, tmout_multip(5*1000));
if (err)
TEST_FAIL("%s: committed() failed: %s", what, rd_kafka_err2str(err));
if (!(rktpar = rd_kafka_topic_partition_list_find(parts,
topic, partition)))
TEST_FAIL("%s: committed(): topic lost\n", what);
if (rktpar->offset != expected_offset)
TEST_FAIL("%s: Expected committed() offset %"PRId64", got %"PRId64,
what, expected_offset, rktpar->offset);
TEST_SAY("%s: Committed offset is at %"PRId64", good!\n",
what, rktpar->offset);
rd_kafka_topic_partition_list_destroy(parts);
test_consumer_close(rk);
rd_kafka_destroy(rk);
/* Fire up a new consumer and continue from where we left off. */
TEST_SAY("%s: phase 2: starting new consumer to resume consumption\n",what);
rk = test_create_handle(RD_KAFKA_CONSUMER, conf);
rd_kafka_poll_set_consumer(rk);
test_consumer_subscribe(rk, topic);
while (cnt < msgcnt) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(rk, 10*1000);
if (!rkm)
continue;
if (rkm->err == RD_KAFKA_RESP_ERR__TIMED_OUT)
TEST_FAIL("%s: Timed out waiting for message %d", what,cnt);
else if (rkm->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
rd_kafka_message_destroy(rkm);
continue;
} else if (rkm->err)
TEST_FAIL("%s: Consumer error: %s",
what, rd_kafka_message_errstr(rkm));
if (rkm->offset != expected_offset)
TEST_FAIL("%s: Received message offset %"PRId64
", expected %"PRId64" at msgcnt %d/%d\n",
what, rkm->offset, expected_offset,
cnt, msgcnt);
rd_kafka_message_destroy(rkm);
expected_offset++;
cnt++;
}
TEST_SAY("%s: phase 2: complete\n", what);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_all);
}
static int nonexist_part (void) {
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
rd_kafka_t *rk;
rd_kafka_topic_partition_list_t *parts;
rd_kafka_resp_err_t err;
test_timing_t t_pos;
const int msgcnt = 1000;
uint64_t testid;
int i;
int it, iterations = 5;
/* Produce messages */
testid = test_produce_msgs_easy(topic, 0,
RD_KAFKA_PARTITION_UA, msgcnt);
for (it = 0 ; it < iterations ; it++) {
char group_id[32];
test_str_id_generate(group_id, sizeof(group_id));
TEST_SAY("Iteration %d/%d, using group.id %s\n", it, iterations,
group_id);
/* Consume messages */
test_consume_msgs_easy(group_id, topic, testid, -1,
msgcnt, NULL);
/*
* Now start a new consumer and query stored offsets (positions)
*/
rk = test_create_consumer(group_id, NULL, NULL, NULL, NULL);
/* Fill in partition set */
parts = rd_kafka_topic_partition_list_new(2);
/* existing */
rd_kafka_topic_partition_list_add(parts, topic, 0);
/* non-existing */
rd_kafka_topic_partition_list_add(parts, topic, 123);
TIMING_START(&t_pos, "COMMITTED");
err = rd_kafka_committed(rk, parts, tmout_multip(5000));
TIMING_STOP(&t_pos);
if (err)
TEST_FAIL("Failed to acquire committed offsets: %s\n",
rd_kafka_err2str(err));
for (i = 0 ; i < parts->cnt ; i++) {
TEST_SAY("%s [%"PRId32"] returned offset %"PRId64
": %s\n",
parts->elems[i].topic,
parts->elems[i].partition,
parts->elems[i].offset,
rd_kafka_err2str(parts->elems[i].err));
if (parts->elems[i].partition == 0 &&
parts->elems[i].offset <= 0)
TEST_FAIL("Partition %"PRId32" should have a "
"proper offset, not %"PRId64"\n",
parts->elems[i].partition,
parts->elems[i].offset);
else if (parts->elems[i].partition == 123 &&
parts->elems[i].offset !=
RD_KAFKA_OFFSET_INVALID)
TEST_FAIL("Partition %"PRId32
" should have failed\n",
parts->elems[i].partition);
}
rd_kafka_topic_partition_list_destroy(parts);
test_consumer_close(rk);
/* Hangs if bug isn't fixed */
rd_kafka_destroy(rk);
}
return 0;
}
int main_0049_consume_conn_close (int argc, char **argv) {
rd_kafka_t *rk;
const char *topic = test_mk_topic_name("0049_consume_conn_close", 1);
uint64_t testid;
int msgcnt = test_on_ci ? 1000 : 10000;
test_msgver_t mv;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *tconf;
rd_kafka_topic_partition_list_t *assignment;
rd_kafka_resp_err_t err;
if (!test_conf_match(NULL, "sasl.mechanisms", "GSSAPI")) {
TEST_SKIP("KNOWN ISSUE: ApiVersionRequest+SaslHandshake "
"will not play well with sudden disconnects\n");
return 0;
}
test_conf_init(&conf, &tconf, 60);
/* Want an even number so it is divisable by two without surprises */
msgcnt = (msgcnt / (int)test_timeout_multiplier) & ~1;
testid = test_id_generate();
test_produce_msgs_easy(topic, testid, RD_KAFKA_PARTITION_UA, msgcnt);
test_socket_enable(conf);
test_curr->connect_cb = connect_cb;
test_curr->is_fatal_cb = is_fatal_cb;
test_topic_conf_set(tconf, "auto.offset.reset", "smallest");
rk = test_create_consumer(topic, NULL, conf, tconf);
test_consumer_subscribe(rk, topic);
test_msgver_init(&mv, testid);
test_consumer_poll("consume.up", rk, testid, -1, 0, msgcnt/2, &mv);
err = rd_kafka_assignment(rk, &assignment);
TEST_ASSERT(!err, "assignment() failed: %s", rd_kafka_err2str(err));
TEST_ASSERT(assignment->cnt > 0, "empty assignment");
TEST_SAY("Bringing down the network\n");
TEST_LOCK();
simulate_network_down = 1;
TEST_UNLOCK();
test_socket_close_all(test_curr, 1/*reinit*/);
TEST_SAY("Waiting for session timeout to expire (6s), and then some\n");
/* Commit an offset, which should fail, to trigger the offset commit
* callback fallback (CONSUMER_ERR) */
assignment->elems[0].offset = 123456789;
TEST_SAY("Committing offsets while down, should fail eventually\n");
err = rd_kafka_commit(rk, assignment, 1/*async*/);
TEST_ASSERT(!err, "async commit failed: %s", rd_kafka_err2str(err));
rd_kafka_topic_partition_list_destroy(assignment);
rd_sleep(10);
TEST_SAY("Bringing network back up\n");
TEST_LOCK();
simulate_network_down = 0;
TEST_UNLOCK();
TEST_SAY("Continuing to consume..\n");
test_consumer_poll("consume.up2", rk, testid, -1, msgcnt/2, msgcnt/2,
&mv);
test_msgver_verify("consume", &mv, TEST_MSGVER_ORDER|TEST_MSGVER_DUP,
0, msgcnt);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
return 0;
}
