static void consumer_consume_loop(HermannInstanceConfig* consumerConfig) {
rd_kafka_message_t *msg;
TRACER("\n");
while (consumerConfig->run) {
#ifdef HAVE_RB_THREAD_BLOCKING_REGION
msg = (rd_kafka_message_t *) rb_thread_blocking_region((rb_blocking_function_t *) consumer_recv_msg,
consumerConfig,
consumer_consume_stop_callback,
consumerConfig);
#elif HAVE_RB_THREAD_CALL_WITHOUT_GVL
msg = rb_thread_call_without_gvl(consumer_recv_msg,
consumerConfig,
consumer_consume_stop_callback,
consumerConfig);
#else
msg = consumer_recv_msg(consumerConfig);
#endif
if ( msg ) {
msg_consume(msg, consumerConfig);
rd_kafka_message_destroy(msg);
}
}
}
int kafka_consumer_::PullData(std::string& data)
{
rd_kafka_message_t *rkmessage;
int consume_result = PULL_DATA_SUCCESS;
printf("217consumer_info->rk:%p, rkt:%p, partition=%d\n", rk_, rkt_, partition_);
printf("217 begin poll\n");
rd_kafka_poll(rk_, 0);
printf("219begin consume\n");
rkmessage = rd_kafka_consume(rkt_, partition_, 1000);
if (!rkmessage)
return PULL_DATA_TIMEOUT;
printf("rkmessage->err:%d\n", (int)rkmessage->err);
if (rkmessage->err) {
if (rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION ||
rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC)
return CONSUMER_CONFIG_ERROR;
return PULL_DATA_FAILED;
}
data = std::string((const char*)rkmessage->payload, rkmessage->len);
rd_kafka_message_destroy(rkmessage);
return consume_result;
}
static void get_assignment (rd_kafka_t *rk_c) {
while (!rebalanced) {
rd_kafka_message_t *rkmsg;
rkmsg = rd_kafka_consumer_poll(rk_c, 500);
if (rkmsg)
rd_kafka_message_destroy(rkmsg);
}
}
static void consume_messages (uint64_t testid, const char *topic,
int32_t partition, int msg_base, int batch_cnt,
int msgcnt) {
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
int i;
test_conf_init(&conf, &topic_conf, 20);
/* Create kafka instance */
rk = test_create_handle(RD_KAFKA_CONSUMER, conf);
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
rd_kafka_err2str(rd_kafka_last_error()));
TEST_SAY("Consuming %i messages from partition %i\n",
batch_cnt, partition);
/* Consume messages */
if (rd_kafka_consume_start(rkt, partition,
RD_KAFKA_OFFSET_TAIL(batch_cnt)) == -1)
TEST_FAIL("consume_start(%i, -%i) failed: %s",
(int)partition, batch_cnt,
rd_kafka_err2str(rd_kafka_last_error()));
for (i = 0 ; i < batch_cnt ; i++) {
rd_kafka_message_t *rkmessage;
rkmessage = rd_kafka_consume(rkt, partition, tmout_multip(5000));
if (!rkmessage)
TEST_FAIL("Failed to consume message %i/%i from "
"partition %i: %s",
i, batch_cnt, (int)partition,
rd_kafka_err2str(rd_kafka_last_error()));
if (rkmessage->err)
TEST_FAIL("Consume message %i/%i from partition %i "
"has error: %s",
i, batch_cnt, (int)partition,
rd_kafka_err2str(rkmessage->err));
verify_consumed_msg(testid, partition, msg_base+i, rkmessage);
rd_kafka_message_destroy(rkmessage);
}
rd_kafka_consume_stop(rkt, partition);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy rdkafka instance */
TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
rd_kafka_destroy(rk);
}
int main (int argc, char **argv) {
if (argc < 0 /* always false */) {
rd_kafka_version();
rd_kafka_version_str();
rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR);
rd_kafka_errno2err(EINVAL);
rd_kafka_conf_new();
rd_kafka_conf_destroy(NULL);
rd_kafka_conf_dup(NULL);
rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_conf_set_dr_cb(NULL, NULL);
rd_kafka_conf_set_error_cb(NULL, NULL);
rd_kafka_conf_set_stats_cb(NULL, NULL);
rd_kafka_conf_set_opaque(NULL, NULL);
rd_kafka_conf_dump(NULL, NULL);
rd_kafka_topic_conf_dump(NULL, NULL);
rd_kafka_conf_dump_free(NULL, 0);
rd_kafka_conf_properties_show(NULL);
rd_kafka_topic_conf_new();
rd_kafka_topic_conf_dup(NULL);
rd_kafka_topic_conf_destroy(NULL);
rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_topic_conf_set_opaque(NULL, NULL);
rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL);
rd_kafka_topic_partition_available(NULL, 0);
rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_new(0, NULL, NULL, 0);
rd_kafka_destroy(NULL);
rd_kafka_name(NULL);
rd_kafka_topic_new(NULL, NULL, NULL);
rd_kafka_topic_destroy(NULL);
rd_kafka_topic_name(NULL);
rd_kafka_message_destroy(NULL);
rd_kafka_message_errstr(NULL);
rd_kafka_consume_start(NULL, 0, 0);
rd_kafka_consume_stop(NULL, 0);
rd_kafka_consume(NULL, 0, 0);
rd_kafka_consume_batch(NULL, 0, 0, NULL, 0);
rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL);
rd_kafka_offset_store(NULL, 0, 0);
rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL);
rd_kafka_poll(NULL, 0);
rd_kafka_brokers_add(NULL, NULL);
rd_kafka_set_logger(NULL, NULL);
rd_kafka_set_log_level(NULL, 0);
rd_kafka_log_print(NULL, 0, NULL, NULL);
rd_kafka_log_syslog(NULL, 0, NULL, NULL);
rd_kafka_outq_len(NULL);
rd_kafka_dump(NULL, NULL);
rd_kafka_thread_cnt();
rd_kafka_wait_destroyed(0);
}
return 0;
}
int test_consumer_poll (const char *what, rd_kafka_t *rk, uint64_t testid,
int exp_eof_cnt, int exp_msg_base, int exp_cnt) {
int eof_cnt = 0;
int cnt = 0;
test_timing_t t_cons;
TEST_SAY("%s: consume %d messages\n", what, exp_cnt);
TIMING_START(&t_cons, "CONSUME");
while ((exp_eof_cnt == -1 || eof_cnt < exp_eof_cnt) &&
(cnt < exp_cnt)) {
rd_kafka_message_t *rkmessage;
rkmessage = rd_kafka_consumer_poll(rk, 10*1000);
if (!rkmessage) /* Shouldn't take this long to get a msg */
TEST_FAIL("%s: consumer_poll() timeout\n", what);
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
TEST_SAY("%s [%"PRId32"] reached EOF at "
"offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset);
eof_cnt++;
} else if (rkmessage->err) {
TEST_SAY("%s [%"PRId32"] error (offset %"PRId64"): %s",
rkmessage->rkt ?
rd_kafka_topic_name(rkmessage->rkt) :
"(no-topic)",
rkmessage->partition,
rkmessage->offset,
rd_kafka_message_errstr(rkmessage));
} else {
if (test_level > 2)
TEST_SAY("%s [%"PRId32"] "
"message at offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset);
test_verify_rkmessage(rkmessage, testid, -1, -1);
cnt++;
}
rd_kafka_message_destroy(rkmessage);
}
TIMING_STOP(&t_cons);
TEST_SAY("%s: consumed %d/%d messages (%d/%d EOFs)\n",
what, cnt, exp_cnt, eof_cnt, exp_eof_cnt);
return cnt;
}
static PyObject *Consumer_poll (Handle *self, PyObject *args,
PyObject *kwargs) {
double tmout = -1.0f;
static char *kws[] = { "timeout", NULL };
rd_kafka_message_t *rkm;
PyObject *msgobj;
CallState cs;
if (!self->rk) {
PyErr_SetString(PyExc_RuntimeError,
"Consumer closed");
return NULL;
}
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|d", kws, &tmout))
return NULL;
CallState_begin(self, &cs);
rkm = rd_kafka_consumer_poll(self->rk, tmout >= 0 ?
(int)(tmout * 1000.0f) : -1);
if (!CallState_end(self, &cs)) {
if (rkm)
rd_kafka_message_destroy(rkm);
return NULL;
}
if (!rkm)
Py_RETURN_NONE;
msgobj = Message_new0(self, rkm);
#ifdef RD_KAFKA_V_HEADERS
// Have to detach headers outside Message_new0 because it declares the
// rk message as a const
rd_kafka_message_detach_headers(rkm, &((Message *)msgobj)->c_headers);
#endif
rd_kafka_message_destroy(rkm);
return msgobj;
}
static int partition_consume (void *args) {
part_consume_info_t *info = (part_consume_info_t *)args;
rd_kafka_queue_t *rkqu = info->rkqu;
int partition = info->partition;
int64_t ts_start = test_clock();
int max_time = (test_session_timeout_ms + 3000) * 1000;
int running = 1;
free(args); /* Free the parameter struct dynamically allocated for us */
while (ts_start + max_time > test_clock() && running &&
is_consuming()) {
rd_kafka_message_t *rkmsg;
rkmsg = rd_kafka_consume_queue(rkqu, 500);
if (!rkmsg)
continue;
else if (rkmsg->err == RD_KAFKA_RESP_ERR__PARTITION_EOF)
running = 0;
else if (rkmsg->err) {
mtx_lock(&lock);
TEST_FAIL("Message error "
"(at offset %" PRId64 " after "
"%d/%d messages and %dms): %s",
rkmsg->offset, consumed_msg_cnt, exp_msg_cnt,
(int)(test_clock() - ts_start) / 1000,
rd_kafka_message_errstr(rkmsg));
mtx_unlock(&lock);
} else {
if (rkmsg->partition != partition) {
mtx_lock(&lock);
TEST_FAIL("Message consumed has partition %d "
"but we expected partition %d.",
rkmsg->partition, partition);
mtx_unlock(&lock);
}
}
rd_kafka_message_destroy(rkmsg);
mtx_lock(&lock);
if (running && ++consumed_msg_cnt >= exp_msg_cnt) {
TEST_SAY("All messages consumed\n");
running = 0;
}
mtx_unlock(&lock);
}
rd_kafka_queue_destroy(rkqu);
return thrd_success;
}
int Http::kafka_consumer_::PullData(std::string& data) {
rd_kafka_message_t *rkmessage;
rd_kafka_poll(rk_, 0);
rkmessage = rd_kafka_consume(rkt_, partition_, 10000);
if (!rkmessage)
return -2;
if (rkmessage->err) {
if (rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION ||
rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC) {
return -1;
}
return -3;
}
data = std::string((const char*)rkmessage->payload, rkmessage->len);
rd_kafka_message_destroy(rkmessage);
return 1;
}
int consumer_pull_data(const wrapper_Info* consumer_info)
{
rd_kafka_message_t *rkmessage;
int consume_result;
printf("217consumer_info->rk:%p, rkt:%p, partition=%d\n", consumer_info->rk, consumer_info->rkt, consumer_info->partition);
printf("217 begin poll\n");
/* Poll for errors, etc. */
rd_kafka_poll(consumer_info->rk, 0);
printf("219begin consume\n");
/* Consume single message.
* See rdkafka_performance.c for high speed
* consuming of messages. */
rkmessage = rd_kafka_consume(consumer_info->rkt, consumer_info->partition, 1000);
if (!rkmessage) /* timeout */
return PULL_DATA_FAILED;
if (rkmessage->err) {
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
// fprintf(stderr,
// "%% Consumer reached end of %s [%"PRId32"] "
// "message queue at offset %"PRId64"\n",
// rd_kafka_topic_name(rkmessage->rkt),
// rkmessage->partition, rkmessage->offset);
consume_result = PULL_DATA_SUCCESS;
}
// fprintf(stderr, "%% Consume error for topic \"%s\" [%"PRId32"] "
// "offset %"PRId64": %s\n",
// rd_kafka_topic_name(rkmessage->rkt),
// rkmessage->partition,
// rkmessage->offset,
// rd_kafka_message_errstr(rkmessage));
if (rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION ||
rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC)
return rkmessage->err;
consume_result = PULL_DATA_FAILED;
}
consumer_info->func_consume_data((const char*)rkmessage->payload, (const int)rkmessage->len);
/* Return message to rdkafka */
rd_kafka_message_destroy(rkmessage);
return consume_result;
}
int test_consumer_poll (const char *what, rd_kafka_t *rk, uint64_t testid,
int exp_eof_cnt, int exp_msg_base, int exp_cnt) {
int eof_cnt = 0;
int cnt = 0;
while (eof_cnt < exp_eof_cnt) {
rd_kafka_message_t *rkmessage;
rkmessage = rd_kafka_consumer_poll(rk, 10*1000);
if (!rkmessage) /* Shouldn't take this long to get a msg */
TEST_FAIL("%s: consumer_poll() timeout\n", what);
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
TEST_SAY("%s [%"PRId32"] reached EOF at "
"offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset);
eof_cnt++;
} else if (rkmessage->err) {
TEST_SAY("%s [%"PRId32"] error (offset %"PRId64"): %s",
rkmessage->rkt ?
rd_kafka_topic_name(rkmessage->rkt) :
"(no-topic)",
rkmessage->partition,
rkmessage->offset,
rd_kafka_message_errstr(rkmessage));
} else {
if (test_level > 2)
TEST_SAY("%s [%"PRId32"] "
"message at offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset);
cnt++;
}
rd_kafka_message_destroy(rkmessage);
}
return cnt;
}
static void consume_all (rd_kafka_t **rk_c, int rk_cnt, int exp_msg_cnt,
int max_time/*ms*/) {
int64_t ts_start = test_clock();
int i;
max_time *= 1000;
while (ts_start + max_time > test_clock()) {
for (i = 0 ; i < rk_cnt ; i++) {
rd_kafka_message_t *rkmsg;
if (!rk_c[i])
continue;
rkmsg = rd_kafka_consumer_poll(rk_c[i], 500);
if (!rkmsg)
continue;
else if (rkmsg->err)
TEST_SAY("Message error "
"(at offset %"PRId64" after "
"%d/%d messages and %dms): %s\n",
rkmsg->offset,
consumed_msg_cnt, exp_msg_cnt,
(int)(test_clock() - ts_start)/1000,
rd_kafka_message_errstr(rkmsg));
else
consumed_msg_cnt++;
rd_kafka_message_destroy(rkmsg);
if (consumed_msg_cnt >= exp_msg_cnt) {
static int once = 0;
if (!once++)
TEST_SAY("All messages consumed\n");
return;
}
}
}
}
static void do_consume (const char *topic, int msgcnt) {
rd_kafka_t *rk;
rd_kafka_topic_partition_list_t *parts;
rk = test_create_consumer(topic, NULL, NULL, NULL);
parts = rd_kafka_topic_partition_list_new(1);
rd_kafka_topic_partition_list_add(parts, topic, 0)->offset =
RD_KAFKA_OFFSET_BEGINNING;
test_consumer_assign("assign", rk, parts);
rd_kafka_topic_partition_list_destroy(parts);
exp_msgid = 0;
while (exp_msgid < msgcnt) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(rk, 1000);
if (!rkm)
continue;
if (rkm->err)
TEST_FAIL("consume error while expecting msgid %d/%d: "
"%s",
exp_msgid, msgcnt,
rd_kafka_message_errstr(rkm));
handle_consumed_msg(rkm);
rd_kafka_message_destroy(rkm);
}
test_consumer_close(rk);
rd_kafka_destroy(rk);
}
static void do_consume (struct _consumer *cons, int timeout_s) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(cons->rk, 100+(timeout_s*1000));
if (!rkm)
return;
TEST_ASSERT(!rkm->err,
"%s consumer error: %s (last poll was %dms ago)",
rd_kafka_name(cons->rk),
rd_kafka_message_errstr(rkm),
(int)((test_clock() - cons->last)/1000));
rd_kafka_message_destroy(rkm);
cons->cnt++;
cons->last = test_clock();
if (timeout_s > 0) {
TEST_SAY("%s: simulate processing by sleeping for %ds\n",
rd_kafka_name(cons->rk), timeout_s);
rd_sleep(timeout_s);
}
}
static int do_test_consume_batch (void) {
#define topic_cnt 2
const char *topics[topic_cnt] = {
test_mk_topic_name(__FUNCTION__, 1),
test_mk_topic_name(__FUNCTION__, 1)
};
const int partition_cnt = 2;
rd_kafka_t *rk;
rd_kafka_queue_t *rkq;
rd_kafka_topic_t *rkts[topic_cnt];
rd_kafka_resp_err_t err;
const int msgcnt = 10000;
uint64_t testid;
int i, p;
int batch_cnt = 0;
int remains;
testid = test_id_generate();
/* Produce messages */
for (i = 0 ; i < topic_cnt ; i++) {
for (p = 0 ; p < partition_cnt ; p++)
test_produce_msgs_easy(topics[i], testid, p,
msgcnt / topic_cnt /
partition_cnt);
}
/* Create simple consumer */
rk = test_create_consumer(NULL, NULL, NULL, NULL);
/* Create generic consume queue */
rkq = rd_kafka_queue_new(rk);
for (i = 0 ; i < topic_cnt ; i++) {
/* Create topic object */
rkts[i] = test_create_topic(rk, topics[i],
"auto.offset.reset", "smallest",
NULL);
/* Start consuming each partition and redirect
* messages to queue */
TEST_SAY("Start consuming topic %s partitions 0..%d\n",
rd_kafka_topic_name(rkts[i]), partition_cnt);
for (p = 0 ; p < partition_cnt ; p++) {
err = rd_kafka_consume_start_queue(
rkts[i], p, RD_KAFKA_OFFSET_BEGINNING, rkq);
if (err)
TEST_FAIL("Failed to start consuming: %s\n",
rd_kafka_err2str(err));
}
}
remains = msgcnt;
/* Consume messages from common queue using batch interface. */
TEST_SAY("Consume %d messages from queue\n", remains);
while (remains > 0) {
rd_kafka_message_t *rkmessage[1000];
ssize_t r;
test_timing_t t_batch;
TIMING_START(&t_batch, "CONSUME.BATCH");
r = rd_kafka_consume_batch_queue(rkq, 1000, rkmessage, 1000);
TIMING_STOP(&t_batch);
TEST_SAY("Batch consume iteration #%d: Consumed %"PRIdsz
"/1000 messages\n", batch_cnt, r);
if (r == -1)
TEST_FAIL("Failed to consume messages: %s\n",
rd_kafka_err2str(rd_kafka_errno2err(errno)));
remains -= r;
for (i = 0 ; i < r ; i++)
rd_kafka_message_destroy(rkmessage[i]);
batch_cnt++;
}
TEST_SAY("Stopping consumer\n");
for (i = 0 ; i < topic_cnt ; i++) {
for (p = 0 ; p < partition_cnt ; p++) {
err = rd_kafka_consume_stop(rkts[i], p);
if (err)
TEST_FAIL("Failed to stop consuming: %s\n",
rd_kafka_err2str(err));
}
rd_kafka_topic_destroy(rkts[i]);
}
rd_kafka_queue_destroy(rkq);
rd_kafka_destroy(rk);
return 0;
}
int main (int argc, char **argv) {
char *brokers = "localhost";
char mode = 'C';
char *topic = NULL;
const char *key = NULL;
int partition = RD_KAFKA_PARTITION_UA; /* random */
int opt;
int msgcnt = -1;
int sendflags = 0;
char *msgpattern = "librdkafka_performance testing!";
int msgsize = strlen(msgpattern);
const char *debug = NULL;
rd_ts_t now;
char errstr[512];
uint64_t seq = 0;
int seed = time(NULL);
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
const char *compression = "no";
int64_t start_offset = 0;
int batch_size = 0;
/* Kafka configuration */
conf = rd_kafka_conf_new();
rd_kafka_conf_set_error_cb(conf, err_cb);
rd_kafka_conf_set_dr_cb(conf, msg_delivered);
/* Producer config */
rd_kafka_conf_set(conf, "queue.buffering.max.messages", "500000",
NULL, 0);
rd_kafka_conf_set(conf, "message.send.max.retries", "3", NULL, 0);
rd_kafka_conf_set(conf, "retry.backoff.ms", "500", NULL, 0);
/* Consumer config */
/* Tell rdkafka to (try to) maintain 1M messages
* in its internal receive buffers. This is to avoid
* application -> rdkafka -> broker per-message ping-pong
* latency.
* The larger the local queue, the higher the performance.
* Try other values with: ... -X queued.min.messages=1000
*/
rd_kafka_conf_set(conf, "queued.min.messages", "1000000", NULL, 0);
/* Kafka topic configuration */
topic_conf = rd_kafka_topic_conf_new();
rd_kafka_topic_conf_set(topic_conf, "message.timeout.ms", "5000",
NULL, 0);
while ((opt =
getopt(argc, argv,
"PCt:p:b:s:k:c:fi:Dd:m:S:x:R:a:z:o:X:B:eT:q")) != -1) {
switch (opt) {
case 'P':
case 'C':
mode = opt;
break;
case 't':
topic = optarg;
break;
case 'p':
partition = atoi(optarg);
break;
case 'b':
brokers = optarg;
break;
case 's':
msgsize = atoi(optarg);
break;
case 'k':
key = optarg;
break;
case 'c':
msgcnt = atoi(optarg);
break;
case 'D':
sendflags |= RD_KAFKA_MSG_F_FREE;
break;
case 'i':
dispintvl = atoi(optarg);
break;
case 'm':
msgpattern = optarg;
break;
case 'S':
seq = strtoull(optarg, NULL, 10);
do_seq = 1;
break;
case 'x':
exit_after = atoi(optarg);
break;
case 'R':
seed = atoi(optarg);
break;
case 'a':
if (rd_kafka_topic_conf_set(topic_conf,
"request.required.acks",
optarg,
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
break;
case 'B':
batch_size = atoi(optarg);
break;
case 'z':
if (rd_kafka_conf_set(conf, "compression.codec",
optarg,
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
compression = optarg;
break;
case 'o':
start_offset = strtoll(optarg, NULL, 10);
break;
case 'e':
exit_eof = 1;
break;
case 'd':
debug = optarg;
break;
case 'X':
{
char *name, *val;
rd_kafka_conf_res_t res;
if (!strcmp(optarg, "list") ||
!strcmp(optarg, "help")) {
rd_kafka_conf_properties_show(stdout);
exit(0);
}
name = optarg;
if (!(val = strchr(name, '='))) {
fprintf(stderr, "%% Expected "
"-X property=value, not %s\n", name);
exit(1);
}
*val = '\0';
val++;
res = RD_KAFKA_CONF_UNKNOWN;
/* Try "topic." prefixed properties on topic
* conf first, and then fall through to global if
* it didnt match a topic configuration property. */
if (!strncmp(name, "topic.", strlen("topic.")))
res = rd_kafka_topic_conf_set(topic_conf,
name+
strlen("topic"),
val,
errstr,
sizeof(errstr));
if (res == RD_KAFKA_CONF_UNKNOWN)
res = rd_kafka_conf_set(conf, name, val,
errstr, sizeof(errstr));
if (res != RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
}
break;
case 'T':
if (rd_kafka_conf_set(conf, "statistics.interval.ms",
optarg, errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
rd_kafka_conf_set_stats_cb(conf, stats_cb);
break;
case 'q':
quiet = 1;
break;
default:
goto usage;
}
}
if (!topic || optind != argc) {
usage:
fprintf(stderr,
"Usage: %s [-C|-P] -t <topic> "
"[-p <partition>] [-b <broker,broker..>] [options..]\n"
"\n"
" Options:\n"
" -C | -P Consumer or Producer mode\n"
" -t <topic> Topic to fetch / produce\n"
" -p <num> Partition (defaults to random)\n"
" -b <brokers> Broker address list (host[:port],..)\n"
" -s <size> Message size (producer)\n"
" -k <key> Message key (producer)\n"
" -c <cnt> Messages to transmit/receive\n"
" -D Copy/Duplicate data buffer (producer)\n"
" -i <ms> Display interval\n"
" -m <msg> Message payload pattern\n"
" -S <start> Send a sequence number starting at "
"<start> as payload\n"
" -R <seed> Random seed value (defaults to time)\n"
" -a <acks> Required acks (producer): "
"-1, 0, 1, >1\n"
" -B <size> Consume batch size (# of msgs)\n"
" -z <codec> Enable compression:\n"
" none|gzip|snappy\n"
" -o <offset> Start offset (consumer)\n"
" -d [facs..] Enable debugging contexts:\n"
" %s\n"
" -X <prop=name> Set arbitrary librdkafka "
"configuration property\n"
" Properties prefixed with \"topic.\" "
"will be set on topic object.\n"
" Use '-X list' to see the full list\n"
" of supported properties.\n"
" -T <intvl> Enable statistics from librdkafka at "
"specified interval (ms)\n"
" -q Be more quiet\n"
"\n"
" In Consumer mode:\n"
" consumes messages and prints thruput\n"
" If -B <..> is supplied the batch consumer\n"
" mode is used, else the callback mode is used.\n"
"\n"
" In Producer mode:\n"
" writes messages of size -s <..> and prints thruput\n"
"\n",
argv[0],
RD_KAFKA_DEBUG_CONTEXTS);
exit(1);
}
dispintvl *= 1000; /* us */
printf("%% Using random seed %i\n", seed);
srand(seed);
signal(SIGINT, stop);
signal(SIGUSR1, sig_usr1);
if (debug &&
rd_kafka_conf_set(conf, "debug", debug, errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
printf("%% Debug configuration failed: %s: %s\n",
errstr, debug);
exit(1);
}
/* Socket hangups are gracefully handled in librdkafka on socket error
* without the use of signals, so SIGPIPE should be ignored by the
* calling program. */
signal(SIGPIPE, SIG_IGN);
if (msgcnt != -1)
forever = 0;
if (mode == 'P') {
/*
* Producer
*/
char *sbuf;
char *pbuf;
int outq;
int i;
int keylen = key ? strlen(key) : 0;
off_t rof = 0;
size_t plen = strlen(msgpattern);
if (do_seq) {
if (msgsize < strlen("18446744073709551615: ")+1)
msgsize = strlen("18446744073709551615: ")+1;
/* Force duplication of payload */
sendflags |= RD_KAFKA_MSG_F_FREE;
}
sbuf = malloc(msgsize);
/* Copy payload content to new buffer */
while (rof < msgsize) {
size_t xlen = RD_MIN(msgsize-rof, plen);
memcpy(sbuf+rof, msgpattern, xlen);
rof += xlen;
}
if (msgcnt == -1)
printf("%% Sending messages of size %i bytes\n",
msgsize);
else
printf("%% Sending %i messages of size %i bytes\n",
msgcnt, msgsize);
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create Kafka producer: %s\n",
errstr);
exit(1);
}
if (debug)
rd_kafka_set_log_level(rk, 7);
/* Add broker(s) */
if (rd_kafka_brokers_add(rk, brokers) < 1) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Explicitly create topic to avoid per-msg lookups. */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
cnt.t_start = rd_clock();
while (run && (msgcnt == -1 || cnt.msgs < msgcnt)) {
/* Send/Produce message. */
if (do_seq) {
snprintf(sbuf, msgsize-1, "%"PRIu64": ", seq);
seq++;
}
if (sendflags & RD_KAFKA_MSG_F_FREE) {
/* Duplicate memory */
pbuf = malloc(msgsize);
memcpy(pbuf, sbuf, msgsize);
} else
pbuf = sbuf;
cnt.tx++;
while (run &&
rd_kafka_produce(rkt, partition,
sendflags, pbuf, msgsize,
key, keylen, NULL) == -1) {
if (!quiet || errno != ENOBUFS)
printf("produce error: %s%s\n",
strerror(errno),
errno == ENOBUFS ?
" (backpressure)":"");
cnt.tx_err++;
if (errno != ENOBUFS) {
run = 0;
break;
}
now = rd_clock();
if (cnt.t_last + dispintvl <= now) {
printf("%% Backpressure %i "
"(tx %"PRIu64", "
"txerr %"PRIu64")\n",
rd_kafka_outq_len(rk),
cnt.tx, cnt.tx_err);
cnt.t_last = now;
}
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 10);
}
msgs_wait_cnt++;
cnt.msgs++;
cnt.bytes += msgsize;
print_stats(mode, 0, compression);
/* Must poll to handle delivery reports */
rd_kafka_poll(rk, 0);
}
forever = 0;
printf("All messages produced, "
"now waiting for %li deliveries\n",
msgs_wait_cnt);
rd_kafka_dump(stdout, rk);
/* Wait for messages to be delivered */
i = 0;
while (run && rd_kafka_poll(rk, 1000) != -1) {
if (!(i++ % (dispintvl/1000)))
printf("%% Waiting for %li, "
"%i messages in outq "
"to be sent. Abort with Ctrl-c\n",
msgs_wait_cnt,
rd_kafka_outq_len(rk));
}
outq = rd_kafka_outq_len(rk);
printf("%% %i messages in outq\n", outq);
cnt.msgs -= outq;
cnt.bytes -= msgsize * outq;
cnt.t_end = t_end;
if (cnt.tx_err > 0)
printf("%% %"PRIu64" backpressures for %"PRIu64
" produce calls: %.3f%% backpressure rate\n",
cnt.tx_err, cnt.tx,
((double)cnt.tx_err / (double)cnt.tx) * 100.0);
rd_kafka_dump(stdout, rk);
/* Destroy the handle */
rd_kafka_destroy(rk);
} else if (mode == 'C') {
/*
* Consumer
*/
rd_kafka_message_t **rkmessages = NULL;
#if 0 /* Future API */
/* The offset storage file is optional but its presence
* avoids starting all over from offset 0 again when
* the program restarts.
* ZooKeeper functionality will be implemented in future
* versions and then the offset will be stored there instead. */
conf.consumer.offset_file = "."; /* current directory */
/* Indicate to rdkafka that the application is responsible
* for storing the offset. This allows the application to
* successfully handle a message before storing the offset.
* If this flag is not set rdkafka will store the offset
* just prior to returning the message from rd_kafka_consume().
*/
conf.flags |= RD_KAFKA_CONF_F_APP_OFFSET_STORE;
#endif
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create Kafka producer: %s\n",
errstr);
exit(1);
}
if (debug)
rd_kafka_set_log_level(rk, 7);
/* Add broker(s) */
if (rd_kafka_brokers_add(rk, brokers) < 1) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Create topic to consume from */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
/* Batch consumer */
if (batch_size)
rkmessages = malloc(sizeof(*rkmessages) * batch_size);
/* Start consuming */
if (rd_kafka_consume_start(rkt, partition, start_offset) == -1){
fprintf(stderr, "%% Failed to start consuming: %s\n",
strerror(errno));
exit(1);
}
cnt.t_start = rd_clock();
while (run && (msgcnt == -1 || msgcnt > cnt.msgs)) {
/* Consume messages.
* A message may either be a real message, or
* an error signaling (if rkmessage->err is set).
*/
uint64_t latency;
int r;
latency = rd_clock();
if (batch_size) {
int i;
/* Batch fetch mode */
r = rd_kafka_consume_batch(rkt, partition,
1000,
rkmessages,
batch_size);
if (r != -1) {
for (i = 0 ; i < r ; i++) {
msg_consume(rkmessages[i],NULL);
rd_kafka_message_destroy(
rkmessages[i]);
}
}
} else {
/* Callback mode */
r = rd_kafka_consume_callback(rkt, partition,
1000/*timeout*/,
msg_consume,
NULL);
}
cnt.t_latency += rd_clock() - latency;
if (r == -1)
fprintf(stderr, "%% Error: %s\n",
strerror(errno));
print_stats(mode, 0, compression);
/* Poll to handle stats callbacks */
rd_kafka_poll(rk, 0);
}
cnt.t_end = rd_clock();
/* Stop consuming */
rd_kafka_consume_stop(rkt, partition);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
if (batch_size)
free(rkmessages);
/* Destroy the handle */
rd_kafka_destroy(rk);
}
print_stats(mode, 1, compression);
if (cnt.t_latency && cnt.msgs)
printf("%% Average application fetch latency: %"PRIu64"us\n",
cnt.t_latency / cnt.msgs);
/* Let background threads clean up and terminate cleanly. */
rd_kafka_wait_destroyed(2000);
return 0;
}
int kafka_consume(rd_kafka_t *r, zval* return_value, char* topic, char* offset, int item_count, int partition)
{
int64_t start_offset = 0;
int read_counter = 0,
run = 1;
//nothing to consume?
if (item_count == 0)
return 0;
if (strlen(offset) != 0)
{
if (!strcmp(offset, "end"))
start_offset = RD_KAFKA_OFFSET_END;
else if (!strcmp(offset, "beginning"))
start_offset = RD_KAFKA_OFFSET_BEGINNING;
else if (!strcmp(offset, "stored"))
start_offset = RD_KAFKA_OFFSET_STORED;
else
{
start_offset = strtoll(offset, NULL, 10);
if (start_offset < 1)
return -1;
}
}
rd_kafka_topic_t *rkt;
if (r == NULL)
{
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "phpkafka - no connection to consume from topic: %s", topic);
}
return -2;
}
rd_kafka_topic_conf_t *topic_conf;
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
/* Create topic */
rkt = rd_kafka_topic_new(r, topic, topic_conf);
if (rkt == NULL)
{
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(
LOG_ERR,
"Failed to consume from topic %s: %s",
topic,
rd_kafka_err2str(
rd_kafka_errno2err(errno)
)
);
}
return -3;
}
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - start_offset: %"PRId64" and offset passed: %s", start_offset, offset);
}
/* Start consuming */
if (rd_kafka_consume_start(rkt, partition, start_offset) == -1)
{
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - %% Failed to start consuming: %s",
rd_kafka_err2str(rd_kafka_errno2err(errno)));
}
return -4;
}
/**
* Keep reading until run == 0, or read_counter == item_count
*/
for (read_counter=0;read_counter!=item_count;++read_counter)
{
if (run == 0)
break;
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "Consuming, count at %d (of %d - run: %d)",
read_counter,
item_count,
run
);
}
rd_kafka_message_t *rkmessage = NULL,
*rkmessage_return = NULL;
/* Consume single message.
* See rdkafka_performance.c for high speed
* consuming of messages. */
rkmessage = rd_kafka_consume(rkt, partition, 1000);
//timeout ONLY if error didn't cause run to be 0
if (!rkmessage)
{
//break on timeout, makes second call redundant
if (errno == ETIMEDOUT)
{
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "Consumer timed out, count at %d (of %d) stop consuming after %d messages",
read_counter,
item_count,
read_counter +1
);
}
break;
}
continue;
}
rkmessage_return = msg_consume(rkmessage, &run);
if (rkmessage_return != NULL)
{
if ((int) rkmessage_return->len > 0)
{
//ensure there is a payload
char payload[(int) rkmessage_return->len];
sprintf(payload, "%.*s", (int) rkmessage_return->len, (char *) rkmessage_return->payload);
add_index_string(return_value, (int) rkmessage_return->offset, payload, 1);
}
else
{
//add empty value
char payload[1] = "";//empty string
add_index_string(return_value, (int) rkmessage_return->offset, payload, 1);
}
}
/* Return message to rdkafka */
rd_kafka_message_destroy(rkmessage);
}
/* Stop consuming */
rd_kafka_consume_stop(rkt, partition);
rd_kafka_topic_destroy(rkt);
return 0;
}
int main_0089_max_poll_interval (int argc, char **argv) {
const char *topic = test_mk_topic_name("0089_max_poll_interval", 1);
uint64_t testid;
const int msgcnt = 10;
rd_kafka_t *c[2];
rd_kafka_conf_t *conf;
int64_t ts_next[2] = { 0, 0 };
int64_t ts_exp_msg[2] = { 0, 0 };
int cmsgcnt = 0;
int i;
int bad = -1;
testid = test_id_generate();
test_create_topic(topic, 1, 1);
test_produce_msgs_easy(topic, testid, -1, msgcnt);
test_conf_init(&conf, NULL, 60);
test_conf_set(conf, "session.timeout.ms", "6000");
test_conf_set(conf, "max.poll.interval.ms", "10000" /*10s*/);
test_conf_set(conf, "auto.offset.reset", "earliest");
c[0] = test_create_consumer(topic, NULL, rd_kafka_conf_dup(conf), NULL);
c[1] = test_create_consumer(topic, NULL, conf, NULL);
test_consumer_subscribe(c[0], topic);
test_consumer_subscribe(c[1], topic);
while (1) {
for (i = 0 ; i < 2 ; i++) {
int64_t now;
rd_kafka_message_t *rkm;
/* Consumer is "processing" */
if (ts_next[i] > test_clock())
continue;
rkm = rd_kafka_consumer_poll(c[i], 100);
if (!rkm)
continue;
if (rkm->err) {
TEST_WARN("Consumer %d error: %s: "
"ignoring\n", i,
rd_kafka_message_errstr(rkm));
continue;
}
now = test_clock();
cmsgcnt++;
TEST_SAY("Consumer %d received message (#%d) "
"at offset %"PRId64"\n",
i, cmsgcnt, rkm->offset);
if (ts_exp_msg[i]) {
/* This consumer is expecting a message
* after a certain time, namely after the
* rebalance following max.poll.. being
* exceeded in the other consumer */
TEST_ASSERT(now > ts_exp_msg[i],
"Consumer %d: did not expect "
"message for at least %dms",
i,
(int)((ts_exp_msg[i] - now)/1000));
TEST_ASSERT(now < ts_exp_msg[i] + 10000*1000,
"Consumer %d: expected message "
"within 10s, not after %dms",
i,
(int)((now - ts_exp_msg[i])/1000));
TEST_SAY("Consumer %d: received message "
"at offset %"PRId64
" after rebalance\n",
i, rkm->offset);
rd_kafka_message_destroy(rkm);
goto done;
} else if (cmsgcnt == 1) {
/* Process this message for 20s */
ts_next[i] = now + (20000 * 1000);
/* Exp message on other consumer after
* max.poll.interval.ms */
ts_exp_msg[i^1] = now + (10000 * 1000);
/* This is the bad consumer */
bad = i;
TEST_SAY("Consumer %d processing message at "
"offset %"PRId64"\n",
i, rkm->offset);
rd_kafka_message_destroy(rkm);
} else {
rd_kafka_message_destroy(rkm);
TEST_FAIL("Consumer %d did not expect "
"a message", i);
}
}
}
done:
TEST_ASSERT(bad != -1, "Bad consumer not set");
/* Wait for error ERR__MAX_POLL_EXCEEDED on the bad consumer. */
while (1) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(c[bad], 1000);
TEST_ASSERT(rkm, "Expected consumer result within 1s");
TEST_ASSERT(rkm->err, "Did not expect message on bad consumer");
TEST_SAY("Consumer error: %s: %s\n",
rd_kafka_err2name(rkm->err),
rd_kafka_message_errstr(rkm));
if (rkm->err == RD_KAFKA_RESP_ERR__MAX_POLL_EXCEEDED) {
rd_kafka_message_destroy(rkm);
break;
}
rd_kafka_message_destroy(rkm);
}
for (i = 0 ; i < 2 ; i++)
rd_kafka_destroy_flags(c[i],
RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE);
return 0;
}
static PyObject *Consumer_consume (Handle *self, PyObject *args,
PyObject *kwargs) {
unsigned int num_messages = 1;
double tmout = -1.0f;
static char *kws[] = { "num_messages", "timeout", NULL };
rd_kafka_message_t **rkmessages;
PyObject *msglist;
rd_kafka_queue_t *rkqu = self->u.Consumer.rkqu;
CallState cs;
Py_ssize_t i, n;
if (!self->rk) {
PyErr_SetString(PyExc_RuntimeError,
"Consumer closed");
return NULL;
}
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|Id", kws,
&num_messages, &tmout))
return NULL;
if (num_messages > 1000000) {
PyErr_SetString(PyExc_ValueError,
"num_messages must be between 0 and 1000000 (1M)");
return NULL;
}
CallState_begin(self, &cs);
rkmessages = malloc(num_messages * sizeof(rd_kafka_message_t *));
n = (Py_ssize_t)rd_kafka_consume_batch_queue(rkqu,
tmout >= 0 ? (int)(tmout * 1000.0f) : -1,
rkmessages,
num_messages);
if (!CallState_end(self, &cs)) {
for (i = 0; i < n; i++) {
rd_kafka_message_destroy(rkmessages[i]);
}
free(rkmessages);
return NULL;
}
if (n < 0) {
free(rkmessages);
cfl_PyErr_Format(rd_kafka_last_error(),
"%s", rd_kafka_err2str(rd_kafka_last_error()));
return NULL;
}
msglist = PyList_New(n);
for (i = 0; i < n; i++) {
PyObject *msgobj = Message_new0(self, rkmessages[i]);
#ifdef RD_KAFKA_V_HEADERS
// Have to detach headers outside Message_new0 because it declares the
// rk message as a const
rd_kafka_message_detach_headers(rkmessages[i], &((Message *)msgobj)->c_headers);
#endif
PyList_SET_ITEM(msglist, i, msgobj);
rd_kafka_message_destroy(rkmessages[i]);
}
free(rkmessages);
return msglist;
}
void kafka_consume(zval* return_value, char* topic, char* offset, int item_count)
{
int read_counter = 0;
if (strlen(offset) != 0) {
if (!strcmp(offset, "end"))
start_offset = RD_KAFKA_OFFSET_END;
else if (!strcmp(offset, "beginning"))
start_offset = RD_KAFKA_OFFSET_BEGINNING;
else if (!strcmp(offset, "stored"))
start_offset = RD_KAFKA_OFFSET_STORED;
else
start_offset = strtoll(offset, NULL, 10);
}
rd_kafka_topic_t *rkt;
char errstr[512];
rd_kafka_conf_t *conf;
/* Kafka configuration */
conf = rd_kafka_conf_new();
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr)))) {
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - failed to create new consumer: %s", errstr);
exit(1);
}
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "php kafka - No valid brokers specified");
exit(1);
}
rd_kafka_topic_conf_t *topic_conf;
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
/* Create topic */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - start_offset: %d and offset passed: %d", start_offset, offset);
/* Start consuming */
if (rd_kafka_consume_start(rkt, partition, start_offset) == -1) {
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - %% Failed to start consuming: %s",
rd_kafka_err2str(rd_kafka_errno2err(errno)));
exit(1);
}
if (item_count != 0) {
read_counter = item_count;
}
while (run) {
if (item_count != 0 && read_counter >= 0) {
read_counter--;
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - read_counter: %d", read_counter);
if (read_counter == -1) {
run = 0;
continue;
}
}
rd_kafka_message_t *rkmessage;
/* Consume single message.
* See rdkafka_performance.c for high speed
* consuming of messages. */
rkmessage = rd_kafka_consume(rkt, partition, 1000);
if (!rkmessage) /* timeout */
continue;
rd_kafka_message_t *rkmessage_return;
rkmessage_return = msg_consume(rkmessage, NULL);
char payload[(int)rkmessage_return->len];
sprintf(payload, "%.*s", (int)rkmessage_return->len, (char *)rkmessage_return->payload);
add_index_string(return_value, (int)rkmessage_return->offset, payload, 1);
/* Return message to rdkafka */
rd_kafka_message_destroy(rkmessage);
}
/* Stop consuming */
rd_kafka_consume_stop(rkt, partition);
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
}
static void consume_messages_with_queues (uint64_t testid, const char *topic,
int partition_cnt, int msgcnt) {
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
rd_kafka_queue_t *rkqu;
int i;
int32_t partition;
int batch_cnt = msgcnt / partition_cnt;
test_conf_init(&conf, &topic_conf, 20);
/* Create kafka instance */
rk = test_create_handle(RD_KAFKA_CONSUMER, conf);
/* Create queue */
rkqu = rd_kafka_queue_new(rk);
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
rd_strerror(errno));
TEST_SAY("Consuming %i messages from one queue serving %i partitions\n",
msgcnt, partition_cnt);
/* Start consuming each partition */
for (partition = 0 ; partition < partition_cnt ; partition++) {
/* Consume messages */
TEST_SAY("Start consuming partition %i at offset -%i\n",
partition, batch_cnt);
if (rd_kafka_consume_start_queue(rkt, partition,
RD_KAFKA_OFFSET_TAIL(batch_cnt),
rkqu) == -1)
TEST_FAIL("consume_start_queue(%i) failed: %s",
(int)partition,
rd_kafka_err2str(rd_kafka_errno2err(errno)));
}
/* Consume messages from queue */
for (i = 0 ; i < msgcnt ; ) {
rd_kafka_message_t *rkmessage;
rkmessage = rd_kafka_consume_queue(rkqu, tmout_multip(5000));
if (!rkmessage)
TEST_FAIL("Failed to consume message %i/%i from "
"queue: %s",
i, msgcnt,
rd_kafka_err2str(rd_kafka_errno2err(errno)));
if (rkmessage->err) {
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF){
TEST_SAY("Topic %s [%"PRId32"] reached "
"EOF at offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset);
rd_kafka_message_destroy(rkmessage);
continue;
}
TEST_FAIL("Consume message %i/%i from queue "
"has error (offset %"PRId64
", partition %"PRId32"): %s",
i, msgcnt,
rkmessage->offset, rkmessage->partition,
rd_kafka_err2str(rkmessage->err));
}
verify_consumed_msg(testid, -1, -1, rkmessage);
rd_kafka_message_destroy(rkmessage);
i++;
}
/* Stop consuming each partition */
for (partition = 0 ; partition < partition_cnt ; partition++)
rd_kafka_consume_stop(rkt, partition);
/* Destroy queue */
rd_kafka_queue_destroy(rkqu);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy rdkafka instance */
TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
rd_kafka_destroy(rk);
}
int main (int argc, char **argv) {
char mode = 'C';
char *brokers = "localhost:9092";
int opt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char errstr[512];
const char *debug = NULL;
int do_conf_dump = 0;
char tmp[16];
rd_kafka_resp_err_t err;
char *group = NULL;
rd_kafka_topic_partition_list_t *topics;
int i;
quiet = !isatty(STDIN_FILENO);
/* Kafka configuration */
conf = rd_kafka_conf_new();
/* Set logger */
rd_kafka_conf_set_log_cb(conf, logger);
/* Quick termination */
snprintf(tmp, sizeof(tmp), "%i", SIGIO);
rd_kafka_conf_set(conf, "internal.termination.signal", tmp, NULL, 0);
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
while ((opt = getopt(argc, argv, "g:b:qd:eX:As:DO")) != -1) {
switch (opt) {
case 'b':
brokers = optarg;
break;
case 'g':
group = optarg;
break;
case 'e':
exit_eof = 1;
break;
case 'd':
debug = optarg;
break;
case 'q':
quiet = 1;
break;
case 'A':
output = OUTPUT_RAW;
break;
case 'X':
{
char *name, *val;
rd_kafka_conf_res_t res;
if (!strcmp(optarg, "list") ||
!strcmp(optarg, "help")) {
rd_kafka_conf_properties_show(stdout);
exit(0);
}
if (!strcmp(optarg, "dump")) {
do_conf_dump = 1;
continue;
}
name = optarg;
if (!(val = strchr(name, '='))) {
fprintf(stderr, "%% Expected "
"-X property=value, not %s\n", name);
exit(1);
}
*val = '\0';
val++;
res = RD_KAFKA_CONF_UNKNOWN;
/* Try "topic." prefixed properties on topic
* conf first, and then fall through to global if
* it didnt match a topic configuration property. */
if (!strncmp(name, "topic.", strlen("topic.")))
res = rd_kafka_topic_conf_set(topic_conf,
name+
strlen("topic."),
val,
errstr,
sizeof(errstr));
if (res == RD_KAFKA_CONF_UNKNOWN)
res = rd_kafka_conf_set(conf, name, val,
errstr, sizeof(errstr));
if (res != RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
}
break;
case 'D':
case 'O':
mode = opt;
break;
default:
goto usage;
}
}
if (do_conf_dump) {
const char **arr;
size_t cnt;
int pass;
for (pass = 0 ; pass < 2 ; pass++) {
if (pass == 0) {
arr = rd_kafka_conf_dump(conf, &cnt);
printf("# Global config\n");
} else {
printf("# Topic config\n");
arr = rd_kafka_topic_conf_dump(topic_conf,
&cnt);
}
for (i = 0 ; i < (int)cnt ; i += 2)
printf("%s = %s\n",
arr[i], arr[i+1]);
printf("\n");
rd_kafka_conf_dump_free(arr, cnt);
}
exit(0);
}
if (strchr("OC", mode) && optind == argc) {
usage:
fprintf(stderr,
"Usage: %s [options] <topic[:part]> <topic[:part]>..\n"
"\n"
"librdkafka version %s (0x%08x)\n"
"\n"
" Options:\n"
" -g <group> Consumer group (%s)\n"
" -b <brokers> Broker address (%s)\n"
" -e Exit consumer when last message\n"
" in partition has been received.\n"
" -D Describe group.\n"
" -O Get commmitted offset(s)\n"
" -d [facs..] Enable debugging contexts:\n"
" %s\n"
" -q Be quiet\n"
" -A Raw payload output (consumer)\n"
" -X <prop=name> Set arbitrary librdkafka "
"configuration property\n"
" Properties prefixed with \"topic.\" "
"will be set on topic object.\n"
" Use '-X list' to see the full list\n"
" of supported properties.\n"
"\n"
"\n",
argv[0],
rd_kafka_version_str(), rd_kafka_version(),
group, brokers,
RD_KAFKA_DEBUG_CONTEXTS);
exit(1);
}
signal(SIGINT, stop);
signal(SIGUSR1, sig_usr1);
if (debug &&
rd_kafka_conf_set(conf, "debug", debug, errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% Debug configuration failed: %s: %s\n",
errstr, debug);
exit(1);
}
/*
* Client/Consumer group
*/
if (strchr("CO", mode)) {
/* Consumer groups require a group id */
if (!group)
group = "rdkafka_consumer_example";
if (rd_kafka_conf_set(conf, "group.id", group,
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
/* Consumer groups always use broker based offset storage */
if (rd_kafka_topic_conf_set(topic_conf, "offset.store.method",
"broker",
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
/* Set default topic config for pattern-matched topics. */
rd_kafka_conf_set_default_topic_conf(conf, topic_conf);
/* Callback called on partition assignment changes */
rd_kafka_conf_set_rebalance_cb(conf, rebalance_cb);
}
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new consumer: %s\n",
errstr);
exit(1);
}
rd_kafka_set_log_level(rk, LOG_DEBUG);
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
if (mode == 'D') {
int r;
/* Describe groups */
r = describe_groups(rk, group);
rd_kafka_destroy(rk);
exit(r == -1 ? 1 : 0);
}
/* Redirect rd_kafka_poll() to consumer_poll() */
rd_kafka_poll_set_consumer(rk);
topics = rd_kafka_topic_partition_list_new(argc - optind);
for (i = optind ; i < argc ; i++) {
/* Parse "topic[:part] */
char *topic = argv[i];
char *t;
int32_t partition = -1;
if ((t = strstr(topic, ":"))) {
*t = '\0';
partition = atoi(t+1);
}
rd_kafka_topic_partition_list_add(topics, topic, partition);
}
if (mode == 'O') {
/* Offset query */
err = rd_kafka_position(rk, topics, 5000);
if (err) {
fprintf(stderr, "%% Failed to fetch offsets: %s\n",
rd_kafka_err2str(err));
exit(1);
}
for (i = 0 ; i < topics->cnt ; i++) {
rd_kafka_topic_partition_t *p = &topics->elems[i];
printf("Topic \"%s\" partition %"PRId32,
p->topic, p->partition);
if (p->err)
printf(" error %s",
rd_kafka_err2str(p->err));
else {
printf(" offset %"PRId64"",
p->offset);
if (p->metadata_size)
printf(" (%d bytes of metadata)",
(int)p->metadata_size);
}
printf("\n");
}
goto done;
}
if ((err = rd_kafka_subscribe(rk, topics))) {
fprintf(stderr, "%% Failed to start consuming topics: %s\n",
rd_kafka_err2str(err));
exit(1);
}
while (run) {
rd_kafka_message_t *rkmessage;
rkmessage = rd_kafka_consumer_poll(rk, 1000);
if (rkmessage) {
msg_consume(rkmessage, NULL);
rd_kafka_message_destroy(rkmessage);
}
}
done:
err = rd_kafka_consumer_close(rk);
if (err)
fprintf(stderr, "%% Failed to close consumer: %s\n",
rd_kafka_err2str(err));
else
fprintf(stderr, "%% Consumer closed\n");
rd_kafka_topic_partition_list_destroy(topics);
/* Destroy handle */
rd_kafka_destroy(rk);
/* Let background threads clean up and terminate cleanly. */
run = 5;
while (run-- > 0 && rd_kafka_wait_destroyed(1000) == -1)
printf("Waiting for librdkafka to decommission\n");
if (run <= 0)
rd_kafka_dump(stdout, rk);
return 0;
}
/**
* msg_consume
*
* Callback on message receipt.
*
* @param rkmessage rd_kafka_message_t* the message
* @param opaque void* opaque context
*/
static void msg_consume(rd_kafka_message_t *rkmessage, HermannInstanceConfig *cfg) {
if (rkmessage->err) {
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
if (cfg->exit_eof) {
fprintf(stderr,
"%% Consumer reached end of %s [%"PRId32"] "
"message queue at offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition, rkmessage->offset);
cfg->run = 0;
}
return;
}
fprintf(stderr, "%% Consume error for topic \"%s\" [%"PRId32"] "
"offset %"PRId64": %s\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset,
rd_kafka_message_errstr(rkmessage));
return;
}
if (DEBUG && rkmessage->key_len) {
if (output == OUTPUT_HEXDUMP) {
hexdump(stdout, "Message Key",
rkmessage->key, rkmessage->key_len);
}
else {
printf("Key: %.*s\n",
(int)rkmessage->key_len, (char *)rkmessage->key);
}
}
if (output == OUTPUT_HEXDUMP) {
if (DEBUG) {
hexdump(stdout, "Message Payload", rkmessage->payload, rkmessage->len);
}
}
else {
if (DEBUG) {
printf("%.*s\n", (int)rkmessage->len, (char *)rkmessage->payload);
}
}
// Yield the data to the Consumer's block
if (rb_block_given_p()) {
VALUE key, data, offset;
data = rb_str_new((char *)rkmessage->payload, rkmessage->len);
offset = rb_ll2inum(rkmessage->offset);
if ( rkmessage->key_len > 0 ) {
key = rb_str_new((char*) rkmessage->key, (int)rkmessage->key_len);
} else {
key = Qnil;
}
rd_kafka_message_destroy(rkmessage);
rb_yield_values(3, data, key, offset);
}
else {
if (DEBUG) {
fprintf(stderr, "No block given\n"); // todo: should this be an error?
}
}
}
int main_0006_symbols (int argc, char **argv) {
if (argc < 0 /* always false */) {
rd_kafka_version();
rd_kafka_version_str();
rd_kafka_get_debug_contexts();
rd_kafka_get_err_descs(NULL, NULL);
rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR);
rd_kafka_err2name(RD_KAFKA_RESP_ERR_NO_ERROR);
rd_kafka_errno2err(EINVAL);
rd_kafka_errno();
rd_kafka_last_error();
rd_kafka_conf_new();
rd_kafka_conf_destroy(NULL);
rd_kafka_conf_dup(NULL);
rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_conf_set_dr_cb(NULL, NULL);
rd_kafka_conf_set_dr_msg_cb(NULL, NULL);
rd_kafka_conf_set_error_cb(NULL, NULL);
rd_kafka_conf_set_stats_cb(NULL, NULL);
rd_kafka_conf_set_log_cb(NULL, NULL);
rd_kafka_conf_set_socket_cb(NULL, NULL);
rd_kafka_conf_set_rebalance_cb(NULL, NULL);
rd_kafka_conf_set_offset_commit_cb(NULL, NULL);
rd_kafka_conf_set_throttle_cb(NULL, NULL);
rd_kafka_conf_set_default_topic_conf(NULL, NULL);
rd_kafka_conf_get(NULL, NULL, NULL, NULL);
#ifndef _MSC_VER
rd_kafka_conf_set_open_cb(NULL, NULL);
#endif
rd_kafka_conf_set_opaque(NULL, NULL);
rd_kafka_opaque(NULL);
rd_kafka_conf_dump(NULL, NULL);
rd_kafka_topic_conf_dump(NULL, NULL);
rd_kafka_conf_dump_free(NULL, 0);
rd_kafka_conf_properties_show(NULL);
rd_kafka_topic_conf_new();
rd_kafka_topic_conf_dup(NULL);
rd_kafka_topic_conf_destroy(NULL);
rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_topic_conf_set_opaque(NULL, NULL);
rd_kafka_topic_conf_get(NULL, NULL, NULL, NULL);
rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL);
rd_kafka_topic_partition_available(NULL, 0);
rd_kafka_topic_opaque(NULL);
rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_msg_partitioner_consistent(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_msg_partitioner_consistent_random(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_new(0, NULL, NULL, 0);
rd_kafka_destroy(NULL);
rd_kafka_name(NULL);
rd_kafka_memberid(NULL);
rd_kafka_topic_new(NULL, NULL, NULL);
rd_kafka_topic_destroy(NULL);
rd_kafka_topic_name(NULL);
rd_kafka_message_destroy(NULL);
rd_kafka_message_errstr(NULL);
rd_kafka_message_timestamp(NULL, NULL);
rd_kafka_consume_start(NULL, 0, 0);
rd_kafka_consume_stop(NULL, 0);
rd_kafka_consume(NULL, 0, 0);
rd_kafka_consume_batch(NULL, 0, 0, NULL, 0);
rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL);
rd_kafka_offset_store(NULL, 0, 0);
rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL);
rd_kafka_produce_batch(NULL, 0, 0, NULL, 0);
rd_kafka_poll(NULL, 0);
rd_kafka_brokers_add(NULL, NULL);
/* DEPRECATED: rd_kafka_set_logger(NULL, NULL); */
rd_kafka_set_log_level(NULL, 0);
rd_kafka_log_print(NULL, 0, NULL, NULL);
#ifndef _MSC_VER
rd_kafka_log_syslog(NULL, 0, NULL, NULL);
#endif
rd_kafka_outq_len(NULL);
rd_kafka_dump(NULL, NULL);
rd_kafka_thread_cnt();
rd_kafka_wait_destroyed(0);
rd_kafka_metadata(NULL, 0, NULL, NULL, 0);
rd_kafka_metadata_destroy(NULL);
rd_kafka_queue_destroy(NULL);
rd_kafka_consume_start_queue(NULL, 0, 0, NULL);
rd_kafka_consume_queue(NULL, 0);
rd_kafka_consume_batch_queue(NULL, 0, NULL, 0);
rd_kafka_consume_callback_queue(NULL, 0, NULL, NULL);
rd_kafka_seek(NULL, 0, 0, 0);
rd_kafka_yield(NULL);
rd_kafka_mem_free(NULL, NULL);
rd_kafka_list_groups(NULL, NULL, NULL, 0);
rd_kafka_group_list_destroy(NULL);
/* KafkaConsumer API */
rd_kafka_subscribe(NULL, NULL);
rd_kafka_unsubscribe(NULL);
rd_kafka_subscription(NULL, NULL);
rd_kafka_consumer_poll(NULL, 0);
rd_kafka_consumer_close(NULL);
rd_kafka_assign(NULL, NULL);
rd_kafka_assignment(NULL, NULL);
rd_kafka_commit(NULL, NULL, 0);
rd_kafka_commit_message(NULL, NULL, 0);
rd_kafka_committed(NULL, NULL, 0);
rd_kafka_position(NULL, NULL);
/* TopicPartition */
rd_kafka_topic_partition_list_new(0);
rd_kafka_topic_partition_list_destroy(NULL);
rd_kafka_topic_partition_list_add(NULL, NULL, 0);
rd_kafka_topic_partition_list_add_range(NULL, NULL, 0, 0);
rd_kafka_topic_partition_list_del(NULL, NULL, 0);
rd_kafka_topic_partition_list_del_by_idx(NULL, 0);
rd_kafka_topic_partition_list_copy(NULL);
rd_kafka_topic_partition_list_set_offset(NULL, NULL, 0, 0);
rd_kafka_topic_partition_list_find(NULL, NULL, 0);
rd_kafka_query_watermark_offsets(NULL, NULL, 0, NULL, NULL, 0);
rd_kafka_get_watermark_offsets(NULL, NULL, 0, NULL, NULL);
}
return 0;
}
/**
* Returns offset of the last message consumed
*/
int64_t test_consume_msgs (const char *what, rd_kafka_topic_t *rkt,
uint64_t testid, int32_t partition, int64_t offset,
int exp_msg_base, int exp_cnt, int parse_fmt) {
int cnt = 0;
int msg_next = exp_msg_base;
int fails = 0;
int64_t offset_last = -1;
test_timing_t t_first, t_all;
TEST_SAY("%s: consume_msgs: %s [%"PRId32"]: expect msg #%d..%d "
"at offset %"PRId64"\n",
what, rd_kafka_topic_name(rkt), partition,
exp_msg_base, exp_cnt, offset);
if (offset != TEST_NO_SEEK) {
rd_kafka_resp_err_t err;
test_timing_t t_seek;
TIMING_START(&t_seek, "SEEK");
if ((err = rd_kafka_seek(rkt, partition, offset, 5000)))
TEST_FAIL("%s: consume_msgs: %s [%"PRId32"]: "
"seek to %"PRId64" failed: %s\n",
what, rd_kafka_topic_name(rkt), partition,
offset, rd_kafka_err2str(err));
TIMING_STOP(&t_seek);
TEST_SAY("%s: seeked to offset %"PRId64"\n", what, offset);
}
TIMING_START(&t_first, "FIRST MSG");
TIMING_START(&t_all, "ALL MSGS");
while (cnt < exp_cnt) {
rd_kafka_message_t *rkmessage;
int msg_id;
rkmessage = rd_kafka_consume(rkt, partition, 5000);
if (!rkmessage)
TEST_FAIL("%s: consume_msgs: %s [%"PRId32"]: "
"expected msg #%d (%d/%d): timed out\n",
what, rd_kafka_topic_name(rkt), partition,
msg_next, cnt, exp_cnt);
if (rkmessage->err)
TEST_FAIL("%s: consume_msgs: %s [%"PRId32"]: "
"expected msg #%d (%d/%d): got error: %s\n",
what, rd_kafka_topic_name(rkt), partition,
msg_next, cnt, exp_cnt,
rd_kafka_err2str(rkmessage->err));
if (cnt == 0)
TIMING_STOP(&t_first);
if (parse_fmt)
test_msg_parse(testid, rkmessage->key,
rkmessage->key_len, partition, &msg_id);
else
msg_id = 0;
if (test_level >= 3)
TEST_SAY("%s: consume_msgs: %s [%"PRId32"]: "
"got msg #%d at offset %"PRId64
" (expect #%d at offset %"PRId64")\n",
what, rd_kafka_topic_name(rkt), partition,
msg_id, rkmessage->offset,
msg_next,
offset >= 0 ? offset + cnt : -1);
if (parse_fmt && msg_id != msg_next) {
TEST_SAY("%s: consume_msgs: %s [%"PRId32"]: "
"expected msg #%d (%d/%d): got msg #%d\n",
what, rd_kafka_topic_name(rkt), partition,
msg_next, cnt, exp_cnt, msg_id);
fails++;
}
cnt++;
msg_next++;
offset_last = rkmessage->offset;
rd_kafka_message_destroy(rkmessage);
}
TIMING_STOP(&t_all);
if (fails)
TEST_FAIL("%s: consume_msgs: %s [%"PRId32"]: %d failures\n",
what, rd_kafka_topic_name(rkt), partition, fails);
TEST_SAY("%s: consume_msgs: %s [%"PRId32"]: "
"%d/%d messages consumed succesfully\n",
what, rd_kafka_topic_name(rkt), partition,
cnt, exp_cnt);
return offset_last;
}
static void consume_messages (uint64_t testid, const char *topic,
int32_t partition, int msg_base, int batch_cnt,
int msgcnt) {
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char errstr[512];
int i;
test_conf_init(&conf, &topic_conf, 20);
/* Create kafka instance */
rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr));
if (!rk)
TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr);
TEST_SAY("Created kafka instance %s\n", rd_kafka_name(rk));
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
rd_strerror(errno));
TEST_SAY("Consuming %i messages from partition %i\n",
batch_cnt, partition);
/* Consume messages */
if (rd_kafka_consume_start(rkt, partition,
RD_KAFKA_OFFSET_TAIL(batch_cnt)) == -1)
TEST_FAIL("consume_start(%i, -%i) failed: %s",
(int)partition, batch_cnt,
rd_kafka_err2str(rd_kafka_errno2err(errno)));
for (i = 0 ; i < batch_cnt ; ) {
rd_kafka_message_t *rkmessage;
rkmessage = rd_kafka_consume(rkt, partition, 5000);
if (!rkmessage)
TEST_FAIL("Failed to consume message %i/%i from "
"partition %i: %s",
i, batch_cnt, (int)partition,
rd_kafka_err2str(rd_kafka_errno2err(errno)));
if (rkmessage->err) {
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF){
rd_kafka_message_destroy(rkmessage);
continue;
}
TEST_FAIL("Consume message %i/%i from partition %i "
"has error: %s",
i, batch_cnt, (int)partition,
rd_kafka_err2str(rkmessage->err));
}
verify_consumed_msg(testid, partition, msg_base+i, rkmessage);
rd_kafka_message_destroy(rkmessage);
i++;
}
rd_kafka_consume_stop(rkt, partition);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy rdkafka instance */
TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
rd_kafka_destroy(rk);
}
int main (int argc, char **argv) {
rd_kafka_topic_t *rkt;
char *brokers = "localhost:9092";
char mode = 'C';
char *topic = NULL;
int partition = RD_KAFKA_PARTITION_UA;
int opt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char errstr[512];
const char *debug = NULL;
int64_t start_offset = 0;
int report_offsets = 0;
int do_conf_dump = 0;
quiet = !isatty(STDIN_FILENO);
/* Kafka configuration */
conf = rd_kafka_conf_new();
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
while ((opt = getopt(argc, argv, "PCLt:p:b:z:qd:o:eX:A")) != -1) {
switch (opt) {
case 'P':
case 'C':
case 'L':
mode = opt;
break;
case 't':
topic = optarg;
break;
case 'p':
partition = atoi(optarg);
break;
case 'b':
brokers = optarg;
break;
case 'z':
if (rd_kafka_conf_set(conf, "compression.codec",
optarg,
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
break;
case 'o':
if (!strcmp(optarg, "end"))
start_offset = RD_KAFKA_OFFSET_END;
else if (!strcmp(optarg, "beginning"))
start_offset = RD_KAFKA_OFFSET_BEGINNING;
else if (!strcmp(optarg, "stored"))
start_offset = RD_KAFKA_OFFSET_STORED;
else if (!strcmp(optarg, "report"))
report_offsets = 1;
else
start_offset = strtoll(optarg, NULL, 10);
break;
case 'e':
exit_eof = 1;
break;
case 'd':
debug = optarg;
break;
case 'q':
quiet = 1;
break;
case 'A':
output = OUTPUT_RAW;
break;
case 'X':
{
char *name, *val;
rd_kafka_conf_res_t res;
if (!strcmp(optarg, "list") ||
!strcmp(optarg, "help")) {
rd_kafka_conf_properties_show(stdout);
exit(0);
}
if (!strcmp(optarg, "dump")) {
do_conf_dump = 1;
continue;
}
name = optarg;
if (!(val = strchr(name, '='))) {
fprintf(stderr, "%% Expected "
"-X property=value, not %s\n", name);
exit(1);
}
*val = '\0';
val++;
res = RD_KAFKA_CONF_UNKNOWN;
/* Try "topic." prefixed properties on topic
* conf first, and then fall through to global if
* it didnt match a topic configuration property. */
if (!strncmp(name, "topic.", strlen("topic.")))
res = rd_kafka_topic_conf_set(topic_conf,
name+
strlen("topic."),
val,
errstr,
sizeof(errstr));
if (res == RD_KAFKA_CONF_UNKNOWN)
res = rd_kafka_conf_set(conf, name, val,
errstr, sizeof(errstr));
if (res != RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
}
break;
default:
goto usage;
}
}
if (do_conf_dump) {
const char **arr;
size_t cnt;
int pass;
for (pass = 0 ; pass < 2 ; pass++) {
int i;
if (pass == 0) {
arr = rd_kafka_conf_dump(conf, &cnt);
printf("# Global config\n");
} else {
printf("# Topic config\n");
arr = rd_kafka_topic_conf_dump(topic_conf,
&cnt);
}
for (i = 0 ; i < cnt ; i += 2)
printf("%s = %s\n",
arr[i], arr[i+1]);
printf("\n");
rd_kafka_conf_dump_free(arr, cnt);
}
exit(0);
}
if (optind != argc || (mode != 'L' && !topic)) {
usage:
fprintf(stderr,
"Usage: %s -C|-P|-L -t <topic> "
"[-p <partition>] [-b <host1:port1,host2:port2,..>]\n"
"\n"
"librdkafka version %s (0x%08x)\n"
"\n"
" Options:\n"
" -C | -P Consumer or Producer mode\n"
" -L Metadata list mode\n"
" -t <topic> Topic to fetch / produce\n"
" -p <num> Partition (random partitioner)\n"
" -b <brokers> Broker address (localhost:9092)\n"
" -z <codec> Enable compression:\n"
" none|gzip|snappy\n"
" -o <offset> Start offset (consumer)\n"
" -o report Report message offsets (producer)\n"
" -e Exit consumer when last message\n"
" in partition has been received.\n"
" -d [facs..] Enable debugging contexts:\n"
" -q Be quiet\n"
" -A Raw payload output (consumer)\n"
" %s\n"
" -X <prop=name> Set arbitrary librdkafka "
"configuration property\n"
" Properties prefixed with \"topic.\" "
"will be set on topic object.\n"
" Use '-X list' to see the full list\n"
" of supported properties.\n"
"\n"
" In Consumer mode:\n"
" writes fetched messages to stdout\n"
" In Producer mode:\n"
" reads messages from stdin and sends to broker\n"
" In List mode:\n"
" queries broker for metadata information, "
"topic is optional.\n"
"\n"
"\n"
"\n",
argv[0],
rd_kafka_version_str(), rd_kafka_version(),
RD_KAFKA_DEBUG_CONTEXTS);
exit(1);
}
signal(SIGINT, stop);
signal(SIGUSR1, sig_usr1);
if (debug &&
rd_kafka_conf_set(conf, "debug", debug, errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% Debug configuration failed: %s: %s\n",
errstr, debug);
exit(1);
}
if (mode == 'P') {
/*
* Producer
*/
char buf[2048];
int sendcnt = 0;
/* Set up a message delivery report callback.
* It will be called once for each message, either on successful
* delivery to broker, or upon failure to deliver to broker. */
/* If offset reporting (-o report) is enabled, use the
* richer dr_msg_cb instead. */
if (report_offsets) {
rd_kafka_topic_conf_set(topic_conf,
"produce.offset.report",
"true", errstr, sizeof(errstr));
rd_kafka_conf_set_dr_msg_cb(conf, msg_delivered2);
} else
rd_kafka_conf_set_dr_cb(conf, msg_delivered);
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new producer: %s\n",
errstr);
exit(1);
}
/* Set logger */
rd_kafka_set_logger(rk, logger);
rd_kafka_set_log_level(rk, LOG_DEBUG);
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Create topic */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (!quiet)
fprintf(stderr,
"%% Type stuff and hit enter to send\n");
while (run && fgets(buf, sizeof(buf), stdin)) {
size_t len = strlen(buf);
if (buf[len-1] == '\n')
buf[--len] = '\0';
/* Send/Produce message. */
if (rd_kafka_produce(rkt, partition,
RD_KAFKA_MSG_F_COPY,
/* Payload and length */
buf, len,
/* Optional key and its length */
NULL, 0,
/* Message opaque, provided in
* delivery report callback as
* msg_opaque. */
NULL) == -1) {
fprintf(stderr,
"%% Failed to produce to topic %s "
"partition %i: %s\n",
rd_kafka_topic_name(rkt), partition,
rd_kafka_err2str(
rd_kafka_errno2err(errno)));
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
continue;
}
if (!quiet)
fprintf(stderr, "%% Sent %zd bytes to topic "
"%s partition %i\n",
len, rd_kafka_topic_name(rkt), partition);
sendcnt++;
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
}
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
/* Wait for messages to be delivered */
while (run && rd_kafka_outq_len(rk) > 0)
rd_kafka_poll(rk, 100);
/* Destroy the handle */
rd_kafka_destroy(rk);
} else if (mode == 'C') {
/*
* Consumer
*/
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new consumer: %s\n",
errstr);
exit(1);
}
/* Set logger */
rd_kafka_set_logger(rk, logger);
rd_kafka_set_log_level(rk, LOG_DEBUG);
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Create topic */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
/* Start consuming */
if (rd_kafka_consume_start(rkt, partition, start_offset) == -1){
fprintf(stderr, "%% Failed to start consuming: %s\n",
rd_kafka_err2str(rd_kafka_errno2err(errno)));
exit(1);
}
while (run) {
rd_kafka_message_t *rkmessage;
/* Consume single message.
* See rdkafka_performance.c for high speed
* consuming of messages. */
rkmessage = rd_kafka_consume(rkt, partition, 1000);
if (!rkmessage) /* timeout */
continue;
msg_consume(rkmessage, NULL);
/* Return message to rdkafka */
rd_kafka_message_destroy(rkmessage);
}
/* Stop consuming */
rd_kafka_consume_stop(rkt, partition);
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
} else if (mode == 'L') {
rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new producer: %s\n",
errstr);
exit(1);
}
/* Set logger */
rd_kafka_set_logger(rk, logger);
rd_kafka_set_log_level(rk, LOG_DEBUG);
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Create topic */
if (topic)
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
else
rkt = NULL;
while (run) {
const struct rd_kafka_metadata *metadata;
/* Fetch metadata */
err = rd_kafka_metadata(rk, rkt ? 0 : 1, rkt,
&metadata, 5000);
if (err != RD_KAFKA_RESP_ERR_NO_ERROR) {
fprintf(stderr,
"%% Failed to acquire metadata: %s\n",
rd_kafka_err2str(err));
run = 0;
break;
}
metadata_print(topic, metadata);
rd_kafka_metadata_destroy(metadata);
run = 0;
}
/* Destroy the handle */
rd_kafka_destroy(rk);
/* Exit right away, dont wait for background cleanup, we haven't
* done anything important anyway. */
exit(err ? 2 : 0);
}
/* Let background threads clean up and terminate cleanly. */
rd_kafka_wait_destroyed(2000);
return 0;
}
static void do_offset_test (const char *what, int auto_commit, int auto_store,
int async) {
test_timing_t t_all;
char groupid[64];
rd_kafka_t *rk;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *tconf;
int cnt = 0;
const int extra_cnt = 5;
rd_kafka_resp_err_t err;
rd_kafka_topic_partition_list_t *parts;
rd_kafka_topic_partition_t *rktpar;
int64_t next_offset = -1;
test_conf_init(&conf, &tconf, 20);
test_conf_set(conf, "enable.auto.commit", auto_commit ? "true":"false");
test_conf_set(conf, "enable.auto.offset.store", auto_store ?"true":"false");
test_conf_set(conf, "auto.commit.interval.ms", "500");
rd_kafka_conf_set_offset_commit_cb(conf, offset_commit_cb);
test_topic_conf_set(tconf, "auto.offset.reset", "smallest");
test_str_id_generate(groupid, sizeof(groupid));
test_conf_set(conf, "group.id", groupid);
rd_kafka_conf_set_default_topic_conf(conf, tconf);
TEST_SAY(_C_MAG "[ do_offset_test: %s with group.id %s ]\n",
what, groupid);
TIMING_START(&t_all, what);
expected_offset = 0;
committed_offset = -1;
/* MO:
* - Create consumer.
* - Start consuming from beginning
* - Perform store & commits according to settings
* - Stop storing&committing when half of the messages are consumed,
* - but consume 5 more to check against.
* - Query position.
* - Destroy consumer.
* - Create new consumer with same group.id using stored offsets
* - Should consume the expected message.
*/
/* Create kafka instance */
rk = test_create_handle(RD_KAFKA_CONSUMER, rd_kafka_conf_dup(conf));
rd_kafka_poll_set_consumer(rk);
test_consumer_subscribe(rk, topic);
while (cnt - extra_cnt < msgcnt / 2) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(rk, 10*1000);
if (!rkm)
continue;
if (rkm->err == RD_KAFKA_RESP_ERR__TIMED_OUT)
TEST_FAIL("%s: Timed out waiting for message %d", what,cnt);
else if (rkm->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
rd_kafka_message_destroy(rkm);
continue;
} else if (rkm->err)
TEST_FAIL("%s: Consumer error: %s",
what, rd_kafka_message_errstr(rkm));
/* Offset of next message. */
next_offset = rkm->offset + 1;
if (cnt < msgcnt / 2) {
if (!auto_store) {
err = rd_kafka_offset_store(rkm->rkt,rkm->partition,
rkm->offset);
if (err)
TEST_FAIL("%s: offset_store failed: %s\n",
what, rd_kafka_err2str(err));
}
expected_offset = rkm->offset+1;
if (!auto_commit) {
test_timing_t t_commit;
TIMING_START(&t_commit,
async?"commit.async":"commit.sync");
err = rd_kafka_commit_message(rk, rkm, async);
TIMING_STOP(&t_commit);
if (err)
TEST_FAIL("%s: commit failed: %s\n",
what, rd_kafka_err2str(err));
}
} else if (auto_store && auto_commit)
expected_offset = rkm->offset+1;
rd_kafka_message_destroy(rkm);
cnt++;
}
TEST_SAY("%s: done consuming after %d messages, at offset %"PRId64"\n",
what, cnt, expected_offset);
if ((err = rd_kafka_assignment(rk, &parts)))
TEST_FAIL("%s: failed to get assignment(): %s\n",
what, rd_kafka_err2str(err));
/* Verify position */
if ((err = rd_kafka_position(rk, parts)))
TEST_FAIL("%s: failed to get position(): %s\n",
what, rd_kafka_err2str(err));
if (!(rktpar = rd_kafka_topic_partition_list_find(parts,
topic, partition)))
TEST_FAIL("%s: position(): topic lost\n", what);
if (rktpar->offset != next_offset)
TEST_FAIL("%s: Expected position() offset %"PRId64", got %"PRId64,
what, next_offset, rktpar->offset);
TEST_SAY("%s: Position is at %"PRId64", good!\n",
what, rktpar->offset);
/* Pause messages while waiting so we can serve callbacks
* without having more messages received. */
if ((err = rd_kafka_pause_partitions(rk, parts)))
TEST_FAIL("%s: failed to pause partitions: %s\n",
what, rd_kafka_err2str(err));
rd_kafka_topic_partition_list_destroy(parts);
/* Fire off any enqueued offset_commit_cb */
test_consumer_poll_no_msgs(what, rk, testid, 0);
TEST_SAY("%s: committed_offset %"PRId64", expected_offset %"PRId64"\n",
what, committed_offset, expected_offset);
if (!auto_commit && !async) {
/* Sync commits should be up to date at this point. */
if (committed_offset != expected_offset)
TEST_FAIL("%s: Sync commit: committed offset %"PRId64
" should be same as expected offset "
"%"PRId64,
what, committed_offset, expected_offset);
} else {
/* Wait for offset commits to catch up */
while (committed_offset < expected_offset) {
TEST_SAYL(3, "%s: Wait for committed offset %"PRId64
" to reach expected offset %"PRId64"\n",
what, committed_offset, expected_offset);
test_consumer_poll_no_msgs(what, rk, testid, 1000);
}
}
TEST_SAY("%s: phase 1 complete, %d messages consumed, "
"next expected offset is %"PRId64"\n",
what, cnt, expected_offset);
/* Issue #827: cause committed() to return prematurely by specifying
* low timeout. The bug (use after free) will only
* be catched by valgrind. */
do {
parts = rd_kafka_topic_partition_list_new(1);
rd_kafka_topic_partition_list_add(parts, topic, partition);
err = rd_kafka_committed(rk, parts, 1);
rd_kafka_topic_partition_list_destroy(parts);
TEST_SAY("Issue #827: committed() returned %s\n",
rd_kafka_err2str(err));
} while (err != RD_KAFKA_RESP_ERR__TIMED_OUT);
/* Query position */
parts = rd_kafka_topic_partition_list_new(1);
rd_kafka_topic_partition_list_add(parts, topic, partition);
err = rd_kafka_committed(rk, parts, tmout_multip(5*1000));
if (err)
TEST_FAIL("%s: committed() failed: %s", what, rd_kafka_err2str(err));
if (!(rktpar = rd_kafka_topic_partition_list_find(parts,
topic, partition)))
TEST_FAIL("%s: committed(): topic lost\n", what);
if (rktpar->offset != expected_offset)
TEST_FAIL("%s: Expected committed() offset %"PRId64", got %"PRId64,
what, expected_offset, rktpar->offset);
TEST_SAY("%s: Committed offset is at %"PRId64", good!\n",
what, rktpar->offset);
rd_kafka_topic_partition_list_destroy(parts);
test_consumer_close(rk);
rd_kafka_destroy(rk);
/* Fire up a new consumer and continue from where we left off. */
TEST_SAY("%s: phase 2: starting new consumer to resume consumption\n",what);
rk = test_create_handle(RD_KAFKA_CONSUMER, conf);
rd_kafka_poll_set_consumer(rk);
test_consumer_subscribe(rk, topic);
while (cnt < msgcnt) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(rk, 10*1000);
if (!rkm)
continue;
if (rkm->err == RD_KAFKA_RESP_ERR__TIMED_OUT)
TEST_FAIL("%s: Timed out waiting for message %d", what,cnt);
else if (rkm->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
rd_kafka_message_destroy(rkm);
continue;
} else if (rkm->err)
TEST_FAIL("%s: Consumer error: %s",
what, rd_kafka_message_errstr(rkm));
if (rkm->offset != expected_offset)
TEST_FAIL("%s: Received message offset %"PRId64
", expected %"PRId64" at msgcnt %d/%d\n",
what, rkm->offset, expected_offset,
cnt, msgcnt);
rd_kafka_message_destroy(rkm);
expected_offset++;
cnt++;
}
TEST_SAY("%s: phase 2: complete\n", what);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_all);
}
/*
* kafka_consume_main
*
* Main function for Kafka consumers running as background workers
*/
void
kafka_consume_main(Datum arg)
{
char err_msg[512];
rd_kafka_topic_conf_t *topic_conf;
rd_kafka_t *kafka;
rd_kafka_topic_t *topic;
rd_kafka_message_t **messages;
const struct rd_kafka_metadata *meta;
struct rd_kafka_metadata_topic topic_meta;
rd_kafka_resp_err_t err;
bool found;
Oid id = (Oid) arg;
ListCell *lc;
KafkaConsumerProc *proc = hash_search(consumer_procs, &id, HASH_FIND, &found);
KafkaConsumer consumer;
CopyStmt *copy;
int valid_brokers = 0;
int i;
int my_partitions = 0;
if (!found)
elog(ERROR, "kafka consumer %d not found", id);
pqsignal(SIGTERM, kafka_consume_main_sigterm);
#define BACKTRACE_SEGFAULTS
#ifdef BACKTRACE_SEGFAULTS
pqsignal(SIGSEGV, debug_segfault);
#endif
/* we're now ready to receive signals */
BackgroundWorkerUnblockSignals();
/* give this proc access to the database */
BackgroundWorkerInitializeConnection(NameStr(proc->dbname), NULL);
/* load saved consumer state */
StartTransactionCommand();
load_consumer_state(proc->consumer_id, &consumer);
copy = get_copy_statement(&consumer);
topic_conf = rd_kafka_topic_conf_new();
kafka = rd_kafka_new(RD_KAFKA_CONSUMER, NULL, err_msg, sizeof(err_msg));
rd_kafka_set_logger(kafka, logger);
/*
* Add all brokers currently in pipeline_kafka_brokers
*/
if (consumer.brokers == NIL)
elog(ERROR, "no valid brokers were found");
foreach(lc, consumer.brokers)
valid_brokers += rd_kafka_brokers_add(kafka, lfirst(lc));
if (!valid_brokers)
elog(ERROR, "no valid brokers were found");
/*
* Set up our topic to read from
*/
topic = rd_kafka_topic_new(kafka, consumer.topic, topic_conf);
err = rd_kafka_metadata(kafka, false, topic, &meta, CONSUMER_TIMEOUT);
if (err != RD_KAFKA_RESP_ERR_NO_ERROR)
elog(ERROR, "failed to acquire metadata: %s", rd_kafka_err2str(err));
Assert(meta->topic_cnt == 1);
topic_meta = meta->topics[0];
load_consumer_offsets(&consumer, &topic_meta, proc->offset);
CommitTransactionCommand();
/*
* Begin consuming all partitions that this process is responsible for
*/
for (i = 0; i < topic_meta.partition_cnt; i++)
{
int partition = topic_meta.partitions[i].id;
Assert(partition <= consumer.num_partitions);
if (partition % consumer.parallelism != proc->partition_group)
continue;
elog(LOG, "[kafka consumer] %s <- %s consuming partition %d from offset %ld",
consumer.rel->relname, consumer.topic, partition, consumer.offsets[partition]);
if (rd_kafka_consume_start(topic, partition, consumer.offsets[partition]) == -1)
elog(ERROR, "failed to start consuming: %s", rd_kafka_err2str(rd_kafka_errno2err(errno)));
my_partitions++;
}
/*
* No point doing anything if we don't have any partitions assigned to us
*/
if (my_partitions == 0)
{
elog(LOG, "[kafka consumer] %s <- %s consumer %d doesn't have any partitions to read from",
consumer.rel->relname, consumer.topic, MyProcPid);
goto done;
}
messages = palloc0(sizeof(rd_kafka_message_t) * consumer.batch_size);
/*
* Consume messages until we are terminated
*/
while (!got_sigterm)
{
ssize_t num_consumed;
int i;
int messages_buffered = 0;
int partition;
StringInfoData buf;
bool xact = false;
for (partition = 0; partition < consumer.num_partitions; partition++)
{
if (partition % consumer.parallelism != proc->partition_group)
continue;
num_consumed = rd_kafka_consume_batch(topic, partition,
CONSUMER_TIMEOUT, messages, consumer.batch_size);
if (num_consumed <= 0)
continue;
if (!xact)
{
StartTransactionCommand();
xact = true;
}
initStringInfo(&buf);
for (i = 0; i < num_consumed; i++)
{
if (messages[i]->payload != NULL)
{
appendBinaryStringInfo(&buf, messages[i]->payload, messages[i]->len);
if (buf.len > 0 && buf.data[buf.len - 1] != '\n')
appendStringInfoChar(&buf, '\n');
messages_buffered++;
}
consumer.offsets[partition] = messages[i]->offset;
rd_kafka_message_destroy(messages[i]);
}
}
if (!xact)
{
pg_usleep(1 * 1000);
continue;
}
/* we don't want to die in the event of any errors */
PG_TRY();
{
if (messages_buffered)
execute_copy(copy, &buf);
}
PG_CATCH();
{
elog(LOG, "[kafka consumer] %s <- %s failed to process batch, dropped %d message%s:",
consumer.rel->relname, consumer.topic, (int) num_consumed, (num_consumed == 1 ? "" : "s"));
EmitErrorReport();
FlushErrorState();
AbortCurrentTransaction();
xact = false;
}
PG_END_TRY();
if (!xact)
StartTransactionCommand();
if (messages_buffered)
save_consumer_state(&consumer, proc->partition_group);
CommitTransactionCommand();
}
done:
hash_search(consumer_procs, &id, HASH_REMOVE, NULL);
rd_kafka_topic_destroy(topic);
rd_kafka_destroy(kafka);
rd_kafka_wait_destroyed(CONSUMER_TIMEOUT);
}
static void consume_messages_with_queues (uint64_t testid, const char *topic,
int partition_cnt, int msgcnt) {
int r;
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
rd_kafka_queue_t *rkqu;
char errstr[512];
char msg[128];
int failcnt;
int i;
rd_kafka_message_t *rkmessages;
int32_t partition;
int cnt = 0;
int batch_cnt = msgcnt / partition_cnt;
test_conf_init(&conf, &topic_conf, 20);
/* Create kafka instance */
rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr));
if (!rk)
TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr);
TEST_SAY("Created kafka instance %s\n", rd_kafka_name(rk));
/* Create queue */
rkqu = rd_kafka_queue_new(rk);
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
strerror(errno));
TEST_SAY("Consuming %i messages from one queue serving %i partitions\n",
msgcnt, partition_cnt);
/* Start consuming each partition */
for (partition = 0 ; partition < partition_cnt ; partition++) {
/* Consume messages */
TEST_SAY("Start consuming partition %i at offset -%i\n",
partition, batch_cnt);
if (rd_kafka_consume_start_queue(rkt, partition,
RD_KAFKA_OFFSET_TAIL(batch_cnt),
rkqu) == -1)
TEST_FAIL("consume_start_queue(%i) failed: %s",
(int)partition,
rd_kafka_err2str(rd_kafka_errno2err(errno)));
}
/* Consume messages from queue */
for (i = 0 ; i < msgcnt ; i++) {
rd_kafka_message_t *rkmessage;
rkmessage = rd_kafka_consume_queue(rkqu, 5000);
if (!rkmessage)
TEST_FAIL("Failed to consume message %i/%i from "
"queue: %s",
i, msgcnt,
rd_kafka_err2str(rd_kafka_errno2err(errno)));
if (rkmessage->err)
TEST_FAIL("Consume message %i/%i from queue "
"has error (partition %"PRId32"): %s",
i, msgcnt,
rkmessage->partition,
rd_kafka_err2str(rkmessage->err));
verify_consumed_msg(testid, -1, -1, rkmessage);
rd_kafka_message_destroy(rkmessage);
}
/* Stop consuming each partition */
for (partition = 0 ; partition < partition_cnt ; partition++)
rd_kafka_consume_stop(rkt, partition);
/* Destroy queue */
rd_kafka_queue_destroy(rkqu);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy rdkafka instance */
TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
rd_kafka_destroy(rk);
}
