/**
* Handle and print a consumed message.
* Internally crafted messages are also used to propagate state from
* librdkafka to the application. The application needs to check
* the `rkmessage->err` field for this purpose.
*/
static void msg_consume (rd_kafka_message_t *rkmessage,
void *opaque) {
if (rkmessage->err) {
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
fprintf(stderr,
"%% Consumer reached end of %s [%"PRId32"] "
"message queue at offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition, rkmessage->offset);
if (exit_eof && --wait_eof == 0)
run = 0;
return;
}
if (rkmessage->rkt)
fprintf(stderr, "%% Consume error for "
"topic \"%s\" [%"PRId32"] "
"offset %"PRId64": %s\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset,
rd_kafka_message_errstr(rkmessage));
else
fprintf(stderr, "%% Consumer error: %s: %s\n",
rd_kafka_err2str(rkmessage->err),
rd_kafka_message_errstr(rkmessage));
if (rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION ||
rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC)
run = 0;
return;
}
if (!quiet)
fprintf(stdout, "%% Message (topic %s [%"PRId32"], "
"offset %"PRId64", %zd bytes):\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset, rkmessage->len);
if (rkmessage->key_len) {
if (output == OUTPUT_HEXDUMP)
hexdump(stdout, "Message Key",
rkmessage->key, rkmessage->key_len);
else
printf("Key: %.*s\n",
(int)rkmessage->key_len, (char *)rkmessage->key);
}
if (output == OUTPUT_HEXDUMP)
hexdump(stdout, "Message Payload",
rkmessage->payload, rkmessage->len);
else
printf("%.*s\n",
(int)rkmessage->len, (char *)rkmessage->payload);
}
static void msg_consume (rd_kafka_message_t *rkmessage,
void *opaque) {
if (rkmessage->err) {
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
printf("%% Consumer reached end of %s [%"PRId32"] "
"message queue at offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition, rkmessage->offset);
if (exit_eof)
run = 0;
return;
}
printf("%% Consume error for topic \"%s\" [%"PRId32"] "
"offset %"PRId64": %s\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset,
rd_kafka_message_errstr(rkmessage));
return;
}
printf("%% Message (offset %"PRId64", %zd bytes):\n",
rkmessage->offset, rkmessage->len);
if (rkmessage->key_len)
hexdump(stdout, "Message Key",
rkmessage->key, rkmessage->key_len);
hexdump(stdout, "Message Payload", rkmessage->payload, rkmessage->len);
}
static rd_kafka_message_t *msg_consume(rd_kafka_message_t *rkmessage,
void *opaque) {
if (rkmessage->err) {
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO,
"phpkafka - %% Consumer reached end of %s [%"PRId32"] "
"message queue at offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition, rkmessage->offset);
if (exit_eof)
run = 0;
return;
}
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - %% Consume error for topic \"%s\" [%"PRId32"] "
"offset %"PRId64": %s\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset,
rd_kafka_message_errstr(rkmessage));
return;
}
//php_printf("%.*s\n", (int)rkmessage->len, (char *)rkmessage->payload);
return rkmessage;
}
/**
* msg_consume
*
* Callback on message receipt.
*
* @param rkmessage rd_kafka_message_t* the message
* @param opaque void* opaque context
*/
static void msg_consume(rd_kafka_message_t *rkmessage, HermannInstanceConfig *cfg) {
if (rkmessage->err) {
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
if (cfg->exit_eof) {
fprintf(stderr,
"%% Consumer reached end of %s [%"PRId32"] "
"message queue at offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition, rkmessage->offset);
cfg->run = 0;
}
return;
}
fprintf(stderr, "%% Consume error for topic \"%s\" [%"PRId32"] "
"offset %"PRId64": %s\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset,
rd_kafka_message_errstr(rkmessage));
return;
}
if (DEBUG && rkmessage->key_len) {
if (output == OUTPUT_HEXDUMP) {
hexdump(stdout, "Message Key",
rkmessage->key, rkmessage->key_len);
}
else {
printf("Key: %.*s\n",
(int)rkmessage->key_len, (char *)rkmessage->key);
}
}
if (output == OUTPUT_HEXDUMP) {
if (DEBUG) {
hexdump(stdout, "Message Payload", rkmessage->payload, rkmessage->len);
}
}
else {
if (DEBUG) {
printf("%.*s\n", (int)rkmessage->len, (char *)rkmessage->payload);
}
}
// Yield the data to the Consumer's block
if (rb_block_given_p()) {
VALUE value = rb_str_new((char *)rkmessage->payload, rkmessage->len);
rb_yield(value);
}
else {
if (DEBUG) {
fprintf(stderr, "No block given\n"); // todo: should this be an error?
}
}
}
int test_consumer_poll (const char *what, rd_kafka_t *rk, uint64_t testid,
int exp_eof_cnt, int exp_msg_base, int exp_cnt) {
int eof_cnt = 0;
int cnt = 0;
test_timing_t t_cons;
TEST_SAY("%s: consume %d messages\n", what, exp_cnt);
TIMING_START(&t_cons, "CONSUME");
while ((exp_eof_cnt == -1 || eof_cnt < exp_eof_cnt) &&
(cnt < exp_cnt)) {
rd_kafka_message_t *rkmessage;
rkmessage = rd_kafka_consumer_poll(rk, 10*1000);
if (!rkmessage) /* Shouldn't take this long to get a msg */
TEST_FAIL("%s: consumer_poll() timeout\n", what);
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
TEST_SAY("%s [%"PRId32"] reached EOF at "
"offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset);
eof_cnt++;
} else if (rkmessage->err) {
TEST_SAY("%s [%"PRId32"] error (offset %"PRId64"): %s",
rkmessage->rkt ?
rd_kafka_topic_name(rkmessage->rkt) :
"(no-topic)",
rkmessage->partition,
rkmessage->offset,
rd_kafka_message_errstr(rkmessage));
} else {
if (test_level > 2)
TEST_SAY("%s [%"PRId32"] "
"message at offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset);
test_verify_rkmessage(rkmessage, testid, -1, -1);
cnt++;
}
rd_kafka_message_destroy(rkmessage);
}
TIMING_STOP(&t_cons);
TEST_SAY("%s: consumed %d/%d messages (%d/%d EOFs)\n",
what, cnt, exp_cnt, eof_cnt, exp_eof_cnt);
return cnt;
}
int main (int argc, char **argv) {
if (argc < 0 /* always false */) {
rd_kafka_version();
rd_kafka_version_str();
rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR);
rd_kafka_errno2err(EINVAL);
rd_kafka_conf_new();
rd_kafka_conf_destroy(NULL);
rd_kafka_conf_dup(NULL);
rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_conf_set_dr_cb(NULL, NULL);
rd_kafka_conf_set_error_cb(NULL, NULL);
rd_kafka_conf_set_stats_cb(NULL, NULL);
rd_kafka_conf_set_opaque(NULL, NULL);
rd_kafka_conf_dump(NULL, NULL);
rd_kafka_topic_conf_dump(NULL, NULL);
rd_kafka_conf_dump_free(NULL, 0);
rd_kafka_conf_properties_show(NULL);
rd_kafka_topic_conf_new();
rd_kafka_topic_conf_dup(NULL);
rd_kafka_topic_conf_destroy(NULL);
rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_topic_conf_set_opaque(NULL, NULL);
rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL);
rd_kafka_topic_partition_available(NULL, 0);
rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_new(0, NULL, NULL, 0);
rd_kafka_destroy(NULL);
rd_kafka_name(NULL);
rd_kafka_topic_new(NULL, NULL, NULL);
rd_kafka_topic_destroy(NULL);
rd_kafka_topic_name(NULL);
rd_kafka_message_destroy(NULL);
rd_kafka_message_errstr(NULL);
rd_kafka_consume_start(NULL, 0, 0);
rd_kafka_consume_stop(NULL, 0);
rd_kafka_consume(NULL, 0, 0);
rd_kafka_consume_batch(NULL, 0, 0, NULL, 0);
rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL);
rd_kafka_offset_store(NULL, 0, 0);
rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL);
rd_kafka_poll(NULL, 0);
rd_kafka_brokers_add(NULL, NULL);
rd_kafka_set_logger(NULL, NULL);
rd_kafka_set_log_level(NULL, 0);
rd_kafka_log_print(NULL, 0, NULL, NULL);
rd_kafka_log_syslog(NULL, 0, NULL, NULL);
rd_kafka_outq_len(NULL);
rd_kafka_dump(NULL, NULL);
rd_kafka_thread_cnt();
rd_kafka_wait_destroyed(0);
}
return 0;
}
/**
* Message delivery report callback using the richer rd_kafka_message_t object.
*/
static void msg_delivered2 (rd_kafka_t *rk,
const rd_kafka_message_t *rkmessage, void *opaque) {
if (rkmessage->err)
fprintf(stderr, "%% Message delivery failed: %s\n",
rd_kafka_message_errstr(rkmessage));
else if (!quiet)
fprintf(stderr,
"%% Message delivered (%zd bytes, offset %"PRId64")\n",
rkmessage->len, rkmessage->offset);
}
static int partition_consume (void *args) {
part_consume_info_t *info = (part_consume_info_t *)args;
rd_kafka_queue_t *rkqu = info->rkqu;
int partition = info->partition;
int64_t ts_start = test_clock();
int max_time = (test_session_timeout_ms + 3000) * 1000;
int running = 1;
free(args); /* Free the parameter struct dynamically allocated for us */
while (ts_start + max_time > test_clock() && running &&
is_consuming()) {
rd_kafka_message_t *rkmsg;
rkmsg = rd_kafka_consume_queue(rkqu, 500);
if (!rkmsg)
continue;
else if (rkmsg->err == RD_KAFKA_RESP_ERR__PARTITION_EOF)
running = 0;
else if (rkmsg->err) {
mtx_lock(&lock);
TEST_FAIL("Message error "
"(at offset %" PRId64 " after "
"%d/%d messages and %dms): %s",
rkmsg->offset, consumed_msg_cnt, exp_msg_cnt,
(int)(test_clock() - ts_start) / 1000,
rd_kafka_message_errstr(rkmsg));
mtx_unlock(&lock);
} else {
if (rkmsg->partition != partition) {
mtx_lock(&lock);
TEST_FAIL("Message consumed has partition %d "
"but we expected partition %d.",
rkmsg->partition, partition);
mtx_unlock(&lock);
}
}
rd_kafka_message_destroy(rkmsg);
mtx_lock(&lock);
if (running && ++consumed_msg_cnt >= exp_msg_cnt) {
TEST_SAY("All messages consumed\n");
running = 0;
}
mtx_unlock(&lock);
}
rd_kafka_queue_destroy(rkqu);
return thrd_success;
}
void kafka_produce_detailed_cb(rd_kafka_t *rk, const rd_kafka_message_t *msg, void *opaque)
{
struct produce_cb_params *params = opaque;
if (params)
{
params->msg_count -= 1;
}
if (msg->err)
{
int offset = params->errmsg_len,
err_len = 0;
const char *errstr = rd_kafka_message_errstr(msg);
err_len = strlen(errstr);
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "Failed to deliver message: %s", errstr);
}
if (params)
{
params->err_count += 1;
params->err_msg = realloc(
params->err_msg,
(offset + err_len + 2) * sizeof params->err_msg
);
if (params->err_msg == NULL)
{
params->errmsg_len = 0;
}
else
{
strcpy(
params->err_msg + offset,
errstr
);
offset += err_len;//get new strlen
params->err_msg[offset] = '\n';//add new line
++offset;
params->err_msg[offset] = '\0';//ensure zero terminated string
}
}
return;
}
if (params)
{
params->offset = msg->offset;
params->partition = msg->partition;
}
}
/**
* Message delivery report callback using the richer rd_kafka_message_t object.
*/
static void msg_delivered2 (rd_kafka_t *rk,
const rd_kafka_message_t *rkmessage, void *opaque) {
printf("del: %s: offset %"PRId64"\n",
rd_kafka_err2str(rkmessage->err), rkmessage->offset);
if (rkmessage->err)
fprintf(stderr, "%% Message delivery failed: %s\n",
rd_kafka_message_errstr(rkmessage));
else if (!quiet)
fprintf(stderr,
"%% Message delivered (%zd bytes, offset %"PRId64", "
"partition %"PRId32"): %.*s\n",
rkmessage->len, rkmessage->offset,
rkmessage->partition,
(int)rkmessage->len, (const char *)rkmessage->payload);
}
int test_consumer_poll (const char *what, rd_kafka_t *rk, uint64_t testid,
int exp_eof_cnt, int exp_msg_base, int exp_cnt) {
int eof_cnt = 0;
int cnt = 0;
while (eof_cnt < exp_eof_cnt) {
rd_kafka_message_t *rkmessage;
rkmessage = rd_kafka_consumer_poll(rk, 10*1000);
if (!rkmessage) /* Shouldn't take this long to get a msg */
TEST_FAIL("%s: consumer_poll() timeout\n", what);
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
TEST_SAY("%s [%"PRId32"] reached EOF at "
"offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset);
eof_cnt++;
} else if (rkmessage->err) {
TEST_SAY("%s [%"PRId32"] error (offset %"PRId64"): %s",
rkmessage->rkt ?
rd_kafka_topic_name(rkmessage->rkt) :
"(no-topic)",
rkmessage->partition,
rkmessage->offset,
rd_kafka_message_errstr(rkmessage));
} else {
if (test_level > 2)
TEST_SAY("%s [%"PRId32"] "
"message at offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset);
cnt++;
}
rd_kafka_message_destroy(rkmessage);
}
return cnt;
}
static void msg_consume (rd_kafka_message_t *rkmessage, void *opaque) {
if (rkmessage->err) {
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
printf("%% Consumer reached end of %s [%"PRId32"] "
"message queue at offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition, rkmessage->offset);
if (exit_eof)
run = 0;
return;
}
printf("%% Consume error for topic \"%s\" [%"PRId32"] "
"offset %"PRId64": %s\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset,
rd_kafka_message_errstr(rkmessage));
msgs_failed++;
return;
}
cnt.msgs++;
cnt.bytes += rkmessage->len;
if (!(cnt.msgs % 1000000))
printf("@%"PRId64": %.*s\n",
rkmessage->offset,
(int)rkmessage->len, (char *)rkmessage->payload);
#if 0 /* Future API */
/* We store offset when we're done processing
* the current message. */
rd_kafka_offset_store(rkmessage->rkt, rkmessage->partition,
rd_kafka_offset_next(rkmessage));
#endif
}
static void consume_all (rd_kafka_t **rk_c, int rk_cnt, int exp_msg_cnt,
int max_time/*ms*/) {
int64_t ts_start = test_clock();
int i;
max_time *= 1000;
while (ts_start + max_time > test_clock()) {
for (i = 0 ; i < rk_cnt ; i++) {
rd_kafka_message_t *rkmsg;
if (!rk_c[i])
continue;
rkmsg = rd_kafka_consumer_poll(rk_c[i], 500);
if (!rkmsg)
continue;
else if (rkmsg->err)
TEST_SAY("Message error "
"(at offset %"PRId64" after "
"%d/%d messages and %dms): %s\n",
rkmsg->offset,
consumed_msg_cnt, exp_msg_cnt,
(int)(test_clock() - ts_start)/1000,
rd_kafka_message_errstr(rkmsg));
else
consumed_msg_cnt++;
rd_kafka_message_destroy(rkmsg);
if (consumed_msg_cnt >= exp_msg_cnt) {
static int once = 0;
if (!once++)
TEST_SAY("All messages consumed\n");
return;
}
}
}
}
static void do_consume (const char *topic, int msgcnt) {
rd_kafka_t *rk;
rd_kafka_topic_partition_list_t *parts;
rk = test_create_consumer(topic, NULL, NULL, NULL);
parts = rd_kafka_topic_partition_list_new(1);
rd_kafka_topic_partition_list_add(parts, topic, 0)->offset =
RD_KAFKA_OFFSET_BEGINNING;
test_consumer_assign("assign", rk, parts);
rd_kafka_topic_partition_list_destroy(parts);
exp_msgid = 0;
while (exp_msgid < msgcnt) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(rk, 1000);
if (!rkm)
continue;
if (rkm->err)
TEST_FAIL("consume error while expecting msgid %d/%d: "
"%s",
exp_msgid, msgcnt,
rd_kafka_message_errstr(rkm));
handle_consumed_msg(rkm);
rd_kafka_message_destroy(rkm);
}
test_consumer_close(rk);
rd_kafka_destroy(rk);
}
static void do_consume (struct _consumer *cons, int timeout_s) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(cons->rk, 100+(timeout_s*1000));
if (!rkm)
return;
TEST_ASSERT(!rkm->err,
"%s consumer error: %s (last poll was %dms ago)",
rd_kafka_name(cons->rk),
rd_kafka_message_errstr(rkm),
(int)((test_clock() - cons->last)/1000));
rd_kafka_message_destroy(rkm);
cons->cnt++;
cons->last = test_clock();
if (timeout_s > 0) {
TEST_SAY("%s: simulate processing by sleeping for %ds\n",
rd_kafka_name(cons->rk), timeout_s);
rd_sleep(timeout_s);
}
}
static rd_kafka_message_t *msg_consume(rd_kafka_message_t *rkmessage,
void *opaque)
{
int *run = opaque;
if (rkmessage->err)
{
*run = 0;
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF)
{
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO,
"phpkafka - %% Consumer reached end of %s [%"PRId32"] "
"message queue at offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition, rkmessage->offset);
}
return NULL;
}
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - %% Consume error for topic \"%s\" [%"PRId32"] "
"offset %"PRId64": %s\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset,
rd_kafka_message_errstr(rkmessage)
);
}
return NULL;
}
return rkmessage;
}
int main_0006_symbols (int argc, char **argv) {
if (argc < 0 /* always false */) {
rd_kafka_version();
rd_kafka_version_str();
rd_kafka_get_debug_contexts();
rd_kafka_get_err_descs(NULL, NULL);
rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR);
rd_kafka_err2name(RD_KAFKA_RESP_ERR_NO_ERROR);
rd_kafka_errno2err(EINVAL);
rd_kafka_errno();
rd_kafka_last_error();
rd_kafka_conf_new();
rd_kafka_conf_destroy(NULL);
rd_kafka_conf_dup(NULL);
rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_conf_set_dr_cb(NULL, NULL);
rd_kafka_conf_set_dr_msg_cb(NULL, NULL);
rd_kafka_conf_set_error_cb(NULL, NULL);
rd_kafka_conf_set_stats_cb(NULL, NULL);
rd_kafka_conf_set_log_cb(NULL, NULL);
rd_kafka_conf_set_socket_cb(NULL, NULL);
rd_kafka_conf_set_rebalance_cb(NULL, NULL);
rd_kafka_conf_set_offset_commit_cb(NULL, NULL);
rd_kafka_conf_set_throttle_cb(NULL, NULL);
rd_kafka_conf_set_default_topic_conf(NULL, NULL);
rd_kafka_conf_get(NULL, NULL, NULL, NULL);
#ifndef _MSC_VER
rd_kafka_conf_set_open_cb(NULL, NULL);
#endif
rd_kafka_conf_set_opaque(NULL, NULL);
rd_kafka_opaque(NULL);
rd_kafka_conf_dump(NULL, NULL);
rd_kafka_topic_conf_dump(NULL, NULL);
rd_kafka_conf_dump_free(NULL, 0);
rd_kafka_conf_properties_show(NULL);
rd_kafka_topic_conf_new();
rd_kafka_topic_conf_dup(NULL);
rd_kafka_topic_conf_destroy(NULL);
rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_topic_conf_set_opaque(NULL, NULL);
rd_kafka_topic_conf_get(NULL, NULL, NULL, NULL);
rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL);
rd_kafka_topic_partition_available(NULL, 0);
rd_kafka_topic_opaque(NULL);
rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_msg_partitioner_consistent(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_msg_partitioner_consistent_random(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_new(0, NULL, NULL, 0);
rd_kafka_destroy(NULL);
rd_kafka_name(NULL);
rd_kafka_memberid(NULL);
rd_kafka_topic_new(NULL, NULL, NULL);
rd_kafka_topic_destroy(NULL);
rd_kafka_topic_name(NULL);
rd_kafka_message_destroy(NULL);
rd_kafka_message_errstr(NULL);
rd_kafka_message_timestamp(NULL, NULL);
rd_kafka_consume_start(NULL, 0, 0);
rd_kafka_consume_stop(NULL, 0);
rd_kafka_consume(NULL, 0, 0);
rd_kafka_consume_batch(NULL, 0, 0, NULL, 0);
rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL);
rd_kafka_offset_store(NULL, 0, 0);
rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL);
rd_kafka_produce_batch(NULL, 0, 0, NULL, 0);
rd_kafka_poll(NULL, 0);
rd_kafka_brokers_add(NULL, NULL);
/* DEPRECATED: rd_kafka_set_logger(NULL, NULL); */
rd_kafka_set_log_level(NULL, 0);
rd_kafka_log_print(NULL, 0, NULL, NULL);
#ifndef _MSC_VER
rd_kafka_log_syslog(NULL, 0, NULL, NULL);
#endif
rd_kafka_outq_len(NULL);
rd_kafka_dump(NULL, NULL);
rd_kafka_thread_cnt();
rd_kafka_wait_destroyed(0);
rd_kafka_metadata(NULL, 0, NULL, NULL, 0);
rd_kafka_metadata_destroy(NULL);
rd_kafka_queue_destroy(NULL);
rd_kafka_consume_start_queue(NULL, 0, 0, NULL);
rd_kafka_consume_queue(NULL, 0);
rd_kafka_consume_batch_queue(NULL, 0, NULL, 0);
rd_kafka_consume_callback_queue(NULL, 0, NULL, NULL);
rd_kafka_seek(NULL, 0, 0, 0);
rd_kafka_yield(NULL);
rd_kafka_mem_free(NULL, NULL);
rd_kafka_list_groups(NULL, NULL, NULL, 0);
rd_kafka_group_list_destroy(NULL);
/* KafkaConsumer API */
rd_kafka_subscribe(NULL, NULL);
rd_kafka_unsubscribe(NULL);
rd_kafka_subscription(NULL, NULL);
rd_kafka_consumer_poll(NULL, 0);
rd_kafka_consumer_close(NULL);
rd_kafka_assign(NULL, NULL);
rd_kafka_assignment(NULL, NULL);
rd_kafka_commit(NULL, NULL, 0);
rd_kafka_commit_message(NULL, NULL, 0);
rd_kafka_committed(NULL, NULL, 0);
rd_kafka_position(NULL, NULL);
/* TopicPartition */
rd_kafka_topic_partition_list_new(0);
rd_kafka_topic_partition_list_destroy(NULL);
rd_kafka_topic_partition_list_add(NULL, NULL, 0);
rd_kafka_topic_partition_list_add_range(NULL, NULL, 0, 0);
rd_kafka_topic_partition_list_del(NULL, NULL, 0);
rd_kafka_topic_partition_list_del_by_idx(NULL, 0);
rd_kafka_topic_partition_list_copy(NULL);
rd_kafka_topic_partition_list_set_offset(NULL, NULL, 0, 0);
rd_kafka_topic_partition_list_find(NULL, NULL, 0);
rd_kafka_query_watermark_offsets(NULL, NULL, 0, NULL, NULL, 0);
rd_kafka_get_watermark_offsets(NULL, NULL, 0, NULL, NULL);
}
return 0;
}
static void msg_consume (rd_kafka_message_t *rkmessage,
void *opaque) {
if (rkmessage->err) {
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
fprintf(stderr,
"%% Consumer reached end of %s [%"PRId32"] "
"message queue at offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition, rkmessage->offset);
if (exit_eof)
run = 0;
return;
}
fprintf(stderr, "%% Consume error for topic \"%s\" [%"PRId32"] "
"offset %"PRId64": %s\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset,
rd_kafka_message_errstr(rkmessage));
if (rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION ||
rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC)
run = 0;
return;
}
if (!quiet) {
rd_kafka_timestamp_type_t tstype;
int64_t timestamp;
fprintf(stdout, "%% Message (offset %"PRId64", %zd bytes):\n",
rkmessage->offset, rkmessage->len);
timestamp = rd_kafka_message_timestamp(rkmessage, &tstype);
if (tstype != RD_KAFKA_TIMESTAMP_NOT_AVAILABLE) {
const char *tsname = "?";
if (tstype == RD_KAFKA_TIMESTAMP_CREATE_TIME)
tsname = "create time";
else if (tstype == RD_KAFKA_TIMESTAMP_LOG_APPEND_TIME)
tsname = "log append time";
fprintf(stdout, "%% Message timestamp: %s %"PRId64
" (%ds ago)\n",
tsname, timestamp,
!timestamp ? 0 :
(int)time(NULL) - (int)(timestamp/1000));
}
}
if (rkmessage->key_len) {
if (output == OUTPUT_HEXDUMP)
hexdump(stdout, "Message Key",
rkmessage->key, rkmessage->key_len);
else
printf("Key: %.*s\n",
(int)rkmessage->key_len, (char *)rkmessage->key);
}
if (output == OUTPUT_HEXDUMP)
hexdump(stdout, "Message Payload",
rkmessage->payload, rkmessage->len);
else
printf("%.*s\n",
(int)rkmessage->len, (char *)rkmessage->payload);
}
static void do_offset_test (const char *what, int auto_commit, int auto_store,
int async) {
test_timing_t t_all;
char groupid[64];
rd_kafka_t *rk;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *tconf;
int cnt = 0;
const int extra_cnt = 5;
rd_kafka_resp_err_t err;
rd_kafka_topic_partition_list_t *parts;
rd_kafka_topic_partition_t *rktpar;
int64_t next_offset = -1;
test_conf_init(&conf, &tconf, 20);
test_conf_set(conf, "enable.auto.commit", auto_commit ? "true":"false");
test_conf_set(conf, "enable.auto.offset.store", auto_store ?"true":"false");
test_conf_set(conf, "auto.commit.interval.ms", "500");
rd_kafka_conf_set_offset_commit_cb(conf, offset_commit_cb);
test_topic_conf_set(tconf, "auto.offset.reset", "smallest");
test_str_id_generate(groupid, sizeof(groupid));
test_conf_set(conf, "group.id", groupid);
rd_kafka_conf_set_default_topic_conf(conf, tconf);
TEST_SAY(_C_MAG "[ do_offset_test: %s with group.id %s ]\n",
what, groupid);
TIMING_START(&t_all, what);
expected_offset = 0;
committed_offset = -1;
/* MO:
* - Create consumer.
* - Start consuming from beginning
* - Perform store & commits according to settings
* - Stop storing&committing when half of the messages are consumed,
* - but consume 5 more to check against.
* - Query position.
* - Destroy consumer.
* - Create new consumer with same group.id using stored offsets
* - Should consume the expected message.
*/
/* Create kafka instance */
rk = test_create_handle(RD_KAFKA_CONSUMER, rd_kafka_conf_dup(conf));
rd_kafka_poll_set_consumer(rk);
test_consumer_subscribe(rk, topic);
while (cnt - extra_cnt < msgcnt / 2) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(rk, 10*1000);
if (!rkm)
continue;
if (rkm->err == RD_KAFKA_RESP_ERR__TIMED_OUT)
TEST_FAIL("%s: Timed out waiting for message %d", what,cnt);
else if (rkm->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
rd_kafka_message_destroy(rkm);
continue;
} else if (rkm->err)
TEST_FAIL("%s: Consumer error: %s",
what, rd_kafka_message_errstr(rkm));
/* Offset of next message. */
next_offset = rkm->offset + 1;
if (cnt < msgcnt / 2) {
if (!auto_store) {
err = rd_kafka_offset_store(rkm->rkt,rkm->partition,
rkm->offset);
if (err)
TEST_FAIL("%s: offset_store failed: %s\n",
what, rd_kafka_err2str(err));
}
expected_offset = rkm->offset+1;
if (!auto_commit) {
test_timing_t t_commit;
TIMING_START(&t_commit,
async?"commit.async":"commit.sync");
err = rd_kafka_commit_message(rk, rkm, async);
TIMING_STOP(&t_commit);
if (err)
TEST_FAIL("%s: commit failed: %s\n",
what, rd_kafka_err2str(err));
}
} else if (auto_store && auto_commit)
expected_offset = rkm->offset+1;
rd_kafka_message_destroy(rkm);
cnt++;
}
TEST_SAY("%s: done consuming after %d messages, at offset %"PRId64"\n",
what, cnt, expected_offset);
if ((err = rd_kafka_assignment(rk, &parts)))
TEST_FAIL("%s: failed to get assignment(): %s\n",
what, rd_kafka_err2str(err));
/* Verify position */
if ((err = rd_kafka_position(rk, parts)))
TEST_FAIL("%s: failed to get position(): %s\n",
what, rd_kafka_err2str(err));
if (!(rktpar = rd_kafka_topic_partition_list_find(parts,
topic, partition)))
TEST_FAIL("%s: position(): topic lost\n", what);
if (rktpar->offset != next_offset)
TEST_FAIL("%s: Expected position() offset %"PRId64", got %"PRId64,
what, next_offset, rktpar->offset);
TEST_SAY("%s: Position is at %"PRId64", good!\n",
what, rktpar->offset);
/* Pause messages while waiting so we can serve callbacks
* without having more messages received. */
if ((err = rd_kafka_pause_partitions(rk, parts)))
TEST_FAIL("%s: failed to pause partitions: %s\n",
what, rd_kafka_err2str(err));
rd_kafka_topic_partition_list_destroy(parts);
/* Fire off any enqueued offset_commit_cb */
test_consumer_poll_no_msgs(what, rk, testid, 0);
TEST_SAY("%s: committed_offset %"PRId64", expected_offset %"PRId64"\n",
what, committed_offset, expected_offset);
if (!auto_commit && !async) {
/* Sync commits should be up to date at this point. */
if (committed_offset != expected_offset)
TEST_FAIL("%s: Sync commit: committed offset %"PRId64
" should be same as expected offset "
"%"PRId64,
what, committed_offset, expected_offset);
} else {
/* Wait for offset commits to catch up */
while (committed_offset < expected_offset) {
TEST_SAYL(3, "%s: Wait for committed offset %"PRId64
" to reach expected offset %"PRId64"\n",
what, committed_offset, expected_offset);
test_consumer_poll_no_msgs(what, rk, testid, 1000);
}
}
TEST_SAY("%s: phase 1 complete, %d messages consumed, "
"next expected offset is %"PRId64"\n",
what, cnt, expected_offset);
/* Issue #827: cause committed() to return prematurely by specifying
* low timeout. The bug (use after free) will only
* be catched by valgrind. */
do {
parts = rd_kafka_topic_partition_list_new(1);
rd_kafka_topic_partition_list_add(parts, topic, partition);
err = rd_kafka_committed(rk, parts, 1);
rd_kafka_topic_partition_list_destroy(parts);
TEST_SAY("Issue #827: committed() returned %s\n",
rd_kafka_err2str(err));
} while (err != RD_KAFKA_RESP_ERR__TIMED_OUT);
/* Query position */
parts = rd_kafka_topic_partition_list_new(1);
rd_kafka_topic_partition_list_add(parts, topic, partition);
err = rd_kafka_committed(rk, parts, tmout_multip(5*1000));
if (err)
TEST_FAIL("%s: committed() failed: %s", what, rd_kafka_err2str(err));
if (!(rktpar = rd_kafka_topic_partition_list_find(parts,
topic, partition)))
TEST_FAIL("%s: committed(): topic lost\n", what);
if (rktpar->offset != expected_offset)
TEST_FAIL("%s: Expected committed() offset %"PRId64", got %"PRId64,
what, expected_offset, rktpar->offset);
TEST_SAY("%s: Committed offset is at %"PRId64", good!\n",
what, rktpar->offset);
rd_kafka_topic_partition_list_destroy(parts);
test_consumer_close(rk);
rd_kafka_destroy(rk);
/* Fire up a new consumer and continue from where we left off. */
TEST_SAY("%s: phase 2: starting new consumer to resume consumption\n",what);
rk = test_create_handle(RD_KAFKA_CONSUMER, conf);
rd_kafka_poll_set_consumer(rk);
test_consumer_subscribe(rk, topic);
while (cnt < msgcnt) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(rk, 10*1000);
if (!rkm)
continue;
if (rkm->err == RD_KAFKA_RESP_ERR__TIMED_OUT)
TEST_FAIL("%s: Timed out waiting for message %d", what,cnt);
else if (rkm->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
rd_kafka_message_destroy(rkm);
continue;
} else if (rkm->err)
TEST_FAIL("%s: Consumer error: %s",
what, rd_kafka_message_errstr(rkm));
if (rkm->offset != expected_offset)
TEST_FAIL("%s: Received message offset %"PRId64
", expected %"PRId64" at msgcnt %d/%d\n",
what, rkm->offset, expected_offset,
cnt, msgcnt);
rd_kafka_message_destroy(rkm);
expected_offset++;
cnt++;
}
TEST_SAY("%s: phase 2: complete\n", what);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_all);
}
int main_0089_max_poll_interval (int argc, char **argv) {
const char *topic = test_mk_topic_name("0089_max_poll_interval", 1);
uint64_t testid;
const int msgcnt = 10;
rd_kafka_t *c[2];
rd_kafka_conf_t *conf;
int64_t ts_next[2] = { 0, 0 };
int64_t ts_exp_msg[2] = { 0, 0 };
int cmsgcnt = 0;
int i;
int bad = -1;
testid = test_id_generate();
test_create_topic(topic, 1, 1);
test_produce_msgs_easy(topic, testid, -1, msgcnt);
test_conf_init(&conf, NULL, 60);
test_conf_set(conf, "session.timeout.ms", "6000");
test_conf_set(conf, "max.poll.interval.ms", "10000" /*10s*/);
test_conf_set(conf, "auto.offset.reset", "earliest");
c[0] = test_create_consumer(topic, NULL, rd_kafka_conf_dup(conf), NULL);
c[1] = test_create_consumer(topic, NULL, conf, NULL);
test_consumer_subscribe(c[0], topic);
test_consumer_subscribe(c[1], topic);
while (1) {
for (i = 0 ; i < 2 ; i++) {
int64_t now;
rd_kafka_message_t *rkm;
/* Consumer is "processing" */
if (ts_next[i] > test_clock())
continue;
rkm = rd_kafka_consumer_poll(c[i], 100);
if (!rkm)
continue;
if (rkm->err) {
TEST_WARN("Consumer %d error: %s: "
"ignoring\n", i,
rd_kafka_message_errstr(rkm));
continue;
}
now = test_clock();
cmsgcnt++;
TEST_SAY("Consumer %d received message (#%d) "
"at offset %"PRId64"\n",
i, cmsgcnt, rkm->offset);
if (ts_exp_msg[i]) {
/* This consumer is expecting a message
* after a certain time, namely after the
* rebalance following max.poll.. being
* exceeded in the other consumer */
TEST_ASSERT(now > ts_exp_msg[i],
"Consumer %d: did not expect "
"message for at least %dms",
i,
(int)((ts_exp_msg[i] - now)/1000));
TEST_ASSERT(now < ts_exp_msg[i] + 10000*1000,
"Consumer %d: expected message "
"within 10s, not after %dms",
i,
(int)((now - ts_exp_msg[i])/1000));
TEST_SAY("Consumer %d: received message "
"at offset %"PRId64
" after rebalance\n",
i, rkm->offset);
rd_kafka_message_destroy(rkm);
goto done;
} else if (cmsgcnt == 1) {
/* Process this message for 20s */
ts_next[i] = now + (20000 * 1000);
/* Exp message on other consumer after
* max.poll.interval.ms */
ts_exp_msg[i^1] = now + (10000 * 1000);
/* This is the bad consumer */
bad = i;
TEST_SAY("Consumer %d processing message at "
"offset %"PRId64"\n",
i, rkm->offset);
rd_kafka_message_destroy(rkm);
} else {
rd_kafka_message_destroy(rkm);
TEST_FAIL("Consumer %d did not expect "
"a message", i);
}
}
}
done:
TEST_ASSERT(bad != -1, "Bad consumer not set");
/* Wait for error ERR__MAX_POLL_EXCEEDED on the bad consumer. */
while (1) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(c[bad], 1000);
TEST_ASSERT(rkm, "Expected consumer result within 1s");
TEST_ASSERT(rkm->err, "Did not expect message on bad consumer");
TEST_SAY("Consumer error: %s: %s\n",
rd_kafka_err2name(rkm->err),
rd_kafka_message_errstr(rkm));
if (rkm->err == RD_KAFKA_RESP_ERR__MAX_POLL_EXCEEDED) {
rd_kafka_message_destroy(rkm);
break;
}
rd_kafka_message_destroy(rkm);
}
for (i = 0 ; i < 2 ; i++)
rd_kafka_destroy_flags(c[i],
RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE);
return 0;
}
static
void queue_consume(rd_kafka_message_t *message, void *opaque)
{
struct consume_cb_params *params = opaque;
zval *return_value = params->return_value;
//all partitions EOF
if (params->eop < 1)
return;
//nothing more to read...
if (params->read_count == 0)
return;
if (message->err)
{
params->error_count += 1;
//if auto-commit is disabled:
if (params->auto_commit == 0)
//store offset
rd_kafka_offset_store(
message->rkt,
message->partition,
message->offset == 0 ? 0 : message->offset -1
);
if (message->err == RD_KAFKA_RESP_ERR__PARTITION_EOF)
{
if (params->partition_ends[message->partition] == 0)
{
params->eop -= 1;
params->partition_ends[message->partition] = 1;
}
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO,
"phpkafka - %% Consumer reached end of %s [%"PRId32"] "
"message queue at offset %"PRId64"\n",
rd_kafka_topic_name(message->rkt),
message->partition, message->offset);
}
return;
}
//add_next_index_string(return_value, rd_kafka_message_errstr(message), 1);
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - %% Consume error for topic \"%s\" [%"PRId32"] "
"offset %"PRId64": %s\n",
rd_kafka_topic_name(message->rkt),
message->partition,
message->offset,
rd_kafka_message_errstr(message)
);
}
return;
}
//only count successful reads!
//-1 means read all from offset until end
if (params->read_count != -1)
params->read_count -= 1;
//add message to return value (perhaps add as array -> offset + msg?
if (message->len > 0) {
//ensure there is a payload
char payload[(int) message->len];
sprintf(payload, "%.*s", (int) message->len, (char *) message->payload);
//add_index_string(return_value, (int) message->offset, payload, 1);
add_next_index_string(return_value, payload, 1);
} else {
add_next_index_string(return_value, "", 1);
}
//store offset if autocommit is disabled
if (params->auto_commit == 0)
rd_kafka_offset_store(
message->rkt,
message->partition,
message->offset
);
}
