int main (int argc, char **argv) {
if (argc < 0 /* always false */) {
rd_kafka_version();
rd_kafka_version_str();
rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR);
rd_kafka_errno2err(EINVAL);
rd_kafka_conf_new();
rd_kafka_conf_destroy(NULL);
rd_kafka_conf_dup(NULL);
rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_conf_set_dr_cb(NULL, NULL);
rd_kafka_conf_set_error_cb(NULL, NULL);
rd_kafka_conf_set_stats_cb(NULL, NULL);
rd_kafka_conf_set_opaque(NULL, NULL);
rd_kafka_conf_dump(NULL, NULL);
rd_kafka_topic_conf_dump(NULL, NULL);
rd_kafka_conf_dump_free(NULL, 0);
rd_kafka_conf_properties_show(NULL);
rd_kafka_topic_conf_new();
rd_kafka_topic_conf_dup(NULL);
rd_kafka_topic_conf_destroy(NULL);
rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_topic_conf_set_opaque(NULL, NULL);
rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL);
rd_kafka_topic_partition_available(NULL, 0);
rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_new(0, NULL, NULL, 0);
rd_kafka_destroy(NULL);
rd_kafka_name(NULL);
rd_kafka_topic_new(NULL, NULL, NULL);
rd_kafka_topic_destroy(NULL);
rd_kafka_topic_name(NULL);
rd_kafka_message_destroy(NULL);
rd_kafka_message_errstr(NULL);
rd_kafka_consume_start(NULL, 0, 0);
rd_kafka_consume_stop(NULL, 0);
rd_kafka_consume(NULL, 0, 0);
rd_kafka_consume_batch(NULL, 0, 0, NULL, 0);
rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL);
rd_kafka_offset_store(NULL, 0, 0);
rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL);
rd_kafka_poll(NULL, 0);
rd_kafka_brokers_add(NULL, NULL);
rd_kafka_set_logger(NULL, NULL);
rd_kafka_set_log_level(NULL, 0);
rd_kafka_log_print(NULL, 0, NULL, NULL);
rd_kafka_log_syslog(NULL, 0, NULL, NULL);
rd_kafka_outq_len(NULL);
rd_kafka_dump(NULL, NULL);
rd_kafka_thread_cnt();
rd_kafka_wait_destroyed(0);
}
return 0;
}
static int kafka_handle(struct kafka_topic_context *ctx) /* {{{ */
{
char errbuf[1024];
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
if (ctx->kafka != NULL && ctx->topic != NULL)
return(0);
if (ctx->kafka == NULL) {
if ((conf = rd_kafka_conf_dup(ctx->kafka_conf)) == NULL) {
ERROR("write_kafka plugin: cannot duplicate kafka config");
return(1);
}
if ((ctx->kafka = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errbuf, sizeof(errbuf))) == NULL) {
ERROR("write_kafka plugin: cannot create kafka handle.");
return 1;
}
rd_kafka_conf_destroy(ctx->kafka_conf);
ctx->kafka_conf = NULL;
INFO ("write_kafka plugin: created KAFKA handle : %s", rd_kafka_name(ctx->kafka));
#if defined(HAVE_LIBRDKAFKA_LOGGER) && !defined(HAVE_LIBRDKAFKA_LOG_CB)
rd_kafka_set_logger(ctx->kafka, kafka_log);
#endif
}
if (ctx->topic == NULL ) {
if ((topic_conf = rd_kafka_topic_conf_dup(ctx->conf)) == NULL) {
ERROR("write_kafka plugin: cannot duplicate kafka topic config");
return 1;
}
if ((ctx->topic = rd_kafka_topic_new(ctx->kafka, ctx->topic_name,
topic_conf)) == NULL) {
ERROR("write_kafka plugin: cannot create topic : %s\n",
rd_kafka_err2str(rd_kafka_errno2err(errno)));
return errno;
}
rd_kafka_topic_conf_destroy(ctx->conf);
ctx->conf = NULL;
INFO ("write_kafka plugin: handle created for topic : %s", rd_kafka_topic_name(ctx->topic));
}
return(0);
} /* }}} int kafka_handle */
static void kafka_topic_context_free(void *p) /* {{{ */
{
struct kafka_topic_context *ctx = p;
if (ctx == NULL)
return;
if (ctx->topic_name != NULL)
sfree(ctx->topic_name);
if (ctx->topic != NULL)
rd_kafka_topic_destroy(ctx->topic);
if (ctx->conf != NULL)
rd_kafka_topic_conf_destroy(ctx->conf);
sfree(ctx);
} /* }}} void kafka_topic_context_free */
/** Creates a new topic handler using global configuration
@param topic_name Topic name
@param partitioner Partitioner function
@return New topic handler */
rd_kafka_topic_t *new_rkt_global_config(const char *topic_name,
rb_rd_kafka_partitioner_t partitioner,char *err,size_t errsize) {
rd_kafka_topic_conf_t *template_config = global_config.kafka_topic_conf;
rd_kafka_topic_conf_t *my_rkt_conf
= rd_kafka_topic_conf_dup(template_config);
if(NULL == my_rkt_conf) {
rdlog(LOG_ERR,"Couldn't topic_conf_dup in topic %s",topic_name);
return NULL;
}
rd_kafka_topic_conf_set_partitioner_cb(my_rkt_conf, partitioner);
rd_kafka_topic_t *ret = rd_kafka_topic_new(global_config.rk, topic_name,
my_rkt_conf);
if (NULL == ret) {
strerror_r(errno, err, errsize);
rd_kafka_topic_conf_destroy(my_rkt_conf);
}
return ret;
}
static PyObject *
RdkHandle_stop(RdkHandle *self)
{
/* We'll only ever get a locking error if we programmed ourselves into a
* deadlock. We'd have to admit defeat, abort, and leak this RdkHandle */
if (RdkHandle_excl_lock(self)) return NULL;
Py_BEGIN_ALLOW_THREADS /* avoid callbacks deadlocking */
if (self->rdk_queue_handle) {
rd_kafka_queue_destroy(self->rdk_queue_handle);
self->rdk_queue_handle = NULL;
}
if (self->rdk_topic_handle) {
rd_kafka_topic_destroy(self->rdk_topic_handle);
self->rdk_topic_handle = NULL;
}
if (self->rdk_handle) {
PyObject *opaque = (PyObject *)rd_kafka_opaque(self->rdk_handle);
Py_XDECREF(opaque);
rd_kafka_destroy(self->rdk_handle);
self->rdk_handle = NULL;
}
if (self->rdk_conf) {
rd_kafka_conf_destroy(self->rdk_conf);
self->rdk_conf = NULL;
}
if (self->rdk_topic_conf) {
rd_kafka_topic_conf_destroy(self->rdk_topic_conf);
self->rdk_topic_conf = NULL;
}
Py_END_ALLOW_THREADS
Py_CLEAR(self->partition_ids);
if (RdkHandle_unlock(self)) return NULL;
Py_INCREF(Py_None);
return Py_None;
}
int main_0004_conf (int argc, char **argv) {
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *ignore_conf, *conf, *conf2;
rd_kafka_topic_conf_t *ignore_topic_conf, *tconf, *tconf2;
char errstr[512];
const char **arr_orig, **arr_dup;
size_t cnt_orig, cnt_dup;
int i;
const char *topic;
static const char *gconfs[] = {
"message.max.bytes", "12345", /* int property */
"client.id", "my id", /* string property */
"debug", "topic,metadata", /* S2F property */
"topic.blacklist", "__.*", /* #778 */
"auto.offset.reset", "earliest", /* Global->Topic fallthru */
#if WITH_ZLIB
"compression.codec", "gzip", /* S2I property */
#endif
NULL
};
static const char *tconfs[] = {
"request.required.acks", "-1", /* int */
"auto.commit.enable", "false", /* bool */
"auto.offset.reset", "error", /* S2I */
"offset.store.path", "my/path", /* string */
NULL
};
test_conf_init(&ignore_conf, &ignore_topic_conf, 10);
rd_kafka_conf_destroy(ignore_conf);
rd_kafka_topic_conf_destroy(ignore_topic_conf);
topic = test_mk_topic_name("0004", 0);
/* Set up a global config object */
conf = rd_kafka_conf_new();
rd_kafka_conf_set_dr_cb(conf, dr_cb);
rd_kafka_conf_set_error_cb(conf, error_cb);
for (i = 0 ; gconfs[i] ; i += 2) {
if (rd_kafka_conf_set(conf, gconfs[i], gconfs[i+1],
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK)
TEST_FAIL("%s\n", errstr);
}
/* Set up a topic config object */
tconf = rd_kafka_topic_conf_new();
rd_kafka_topic_conf_set_partitioner_cb(tconf, partitioner);
rd_kafka_topic_conf_set_opaque(tconf, (void *)0xbeef);
for (i = 0 ; tconfs[i] ; i += 2) {
if (rd_kafka_topic_conf_set(tconf, tconfs[i], tconfs[i+1],
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK)
TEST_FAIL("%s\n", errstr);
}
/* Verify global config */
arr_orig = rd_kafka_conf_dump(conf, &cnt_orig);
conf_verify(__LINE__, arr_orig, cnt_orig, gconfs);
/* Verify copied global config */
conf2 = rd_kafka_conf_dup(conf);
arr_dup = rd_kafka_conf_dump(conf2, &cnt_dup);
conf_verify(__LINE__, arr_dup, cnt_dup, gconfs);
conf_cmp("global", arr_orig, cnt_orig, arr_dup, cnt_dup);
rd_kafka_conf_dump_free(arr_orig, cnt_orig);
rd_kafka_conf_dump_free(arr_dup, cnt_dup);
/* Verify topic config */
arr_orig = rd_kafka_topic_conf_dump(tconf, &cnt_orig);
conf_verify(__LINE__, arr_orig, cnt_orig, tconfs);
/* Verify copied topic config */
tconf2 = rd_kafka_topic_conf_dup(tconf);
arr_dup = rd_kafka_topic_conf_dump(tconf2, &cnt_dup);
conf_verify(__LINE__, arr_dup, cnt_dup, tconfs);
conf_cmp("topic", arr_orig, cnt_orig, arr_dup, cnt_dup);
rd_kafka_conf_dump_free(arr_orig, cnt_orig);
rd_kafka_conf_dump_free(arr_dup, cnt_dup);
/*
* Create kafka instances using original and copied confs
*/
/* original */
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
rkt = rd_kafka_topic_new(rk, topic, tconf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
rd_strerror(errno));
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
/* copied */
rk = test_create_handle(RD_KAFKA_PRODUCER, conf2);
rkt = rd_kafka_topic_new(rk, topic, tconf2);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
rd_strerror(errno));
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
/* Incremental S2F property.
* NOTE: The order of fields returned in get() is hardcoded here. */
{
static const char *s2fs[] = {
"generic,broker,queue,cgrp",
"generic,broker,queue,cgrp",
"-broker,+queue,topic",
"generic,topic,queue,cgrp",
"-all,security,-fetch,+metadata",
"metadata,security",
NULL
};
TEST_SAY("Incremental S2F tests\n");
conf = rd_kafka_conf_new();
for (i = 0 ; s2fs[i] ; i += 2) {
const char *val;
TEST_SAY(" Set: %s\n", s2fs[i]);
test_conf_set(conf, "debug", s2fs[i]);
val = test_conf_get(conf, "debug");
TEST_SAY(" Now: %s\n", val);
if (strcmp(val, s2fs[i+1]))
TEST_FAIL_LATER("\n"
"Expected: %s\n"
" Got: %s",
s2fs[i+1], val);
}
rd_kafka_conf_destroy(conf);
}
/* Canonical int values, aliases, s2i-verified strings */
{
static const struct {
const char *prop;
const char *val;
const char *exp;
int is_global;
} props[] = {
{ "request.required.acks", "0", "0" },
{ "request.required.acks", "-1", "-1" },
{ "request.required.acks", "1", "1" },
{ "acks", "3", "3" }, /* alias test */
{ "request.required.acks", "393", "393" },
{ "request.required.acks", "bad", NULL },
{ "request.required.acks", "all", "-1" },
{ "request.required.acks", "all", "-1", 1/*fallthru*/ },
{ "acks", "0", "0" }, /* alias test */
#if WITH_SASL
{ "sasl.mechanisms", "GSSAPI", "GSSAPI", 1 },
{ "sasl.mechanisms", "PLAIN", "PLAIN", 1 },
{ "sasl.mechanisms", "GSSAPI,PLAIN", NULL, 1 },
{ "sasl.mechanisms", "", NULL, 1 },
#endif
{ NULL }
};
TEST_SAY("Canonical tests\n");
tconf = rd_kafka_topic_conf_new();
conf = rd_kafka_conf_new();
for (i = 0 ; props[i].prop ; i++) {
char dest[64];
size_t destsz;
rd_kafka_conf_res_t res;
TEST_SAY(" Set: %s=%s expect %s (%s)\n",
props[i].prop, props[i].val, props[i].exp,
props[i].is_global ? "global":"topic");
/* Set value */
if (props[i].is_global)
res = rd_kafka_conf_set(conf,
props[i].prop,
props[i].val,
errstr, sizeof(errstr));
else
res = rd_kafka_topic_conf_set(tconf,
props[i].prop,
props[i].val,
errstr,
sizeof(errstr));
if ((res == RD_KAFKA_CONF_OK ? 1:0) !=
(props[i].exp ? 1:0))
TEST_FAIL("Expected %s, got %s",
props[i].exp ? "success" : "failure",
(res == RD_KAFKA_CONF_OK ? "OK" :
(res == RD_KAFKA_CONF_INVALID ? "INVALID" :
"UNKNOWN")));
if (!props[i].exp)
continue;
/* Get value and compare to expected result */
destsz = sizeof(dest);
if (props[i].is_global)
res = rd_kafka_conf_get(conf,
props[i].prop,
dest, &destsz);
else
res = rd_kafka_topic_conf_get(tconf,
props[i].prop,
dest, &destsz);
TEST_ASSERT(res == RD_KAFKA_CONF_OK,
".._conf_get(%s) returned %d",
props[i].prop, res);
TEST_ASSERT(!strcmp(props[i].exp, dest),
"Expected \"%s\", got \"%s\"",
props[i].exp, dest);
}
rd_kafka_topic_conf_destroy(tconf);
rd_kafka_conf_destroy(conf);
}
return 0;
}
int main (int argc, char **argv) {
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *ignore_conf, *conf, *conf2;
rd_kafka_topic_conf_t *ignore_topic_conf, *tconf, *tconf2;
char errstr[512];
const char **arr_orig, **arr_dup;
size_t cnt_orig, cnt_dup;
int i;
const char *topic;
static const char *gconfs[] = {
"message.max.bytes", "12345", /* int property */
"client.id", "my id", /* string property */
"debug", "topic,metadata", /* S2F property */
"compression.codec", "gzip", /* S2I property */
NULL
};
static const char *tconfs[] = {
"request.required.acks", "-1", /* int */
"auto.commit.enable", "false", /* bool */
"auto.offset.reset", "error", /* S2I */
"offset.store.path", "my/path", /* string */
NULL
};
test_conf_init(&ignore_conf, &ignore_topic_conf, 10);
rd_kafka_conf_destroy(ignore_conf);
rd_kafka_topic_conf_destroy(ignore_topic_conf);
topic = test_mk_topic_name("generic", 0);
/* Set up a global config object */
conf = rd_kafka_conf_new();
rd_kafka_conf_set_dr_cb(conf, dr_cb);
rd_kafka_conf_set_error_cb(conf, error_cb);
for (i = 0 ; gconfs[i] ; i += 2) {
if (rd_kafka_conf_set(conf, gconfs[i], gconfs[i+1],
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK)
TEST_FAIL("%s\n", errstr);
}
/* Set up a topic config object */
tconf = rd_kafka_topic_conf_new();
rd_kafka_topic_conf_set_partitioner_cb(tconf, partitioner);
rd_kafka_topic_conf_set_opaque(tconf, (void *)0xbeef);
for (i = 0 ; tconfs[i] ; i += 2) {
if (rd_kafka_topic_conf_set(tconf, tconfs[i], tconfs[i+1],
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK)
TEST_FAIL("%s\n", errstr);
}
/* Verify global config */
arr_orig = rd_kafka_conf_dump(conf, &cnt_orig);
conf_verify(__LINE__, arr_orig, cnt_orig, gconfs);
/* Verify copied global config */
conf2 = rd_kafka_conf_dup(conf);
arr_dup = rd_kafka_conf_dump(conf2, &cnt_dup);
conf_verify(__LINE__, arr_dup, cnt_dup, gconfs);
conf_cmp("global", arr_orig, cnt_orig, arr_dup, cnt_dup);
rd_kafka_conf_dump_free(arr_orig, cnt_orig);
rd_kafka_conf_dump_free(arr_dup, cnt_dup);
/* Verify topic config */
arr_orig = rd_kafka_topic_conf_dump(tconf, &cnt_orig);
conf_verify(__LINE__, arr_orig, cnt_orig, tconfs);
/* Verify copied topic config */
tconf2 = rd_kafka_topic_conf_dup(tconf);
arr_dup = rd_kafka_topic_conf_dump(tconf2, &cnt_dup);
conf_verify(__LINE__, arr_dup, cnt_dup, tconfs);
conf_cmp("topic", arr_orig, cnt_orig, arr_dup, cnt_dup);
rd_kafka_conf_dump_free(arr_orig, cnt_orig);
rd_kafka_conf_dump_free(arr_dup, cnt_dup);
/*
* Create kafka instances using original and copied confs
*/
/* original */
rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr));
if (!rk)
TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr);
rkt = rd_kafka_topic_new(rk, topic, tconf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
strerror(errno));
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
/* copied */
rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf2,
errstr, sizeof(errstr));
if (!rk)
TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr);
rkt = rd_kafka_topic_new(rk, topic, tconf2);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
strerror(errno));
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
/* Wait for everything to be cleaned up since broker destroys are
* handled in its own thread. */
test_wait_exit(2);
/* If we havent failed at this point then
* there were no threads leaked */
return 0;
}
int main (int argc, char **argv) {
rd_kafka_topic_t *rkt;
char *brokers = "localhost:9092";
char mode = 'C';
char *topic = NULL;
int partition = RD_KAFKA_PARTITION_UA;
int opt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char errstr[512];
int64_t start_offset = 0;
int report_offsets = 0;
int do_conf_dump = 0;
char tmp[16];
int64_t seek_offset = 0;
int64_t tmp_offset = 0;
int get_wmarks = 0;
/* Kafka configuration */
conf = rd_kafka_conf_new();
/* Set logger */
rd_kafka_conf_set_log_cb(conf, logger);
/* Quick termination */
snprintf(tmp, sizeof(tmp), "%i", SIGIO);
rd_kafka_conf_set(conf, "internal.termination.signal", tmp, NULL, 0);
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
while ((opt = getopt(argc, argv, "PCLt:p:b:z:qd:o:eX:As:")) != -1) {
switch (opt) {
case 'P':
case 'C':
case 'L':
mode = opt;
break;
case 't':
topic = optarg;
break;
case 'p':
partition = atoi(optarg);
break;
case 'b':
brokers = optarg;
break;
case 'z':
if (rd_kafka_conf_set(conf, "compression.codec",
optarg,
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
break;
case 'o':
case 's':
if (!strcmp(optarg, "end"))
tmp_offset = RD_KAFKA_OFFSET_END;
else if (!strcmp(optarg, "beginning"))
tmp_offset = RD_KAFKA_OFFSET_BEGINNING;
else if (!strcmp(optarg, "stored"))
tmp_offset = RD_KAFKA_OFFSET_STORED;
else if (!strcmp(optarg, "report"))
report_offsets = 1;
else if (!strcmp(optarg, "wmark"))
get_wmarks = 1;
else {
tmp_offset = strtoll(optarg, NULL, 10);
if (tmp_offset < 0)
tmp_offset = RD_KAFKA_OFFSET_TAIL(-tmp_offset);
}
if (opt == 'o')
start_offset = tmp_offset;
else if (opt == 's')
seek_offset = tmp_offset;
break;
case 'e':
exit_eof = 1;
break;
case 'd':
if (rd_kafka_conf_set(conf, "debug", optarg,
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr,
"%% Debug configuration failed: "
"%s: %s\n",
errstr, optarg);
exit(1);
}
break;
case 'q':
quiet = 1;
break;
case 'A':
output = OUTPUT_RAW;
break;
case 'X':
{
char *name, *val;
rd_kafka_conf_res_t res;
if (!strcmp(optarg, "list") ||
!strcmp(optarg, "help")) {
rd_kafka_conf_properties_show(stdout);
exit(0);
}
if (!strcmp(optarg, "dump")) {
do_conf_dump = 1;
continue;
}
name = optarg;
if (!(val = strchr(name, '='))) {
char dest[512];
size_t dest_size = sizeof(dest);
/* Return current value for property. */
res = RD_KAFKA_CONF_UNKNOWN;
if (!strncmp(name, "topic.", strlen("topic.")))
res = rd_kafka_topic_conf_get(
topic_conf,
name+strlen("topic."),
dest, &dest_size);
if (res == RD_KAFKA_CONF_UNKNOWN)
res = rd_kafka_conf_get(
conf, name, dest, &dest_size);
if (res == RD_KAFKA_CONF_OK) {
printf("%s = %s\n", name, dest);
exit(0);
} else {
fprintf(stderr,
"%% %s property\n",
res == RD_KAFKA_CONF_UNKNOWN ?
"Unknown" : "Invalid");
exit(1);
}
}
*val = '\0';
val++;
res = RD_KAFKA_CONF_UNKNOWN;
/* Try "topic." prefixed properties on topic
* conf first, and then fall through to global if
* it didnt match a topic configuration property. */
if (!strncmp(name, "topic.", strlen("topic.")))
res = rd_kafka_topic_conf_set(topic_conf,
name+
strlen("topic."),
val,
errstr,
sizeof(errstr));
if (res == RD_KAFKA_CONF_UNKNOWN)
res = rd_kafka_conf_set(conf, name, val,
errstr, sizeof(errstr));
if (res != RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
}
break;
default:
goto usage;
}
}
if (do_conf_dump) {
const char **arr;
size_t cnt;
int pass;
for (pass = 0 ; pass < 2 ; pass++) {
int i;
if (pass == 0) {
arr = rd_kafka_conf_dump(conf, &cnt);
printf("# Global config\n");
} else {
printf("# Topic config\n");
arr = rd_kafka_topic_conf_dump(topic_conf,
&cnt);
}
for (i = 0 ; i < (int)cnt ; i += 2)
printf("%s = %s\n",
arr[i], arr[i+1]);
printf("\n");
rd_kafka_conf_dump_free(arr, cnt);
}
exit(0);
}
if (optind != argc || (mode != 'L' && !topic)) {
usage:
fprintf(stderr,
"Usage: %s -C|-P|-L -t <topic> "
"[-p <partition>] [-b <host1:port1,host2:port2,..>]\n"
"\n"
"librdkafka version %s (0x%08x)\n"
"\n"
" Options:\n"
" -C | -P Consumer or Producer mode\n"
" -L Metadata list mode\n"
" -t <topic> Topic to fetch / produce\n"
" -p <num> Partition (random partitioner)\n"
" -b <brokers> Broker address (localhost:9092)\n"
" -z <codec> Enable compression:\n"
" none|gzip|snappy\n"
" -o <offset> Start offset (consumer):\n"
" beginning, end, NNNNN or -NNNNN\n"
" wmark returns the current hi&lo "
"watermarks.\n"
" -o report Report message offsets (producer)\n"
" -e Exit consumer when last message\n"
" in partition has been received.\n"
" -d [facs..] Enable debugging contexts:\n"
" %s\n"
" -q Be quiet\n"
" -A Raw payload output (consumer)\n"
" -X <prop=name> Set arbitrary librdkafka "
"configuration property\n"
" Properties prefixed with \"topic.\" "
"will be set on topic object.\n"
" -X list Show full list of supported "
"properties.\n"
" -X <prop> Get single property value\n"
"\n"
" In Consumer mode:\n"
" writes fetched messages to stdout\n"
" In Producer mode:\n"
" reads messages from stdin and sends to broker\n"
" In List mode:\n"
" queries broker for metadata information, "
"topic is optional.\n"
"\n"
"\n"
"\n",
argv[0],
rd_kafka_version_str(), rd_kafka_version(),
RD_KAFKA_DEBUG_CONTEXTS);
exit(1);
}
if ((mode == 'C' && !isatty(STDIN_FILENO)) ||
(mode != 'C' && !isatty(STDOUT_FILENO)))
quiet = 1;
signal(SIGINT, stop);
signal(SIGUSR1, sig_usr1);
if (mode == 'P') {
/*
* Producer
*/
char buf[2048];
int sendcnt = 0;
/* Set up a message delivery report callback.
* It will be called once for each message, either on successful
* delivery to broker, or upon failure to deliver to broker. */
/* If offset reporting (-o report) is enabled, use the
* richer dr_msg_cb instead. */
if (report_offsets) {
rd_kafka_topic_conf_set(topic_conf,
"produce.offset.report",
"true", errstr, sizeof(errstr));
rd_kafka_conf_set_dr_msg_cb(conf, msg_delivered2);
} else
rd_kafka_conf_set_dr_cb(conf, msg_delivered);
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new producer: %s\n",
errstr);
exit(1);
}
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Create topic */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
topic_conf = NULL; /* Now owned by topic */
if (!quiet)
fprintf(stderr,
"%% Type stuff and hit enter to send\n");
while (run && fgets(buf, sizeof(buf), stdin)) {
size_t len = strlen(buf);
if (buf[len-1] == '\n')
buf[--len] = '\0';
/* Send/Produce message. */
if (rd_kafka_produce(rkt, partition,
RD_KAFKA_MSG_F_COPY,
/* Payload and length */
buf, len,
/* Optional key and its length */
NULL, 0,
/* Message opaque, provided in
* delivery report callback as
* msg_opaque. */
NULL) == -1) {
fprintf(stderr,
"%% Failed to produce to topic %s "
"partition %i: %s\n",
rd_kafka_topic_name(rkt), partition,
rd_kafka_err2str(rd_kafka_last_error()));
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
continue;
}
if (!quiet)
fprintf(stderr, "%% Sent %zd bytes to topic "
"%s partition %i\n",
len, rd_kafka_topic_name(rkt), partition);
sendcnt++;
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
}
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
/* Wait for messages to be delivered */
while (run && rd_kafka_outq_len(rk) > 0)
rd_kafka_poll(rk, 100);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy the handle */
rd_kafka_destroy(rk);
} else if (mode == 'C') {
/*
* Consumer
*/
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new consumer: %s\n",
errstr);
exit(1);
}
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
if (get_wmarks) {
int64_t lo, hi;
rd_kafka_resp_err_t err;
/* Only query for hi&lo partition watermarks */
if ((err = rd_kafka_query_watermark_offsets(
rk, topic, partition, &lo, &hi, 5000))) {
fprintf(stderr, "%% query_watermark_offsets() "
"failed: %s\n",
rd_kafka_err2str(err));
exit(1);
}
printf("%s [%d]: low - high offsets: "
"%"PRId64" - %"PRId64"\n",
topic, partition, lo, hi);
rd_kafka_destroy(rk);
exit(0);
}
/* Create topic */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
topic_conf = NULL; /* Now owned by topic */
/* Start consuming */
if (rd_kafka_consume_start(rkt, partition, start_offset) == -1){
rd_kafka_resp_err_t err = rd_kafka_last_error();
fprintf(stderr, "%% Failed to start consuming: %s\n",
rd_kafka_err2str(err));
if (err == RD_KAFKA_RESP_ERR__INVALID_ARG)
fprintf(stderr,
"%% Broker based offset storage "
"requires a group.id, "
"add: -X group.id=yourGroup\n");
exit(1);
}
while (run) {
rd_kafka_message_t *rkmessage;
rd_kafka_resp_err_t err;
/* Poll for errors, etc. */
rd_kafka_poll(rk, 0);
/* Consume single message.
* See rdkafka_performance.c for high speed
* consuming of messages. */
rkmessage = rd_kafka_consume(rkt, partition, 1000);
if (!rkmessage) /* timeout */
continue;
msg_consume(rkmessage, NULL);
/* Return message to rdkafka */
rd_kafka_message_destroy(rkmessage);
if (seek_offset) {
err = rd_kafka_seek(rkt, partition, seek_offset,
2000);
if (err)
printf("Seek failed: %s\n",
rd_kafka_err2str(err));
else
printf("Seeked to %"PRId64"\n",
seek_offset);
seek_offset = 0;
}
}
/* Stop consuming */
rd_kafka_consume_stop(rkt, partition);
while (rd_kafka_outq_len(rk) > 0)
rd_kafka_poll(rk, 10);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy handle */
rd_kafka_destroy(rk);
} else if (mode == 'L') {
rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new producer: %s\n",
errstr);
exit(1);
}
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Create topic */
if (topic) {
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
topic_conf = NULL; /* Now owned by topic */
} else
rkt = NULL;
while (run) {
const struct rd_kafka_metadata *metadata;
/* Fetch metadata */
err = rd_kafka_metadata(rk, rkt ? 0 : 1, rkt,
&metadata, 5000);
if (err != RD_KAFKA_RESP_ERR_NO_ERROR) {
fprintf(stderr,
"%% Failed to acquire metadata: %s\n",
rd_kafka_err2str(err));
run = 0;
break;
}
metadata_print(topic, metadata);
rd_kafka_metadata_destroy(metadata);
run = 0;
}
/* Destroy topic */
if (rkt)
rd_kafka_topic_destroy(rkt);
/* Destroy the handle */
rd_kafka_destroy(rk);
if (topic_conf)
rd_kafka_topic_conf_destroy(topic_conf);
/* Exit right away, dont wait for background cleanup, we haven't
* done anything important anyway. */
exit(err ? 2 : 0);
}
if (topic_conf)
rd_kafka_topic_conf_destroy(topic_conf);
/* Let background threads clean up and terminate cleanly. */
run = 5;
while (run-- > 0 && rd_kafka_wait_destroyed(1000) == -1)
printf("Waiting for librdkafka to decommission\n");
if (run <= 0)
rd_kafka_dump(stdout, rk);
return 0;
}
static void kafka_config_topic(rd_kafka_conf_t *conf, oconfig_item_t *ci) /* {{{ */
{
int status;
int i;
struct kafka_topic_context *tctx;
char *key;
char *val;
char callback_name[DATA_MAX_NAME_LEN];
char errbuf[1024];
user_data_t ud;
oconfig_item_t *child;
rd_kafka_conf_res_t ret;
if ((tctx = calloc(1, sizeof (*tctx))) == NULL) {
ERROR ("write_kafka plugin: calloc failed.");
return;
}
tctx->escape_char = '.';
tctx->store_rates = 1;
rd_kafka_conf_set_log_cb(conf, kafka_log);
if ((tctx->kafka = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errbuf, sizeof(errbuf))) == NULL) {
sfree(tctx);
ERROR("write_kafka plugin: cannot create kafka handle.");
return;
}
conf = NULL;
if ((tctx->conf = rd_kafka_topic_conf_new()) == NULL) {
rd_kafka_destroy(tctx->kafka);
sfree(tctx);
ERROR ("write_kafka plugin: cannot create topic configuration.");
return;
}
if (ci->values_num != 1) {
WARNING("kafka topic name needed.");
goto errout;
}
if (ci->values[0].type != OCONFIG_TYPE_STRING) {
WARNING("kafka topic needs a string argument.");
goto errout;
}
if ((tctx->topic_name = strdup(ci->values[0].value.string)) == NULL) {
ERROR("write_kafka plugin: cannot copy topic name.");
goto errout;
}
for (i = 0; i < ci->children_num; i++) {
/*
* The code here could be simplified but makes room
* for easy adding of new options later on.
*/
child = &ci->children[i];
status = 0;
if (strcasecmp ("Property", child->key) == 0) {
if (child->values_num != 2) {
WARNING("kafka properties need both a key and a value.");
goto errout;
}
if (child->values[0].type != OCONFIG_TYPE_STRING ||
child->values[1].type != OCONFIG_TYPE_STRING) {
WARNING("kafka properties needs string arguments.");
goto errout;
}
key = child->values[0].value.string;
val = child->values[0].value.string;
ret = rd_kafka_topic_conf_set(tctx->conf,key, val,
errbuf, sizeof(errbuf));
if (ret != RD_KAFKA_CONF_OK) {
WARNING("cannot set kafka topic property %s to %s: %s.",
key, val, errbuf);
goto errout;
}
} else if (strcasecmp ("Key", child->key) == 0) {
char *tmp_buf = NULL;
status = cf_util_get_string(child, &tmp_buf);
if (status != 0) {
WARNING("write_kafka plugin: invalid key supplied");
break;
}
if (strcasecmp(tmp_buf, "Random") != 0) {
tctx->has_key = 1;
tctx->key = crc32_buffer((u_char *)tmp_buf, strlen(tmp_buf));
}
sfree(tmp_buf);
} else if (strcasecmp ("Format", child->key) == 0) {
status = cf_util_get_string(child, &key);
if (status != 0)
goto errout;
assert(key != NULL);
if (strcasecmp(key, "Command") == 0) {
tctx->format = KAFKA_FORMAT_COMMAND;
} else if (strcasecmp(key, "Graphite") == 0) {
tctx->format = KAFKA_FORMAT_GRAPHITE;
} else if (strcasecmp(key, "Json") == 0) {
tctx->format = KAFKA_FORMAT_JSON;
} else {
WARNING ("write_kafka plugin: Invalid format string: %s",
key);
}
sfree(key);
} else if (strcasecmp ("StoreRates", child->key) == 0) {
status = cf_util_get_boolean (child, &tctx->store_rates);
(void) cf_util_get_flag (child, &tctx->graphite_flags,
GRAPHITE_STORE_RATES);
} else if (strcasecmp ("GraphiteSeparateInstances", child->key) == 0) {
status = cf_util_get_flag (child, &tctx->graphite_flags,
GRAPHITE_SEPARATE_INSTANCES);
} else if (strcasecmp ("GraphiteAlwaysAppendDS", child->key) == 0) {
status = cf_util_get_flag (child, &tctx->graphite_flags,
GRAPHITE_ALWAYS_APPEND_DS);
} else if (strcasecmp ("GraphitePrefix", child->key) == 0) {
status = cf_util_get_string (child, &tctx->prefix);
} else if (strcasecmp ("GraphitePostfix", child->key) == 0) {
status = cf_util_get_string (child, &tctx->postfix);
} else if (strcasecmp ("GraphiteEscapeChar", child->key) == 0) {
char *tmp_buff = NULL;
status = cf_util_get_string (child, &tmp_buff);
if (strlen (tmp_buff) > 1)
WARNING ("write_kafka plugin: The option \"GraphiteEscapeChar\" handles "
"only one character. Others will be ignored.");
tctx->escape_char = tmp_buff[0];
sfree (tmp_buff);
} else {
WARNING ("write_kafka plugin: Invalid directive: %s.", child->key);
}
if (status != 0)
break;
}
rd_kafka_topic_conf_set_partitioner_cb(tctx->conf, kafka_partition);
rd_kafka_topic_conf_set_opaque(tctx->conf, tctx);
if ((tctx->topic = rd_kafka_topic_new(tctx->kafka, tctx->topic_name,
tctx->conf)) == NULL) {
ERROR("write_kafka plugin: cannot create topic.");
goto errout;
}
tctx->conf = NULL;
ssnprintf(callback_name, sizeof(callback_name),
"write_kafka/%s", tctx->topic_name);
ud.data = tctx;
ud.free_func = kafka_topic_context_free;
status = plugin_register_write (callback_name, kafka_write, &ud);
if (status != 0) {
WARNING ("write_kafka plugin: plugin_register_write (\"%s\") "
"failed with status %i.",
callback_name, status);
goto errout;
}
return;
errout:
if (conf != NULL)
rd_kafka_conf_destroy(conf);
if (tctx->kafka != NULL)
rd_kafka_destroy(tctx->kafka);
if (tctx->topic != NULL)
rd_kafka_topic_destroy(tctx->topic);
if (tctx->topic_name != NULL)
free(tctx->topic_name);
if (tctx->conf != NULL)
rd_kafka_topic_conf_destroy(tctx->conf);
sfree(tctx);
} /* }}} int kafka_config_topic */
int kafka_produce(rd_kafka_t *r, char* topic, char* msg, int msg_len, int report, long timeout)
{
char errstr[512];
rd_kafka_topic_t *rkt;
struct produce_cb_params pcb = {1, 0, 0, 0, 0, NULL};
void *opaque;
int partition = RD_KAFKA_PARTITION_UA;
//decide whether to pass callback params or not...
if (report)
opaque = &pcb;
else
opaque = NULL;
rd_kafka_topic_conf_t *topic_conf;
if (r == NULL)
{
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "phpkafka - no connection to produce to topic: %s", topic);
}
return -2;
}
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
char timeoutStr[64];
snprintf(timeoutStr, 64, "%lu", timeout);
if (rd_kafka_topic_conf_set(topic_conf, "message.timeout.ms", timeoutStr, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK)
{
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(
LOG_ERR,
"Failed to configure topic param 'message.timeout.ms' to %lu before producing; config err was: %s",
timeout,
errstr
);
}
rd_kafka_topic_conf_destroy(topic_conf);
return -3;
}
/* Create topic */
rkt = rd_kafka_topic_new(r, topic, topic_conf);
if (rd_kafka_produce(rkt, partition,
RD_KAFKA_MSG_F_COPY,
/* Payload and length */
msg, msg_len,
/* Optional key and its length */
NULL, 0,
/* Message opaque, provided in
* delivery report callback as
* msg_opaque. */
opaque) == -1) {
if (log_level) {
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - %% Failed to produce to topic %s "
"partition %i: %s",
rd_kafka_topic_name(rkt), partition,
rd_kafka_err2str(
rd_kafka_errno2err(errno)));
}
rd_kafka_topic_destroy(rkt);
return -1;
}
/* Poll to handle delivery reports */
rd_kafka_poll(r, 0);
/* Wait for messages to be delivered */
while (report && pcb.msg_count && rd_kafka_outq_len(r) > 0)
rd_kafka_poll(r, 10);
//set global to NULL again
rd_kafka_topic_destroy(rkt);
return 0;
}
int kafka_produce_batch(rd_kafka_t *r, char *topic, char **msg, int *msg_len, int msg_cnt, int report, long timeout)
{
char errstr[512];
rd_kafka_topic_t *rkt;
struct produce_cb_params pcb = {msg_cnt, 0, 0, 0, 0, NULL};
void *opaque;
int partition = RD_KAFKA_PARTITION_UA;
int i,
err_cnt = 0;
if (report)
opaque = &pcb;
else
opaque = NULL;
rd_kafka_topic_conf_t *topic_conf;
if (r == NULL)
{
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "phpkafka - no connection to produce to topic: %s", topic);
}
return -2;
}
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
char timeoutStr[64];
snprintf(timeoutStr, 64, "%lu", timeout);
if (rd_kafka_topic_conf_set(topic_conf, "message.timeout.ms", timeoutStr, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK)
{
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(
LOG_ERR,
"Failed to configure topic param 'message.timeout.ms' to %lu before producing; config err was: %s",
timeout,
errstr
);
}
rd_kafka_topic_conf_destroy(topic_conf);
return -3;
}
/* Create topic */
rkt = rd_kafka_topic_new(r, topic, topic_conf);
//do we have VLA?
rd_kafka_message_t *messages = calloc(sizeof *messages, msg_cnt);
if (messages == NULL)
{//fallback to individual produce calls
for (i=0;i<msg_cnt;++i)
{
if (rd_kafka_produce(rkt, partition, RD_KAFKA_MSG_F_COPY, msg[i], msg_len[i], NULL, 0, opaque) == -1)
{
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - %% Failed to produce to topic %s "
"partition %i: %s",
rd_kafka_topic_name(rkt), partition,
rd_kafka_err2str(
rd_kafka_errno2err(errno)));
}
}
}
}
else
{
for (i=0;i<msg_cnt;++i)
{
messages[i].payload = msg[i];
messages[i].len = msg_len[i];
}
i = rd_kafka_produce_batch(rkt, partition, RD_KAFKA_MSG_F_COPY, messages, msg_cnt);
if (i < msg_cnt)
{
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_WARNING, "Failed to queue full message batch, %d of %d were put in queue", i, msg_cnt);
}
}
err_cnt = msg_cnt - i;
free(messages);
messages = NULL;
}
/* Poll to handle delivery reports */
rd_kafka_poll(r, 0);
/* Wait for messages to be delivered */
while (report && pcb.msg_count && rd_kafka_outq_len(r) > 0)
rd_kafka_poll(r, 10);
//set global to NULL again
rd_kafka_topic_destroy(rkt);
if (report)
err_cnt = pcb.err_count;
return err_cnt;
}
int kafka_produce_report(rd_kafka_t *r, const char *topic, char *msg, int msg_len, long timeout)
{
char errstr[512];
rd_kafka_topic_t *rkt = NULL;
int partition = RD_KAFKA_PARTITION_UA;
rd_kafka_topic_conf_t *conf = NULL;
struct produce_cb_params pcb = {1, 0, 0, 0, 0, NULL};
if (r == NULL)
{
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "No connection provided to produce to topic %s", topic);
}
return -2;
}
/* Topic configuration */
conf = rd_kafka_topic_conf_new();
rd_kafka_topic_conf_set(conf,"produce.offset.report", "true", errstr, sizeof errstr );
char timeoutStr[64];
snprintf(timeoutStr, 64, "%lu", timeout);
if (rd_kafka_topic_conf_set(conf, "message.timeout.ms", timeoutStr, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK)
{
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(
LOG_ERR,
"Failed to configure topic param 'message.timeout.ms' to %lu before producing; config err was: %s",
timeout,
errstr
);
}
rd_kafka_topic_conf_destroy(conf);
return -3;
}
//callback already set in kafka_set_connection
rkt = rd_kafka_topic_new(r, topic, conf);
if (!rkt)
{
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "Failed to open topic %s", topic);
}
rd_kafka_topic_conf_destroy(conf);
return -1;
}
//begin producing:
if (rd_kafka_produce(rkt, partition, RD_KAFKA_MSG_F_COPY, msg, msg_len,NULL, 0,&pcb) == -1)
{
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "Failed to produce message: %s", rd_kafka_err2str(rd_kafka_errno2err(errno)));
}
//handle delivery response (callback)
rd_kafka_poll(rk, 0);
rd_kafka_topic_destroy(rkt);
return -1;
}
rd_kafka_poll(rk, 0);
while(pcb.msg_count && rd_kafka_outq_len(r) > 0)
rd_kafka_poll(r, 10);
rd_kafka_topic_destroy(rkt);
return 0;
}
int main_0029_assign_offset (int argc, char **argv) {
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_topic_conf_t *tconf;
rd_kafka_topic_partition_list_t *parts;
uint64_t testid;
int i;
test_timing_t t_simple, t_hl;
test_msgver_t mv;
test_conf_init(NULL, &tconf, 20 + (test_session_timeout_ms * 3 / 1000));
test_topic_conf_set(tconf, "auto.offset.reset", "smallest");
/* Produce X messages to Y partitions so we get a
* nice seekable 0..X offset one each partition. */
/* Produce messages */
testid = test_id_generate();
rk = test_create_producer();
rkt = test_create_producer_topic(rk, topic, NULL);
parts = rd_kafka_topic_partition_list_new(partitions);
for (i = 0 ; i < partitions ; i++) {
test_produce_msgs(rk, rkt, testid, i, 0, msgcnt, NULL, 0);
rd_kafka_topic_partition_list_add(parts, topic, i)->offset =
msgcnt / 2;
}
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
/* Simple consumer */
TIMING_START(&t_simple, "SIMPLE.CONSUMER");
rk = test_create_consumer(topic, NULL, NULL, NULL);
test_msgver_init(&mv, testid);
test_consumer_assign("SIMPLE.ASSIGN", rk, parts);
test_consumer_poll("SIMPLE.CONSUME", rk, testid, -1, 0,
partitions * (msgcnt / 2), &mv);
for (i = 0 ; i < partitions ; i++)
test_msgver_verify_part("HL.MSGS", &mv, TEST_MSGVER_ALL_PART,
topic, i, msgcnt/2, msgcnt/2);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_simple);
rd_kafka_topic_partition_list_destroy(parts);
/* High-level consumer.
* Offsets are set in rebalance callback. */
TIMING_START(&t_hl, "HL.CONSUMER");
test_msgver_init(&mv, testid);
rk = test_create_consumer(topic, rebalance_cb, NULL, NULL);
test_consumer_subscribe(rk, topic);
test_consumer_poll("HL.CONSUME", rk, testid, -1, 0,
partitions * (msgcnt / 2), &mv);
for (i = 0 ; i < partitions ; i++)
test_msgver_verify_part("HL.MSGS", &mv, TEST_MSGVER_ALL_PART,
topic, i, msgcnt/2, msgcnt/2);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_hl);
rd_kafka_topic_conf_destroy(tconf);
return 0;
}
int main_0006_symbols (int argc, char **argv) {
if (argc < 0 /* always false */) {
rd_kafka_version();
rd_kafka_version_str();
rd_kafka_get_debug_contexts();
rd_kafka_get_err_descs(NULL, NULL);
rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR);
rd_kafka_err2name(RD_KAFKA_RESP_ERR_NO_ERROR);
rd_kafka_errno2err(EINVAL);
rd_kafka_errno();
rd_kafka_last_error();
rd_kafka_conf_new();
rd_kafka_conf_destroy(NULL);
rd_kafka_conf_dup(NULL);
rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_conf_set_dr_cb(NULL, NULL);
rd_kafka_conf_set_dr_msg_cb(NULL, NULL);
rd_kafka_conf_set_error_cb(NULL, NULL);
rd_kafka_conf_set_stats_cb(NULL, NULL);
rd_kafka_conf_set_log_cb(NULL, NULL);
rd_kafka_conf_set_socket_cb(NULL, NULL);
rd_kafka_conf_set_rebalance_cb(NULL, NULL);
rd_kafka_conf_set_offset_commit_cb(NULL, NULL);
rd_kafka_conf_set_throttle_cb(NULL, NULL);
rd_kafka_conf_set_default_topic_conf(NULL, NULL);
rd_kafka_conf_get(NULL, NULL, NULL, NULL);
#ifndef _MSC_VER
rd_kafka_conf_set_open_cb(NULL, NULL);
#endif
rd_kafka_conf_set_opaque(NULL, NULL);
rd_kafka_opaque(NULL);
rd_kafka_conf_dump(NULL, NULL);
rd_kafka_topic_conf_dump(NULL, NULL);
rd_kafka_conf_dump_free(NULL, 0);
rd_kafka_conf_properties_show(NULL);
rd_kafka_topic_conf_new();
rd_kafka_topic_conf_dup(NULL);
rd_kafka_topic_conf_destroy(NULL);
rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_topic_conf_set_opaque(NULL, NULL);
rd_kafka_topic_conf_get(NULL, NULL, NULL, NULL);
rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL);
rd_kafka_topic_partition_available(NULL, 0);
rd_kafka_topic_opaque(NULL);
rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_msg_partitioner_consistent(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_msg_partitioner_consistent_random(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_new(0, NULL, NULL, 0);
rd_kafka_destroy(NULL);
rd_kafka_name(NULL);
rd_kafka_memberid(NULL);
rd_kafka_topic_new(NULL, NULL, NULL);
rd_kafka_topic_destroy(NULL);
rd_kafka_topic_name(NULL);
rd_kafka_message_destroy(NULL);
rd_kafka_message_errstr(NULL);
rd_kafka_message_timestamp(NULL, NULL);
rd_kafka_consume_start(NULL, 0, 0);
rd_kafka_consume_stop(NULL, 0);
rd_kafka_consume(NULL, 0, 0);
rd_kafka_consume_batch(NULL, 0, 0, NULL, 0);
rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL);
rd_kafka_offset_store(NULL, 0, 0);
rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL);
rd_kafka_produce_batch(NULL, 0, 0, NULL, 0);
rd_kafka_poll(NULL, 0);
rd_kafka_brokers_add(NULL, NULL);
/* DEPRECATED: rd_kafka_set_logger(NULL, NULL); */
rd_kafka_set_log_level(NULL, 0);
rd_kafka_log_print(NULL, 0, NULL, NULL);
#ifndef _MSC_VER
rd_kafka_log_syslog(NULL, 0, NULL, NULL);
#endif
rd_kafka_outq_len(NULL);
rd_kafka_dump(NULL, NULL);
rd_kafka_thread_cnt();
rd_kafka_wait_destroyed(0);
rd_kafka_metadata(NULL, 0, NULL, NULL, 0);
rd_kafka_metadata_destroy(NULL);
rd_kafka_queue_destroy(NULL);
rd_kafka_consume_start_queue(NULL, 0, 0, NULL);
rd_kafka_consume_queue(NULL, 0);
rd_kafka_consume_batch_queue(NULL, 0, NULL, 0);
rd_kafka_consume_callback_queue(NULL, 0, NULL, NULL);
rd_kafka_seek(NULL, 0, 0, 0);
rd_kafka_yield(NULL);
rd_kafka_mem_free(NULL, NULL);
rd_kafka_list_groups(NULL, NULL, NULL, 0);
rd_kafka_group_list_destroy(NULL);
/* KafkaConsumer API */
rd_kafka_subscribe(NULL, NULL);
rd_kafka_unsubscribe(NULL);
rd_kafka_subscription(NULL, NULL);
rd_kafka_consumer_poll(NULL, 0);
rd_kafka_consumer_close(NULL);
rd_kafka_assign(NULL, NULL);
rd_kafka_assignment(NULL, NULL);
rd_kafka_commit(NULL, NULL, 0);
rd_kafka_commit_message(NULL, NULL, 0);
rd_kafka_committed(NULL, NULL, 0);
rd_kafka_position(NULL, NULL);
/* TopicPartition */
rd_kafka_topic_partition_list_new(0);
rd_kafka_topic_partition_list_destroy(NULL);
rd_kafka_topic_partition_list_add(NULL, NULL, 0);
rd_kafka_topic_partition_list_add_range(NULL, NULL, 0, 0);
rd_kafka_topic_partition_list_del(NULL, NULL, 0);
rd_kafka_topic_partition_list_del_by_idx(NULL, 0);
rd_kafka_topic_partition_list_copy(NULL);
rd_kafka_topic_partition_list_set_offset(NULL, NULL, 0, 0);
rd_kafka_topic_partition_list_find(NULL, NULL, 0);
rd_kafka_query_watermark_offsets(NULL, NULL, 0, NULL, NULL, 0);
rd_kafka_get_watermark_offsets(NULL, NULL, 0, NULL, NULL);
}
return 0;
}
int main_0018_cgrp_term (int argc, char **argv) {
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
#define _CONS_CNT 2
rd_kafka_t *rk_p, *rk_c[_CONS_CNT];
rd_kafka_topic_t *rkt_p;
int msg_cnt = 1000;
int msg_base = 0;
int partition_cnt = 2;
int partition;
uint64_t testid;
rd_kafka_topic_conf_t *default_topic_conf;
rd_kafka_topic_partition_list_t *topics;
rd_kafka_resp_err_t err;
test_timing_t t_assign, t_consume;
char errstr[512];
int i;
testid = test_id_generate();
/* Produce messages */
rk_p = test_create_producer();
rkt_p = test_create_producer_topic(rk_p, topic, NULL);
for (partition = 0 ; partition < partition_cnt ; partition++) {
test_produce_msgs(rk_p, rkt_p, testid, partition,
msg_base+(partition*msg_cnt), msg_cnt,
NULL, 0);
}
rd_kafka_topic_destroy(rkt_p);
rd_kafka_destroy(rk_p);
test_conf_init(NULL, &default_topic_conf,
(test_session_timeout_ms * 3) / 1000);
if (rd_kafka_topic_conf_set(default_topic_conf, "auto.offset.reset",
"smallest", errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK)
TEST_FAIL("%s\n", errstr);
/* Fill in topic subscription set */
topics = rd_kafka_topic_partition_list_new(1);
rd_kafka_topic_partition_list_add(topics, topic, -1);
/* Create consumers and start subscription */
for (i = 0 ; i < _CONS_CNT ; i++) {
rk_c[i] = test_create_consumer(topic/*group_id*/,
rebalance_cb, NULL,
rd_kafka_topic_conf_dup(
default_topic_conf),
NULL);
err = rd_kafka_poll_set_consumer(rk_c[i]);
if (err)
TEST_FAIL("poll_set_consumer: %s\n",
rd_kafka_err2str(err));
err = rd_kafka_subscribe(rk_c[i], topics);
if (err)
TEST_FAIL("subscribe: %s\n", rd_kafka_err2str(err));
}
rd_kafka_topic_conf_destroy(default_topic_conf);
rd_kafka_topic_partition_list_destroy(topics);
/* Wait for both consumers to get an assignment */
TIMING_START(&t_assign, "WAIT.ASSIGN");
while (assign_cnt < _CONS_CNT)
consume_all(rk_c, _CONS_CNT, msg_cnt,
test_session_timeout_ms + 3000);
TIMING_STOP(&t_assign);
/* Now close one of the consumers, this will cause a rebalance. */
TEST_SAY("Closing down 1/%d consumer(s): %s\n", _CONS_CNT,
rd_kafka_name(rk_c[0]));
err = rd_kafka_consumer_close(rk_c[0]);
if (err)
TEST_FAIL("consumer_close failed: %s\n", rd_kafka_err2str(err));
rd_kafka_destroy(rk_c[0]);
rk_c[0] = NULL;
/* Let remaining consumers run for a while to take over the now
* lost partitions. */
if (assign_cnt != _CONS_CNT-1)
TEST_FAIL("assign_cnt %d, should be %d\n",
assign_cnt, _CONS_CNT-1);
TIMING_START(&t_consume, "CONSUME.WAIT");
consume_all(rk_c, _CONS_CNT, msg_cnt, test_session_timeout_ms + 3000);
TIMING_STOP(&t_consume);
TEST_SAY("Closing remaining consumers\n");
for (i = 0 ; i < _CONS_CNT ; i++) {
test_timing_t t_close;
rd_kafka_topic_partition_list_t *sub;
int j;
if (!rk_c[i])
continue;
/* Query subscription */
err = rd_kafka_subscription(rk_c[i], &sub);
if (err)
TEST_FAIL("%s: subscription() failed: %s\n",
rd_kafka_name(rk_c[i]),
rd_kafka_err2str(err));
TEST_SAY("%s: subscription (%d):\n",
rd_kafka_name(rk_c[i]), sub->cnt);
for (j = 0 ; j < sub->cnt ; j++)
TEST_SAY(" %s\n", sub->elems[j].topic);
rd_kafka_topic_partition_list_destroy(sub);
/* Run an explicit unsubscribe() (async) prior to close()
* to trigger race condition issues on termination. */
TEST_SAY("Unsubscribing instance %s\n", rd_kafka_name(rk_c[i]));
err = rd_kafka_unsubscribe(rk_c[i]);
if (err)
TEST_FAIL("%s: unsubscribe failed: %s\n",
rd_kafka_name(rk_c[i]),
rd_kafka_err2str(err));
TEST_SAY("Closing %s\n", rd_kafka_name(rk_c[i]));
TIMING_START(&t_close, "CONSUMER.CLOSE");
err = rd_kafka_consumer_close(rk_c[i]);
TIMING_STOP(&t_close);
if (err)
TEST_FAIL("consumer_close failed: %s\n",
rd_kafka_err2str(err));
rd_kafka_destroy(rk_c[i]);
rk_c[i] = NULL;
}
TEST_SAY("%d/%d messages consumed\n", consumed_msg_cnt, msg_cnt);
if (consumed_msg_cnt < msg_cnt)
TEST_FAIL("Only %d/%d messages were consumed\n",
consumed_msg_cnt, msg_cnt);
else if (consumed_msg_cnt > msg_cnt)
TEST_SAY("At least %d/%d messages were consumed "
"multiple times\n",
consumed_msg_cnt - msg_cnt, msg_cnt);
return 0;
}
