static int Producer_init (PyObject *selfobj, PyObject *args, PyObject *kwargs) {
Handle *self = (Handle *)selfobj;
char errstr[256];
rd_kafka_conf_t *conf;
if (self->rk) {
PyErr_SetString(PyExc_RuntimeError,
"Producer already __init__:ialized");
return -1;
}
self->type = RD_KAFKA_PRODUCER;
if (!(conf = common_conf_setup(RD_KAFKA_PRODUCER, self,
args, kwargs)))
return -1;
rd_kafka_conf_set_dr_msg_cb(conf, dr_msg_cb);
self->rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr));
if (!self->rk) {
cfl_PyErr_Format(rd_kafka_last_error(),
"Failed to create producer: %s", errstr);
rd_kafka_conf_destroy(conf);
return -1;
}
/* Forward log messages to poll queue */
if (self->logger)
rd_kafka_set_log_queue(self->rk, NULL);
return 0;
}
int main (int argc, char **argv) {
if (argc < 0 /* always false */) {
rd_kafka_version();
rd_kafka_version_str();
rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR);
rd_kafka_errno2err(EINVAL);
rd_kafka_conf_new();
rd_kafka_conf_destroy(NULL);
rd_kafka_conf_dup(NULL);
rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_conf_set_dr_cb(NULL, NULL);
rd_kafka_conf_set_error_cb(NULL, NULL);
rd_kafka_conf_set_stats_cb(NULL, NULL);
rd_kafka_conf_set_opaque(NULL, NULL);
rd_kafka_conf_dump(NULL, NULL);
rd_kafka_topic_conf_dump(NULL, NULL);
rd_kafka_conf_dump_free(NULL, 0);
rd_kafka_conf_properties_show(NULL);
rd_kafka_topic_conf_new();
rd_kafka_topic_conf_dup(NULL);
rd_kafka_topic_conf_destroy(NULL);
rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_topic_conf_set_opaque(NULL, NULL);
rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL);
rd_kafka_topic_partition_available(NULL, 0);
rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_new(0, NULL, NULL, 0);
rd_kafka_destroy(NULL);
rd_kafka_name(NULL);
rd_kafka_topic_new(NULL, NULL, NULL);
rd_kafka_topic_destroy(NULL);
rd_kafka_topic_name(NULL);
rd_kafka_message_destroy(NULL);
rd_kafka_message_errstr(NULL);
rd_kafka_consume_start(NULL, 0, 0);
rd_kafka_consume_stop(NULL, 0);
rd_kafka_consume(NULL, 0, 0);
rd_kafka_consume_batch(NULL, 0, 0, NULL, 0);
rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL);
rd_kafka_offset_store(NULL, 0, 0);
rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL);
rd_kafka_poll(NULL, 0);
rd_kafka_brokers_add(NULL, NULL);
rd_kafka_set_logger(NULL, NULL);
rd_kafka_set_log_level(NULL, 0);
rd_kafka_log_print(NULL, 0, NULL, NULL);
rd_kafka_log_syslog(NULL, 0, NULL, NULL);
rd_kafka_outq_len(NULL);
rd_kafka_dump(NULL, NULL);
rd_kafka_thread_cnt();
rd_kafka_wait_destroyed(0);
}
return 0;
}
static int kafka_handle(struct kafka_topic_context *ctx) /* {{{ */
{
char errbuf[1024];
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
if (ctx->kafka != NULL && ctx->topic != NULL)
return(0);
if (ctx->kafka == NULL) {
if ((conf = rd_kafka_conf_dup(ctx->kafka_conf)) == NULL) {
ERROR("write_kafka plugin: cannot duplicate kafka config");
return(1);
}
if ((ctx->kafka = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errbuf, sizeof(errbuf))) == NULL) {
ERROR("write_kafka plugin: cannot create kafka handle.");
return 1;
}
rd_kafka_conf_destroy(ctx->kafka_conf);
ctx->kafka_conf = NULL;
INFO ("write_kafka plugin: created KAFKA handle : %s", rd_kafka_name(ctx->kafka));
#if defined(HAVE_LIBRDKAFKA_LOGGER) && !defined(HAVE_LIBRDKAFKA_LOG_CB)
rd_kafka_set_logger(ctx->kafka, kafka_log);
#endif
}
if (ctx->topic == NULL ) {
if ((topic_conf = rd_kafka_topic_conf_dup(ctx->conf)) == NULL) {
ERROR("write_kafka plugin: cannot duplicate kafka topic config");
return 1;
}
if ((ctx->topic = rd_kafka_topic_new(ctx->kafka, ctx->topic_name,
topic_conf)) == NULL) {
ERROR("write_kafka plugin: cannot create topic : %s\n",
rd_kafka_err2str(rd_kafka_errno2err(errno)));
return errno;
}
rd_kafka_topic_conf_destroy(ctx->conf);
ctx->conf = NULL;
INFO ("write_kafka plugin: handle created for topic : %s", rd_kafka_topic_name(ctx->topic));
}
return(0);
} /* }}} int kafka_handle */
static void kafka_topic_context_free(void *p) /* {{{ */
{
struct kafka_topic_context *ctx = p;
if (ctx == NULL)
return;
if (ctx->topic_name != NULL)
sfree(ctx->topic_name);
if (ctx->topic != NULL)
rd_kafka_topic_destroy(ctx->topic);
if (ctx->conf != NULL)
rd_kafka_topic_conf_destroy(ctx->conf);
if (ctx->kafka_conf != NULL)
rd_kafka_conf_destroy(ctx->kafka_conf);
if (ctx->kafka != NULL)
rd_kafka_destroy(ctx->kafka);
sfree(ctx);
} /* }}} void kafka_topic_context_free */
static int Consumer_init (PyObject *selfobj, PyObject *args, PyObject *kwargs) {
Handle *self = (Handle *)selfobj;
char errstr[256];
rd_kafka_conf_t *conf;
if (self->rk) {
PyErr_SetString(PyExc_RuntimeError,
"Consumer already initialized");
return -1;
}
self->type = RD_KAFKA_CONSUMER;
if (!(conf = common_conf_setup(RD_KAFKA_CONSUMER, self,
args, kwargs)))
return -1; /* Exception raised by ..conf_setup() */
rd_kafka_conf_set_rebalance_cb(conf, Consumer_rebalance_cb);
rd_kafka_conf_set_offset_commit_cb(conf, Consumer_offset_commit_cb);
self->rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf,
errstr, sizeof(errstr));
if (!self->rk) {
cfl_PyErr_Format(rd_kafka_last_error(),
"Failed to create consumer: %s", errstr);
rd_kafka_conf_destroy(conf);
return -1;
}
/* Forward log messages to main queue which is then forwarded
* to the consumer queue */
if (self->logger)
rd_kafka_set_log_queue(self->rk, NULL);
rd_kafka_poll_set_consumer(self->rk);
self->u.Consumer.rkqu = rd_kafka_queue_get_consumer(self->rk);
assert(self->u.Consumer.rkqu);
return 0;
}
static PyObject *
RdkHandle_stop(RdkHandle *self)
{
/* We'll only ever get a locking error if we programmed ourselves into a
* deadlock. We'd have to admit defeat, abort, and leak this RdkHandle */
if (RdkHandle_excl_lock(self)) return NULL;
Py_BEGIN_ALLOW_THREADS /* avoid callbacks deadlocking */
if (self->rdk_queue_handle) {
rd_kafka_queue_destroy(self->rdk_queue_handle);
self->rdk_queue_handle = NULL;
}
if (self->rdk_topic_handle) {
rd_kafka_topic_destroy(self->rdk_topic_handle);
self->rdk_topic_handle = NULL;
}
if (self->rdk_handle) {
PyObject *opaque = (PyObject *)rd_kafka_opaque(self->rdk_handle);
Py_XDECREF(opaque);
rd_kafka_destroy(self->rdk_handle);
self->rdk_handle = NULL;
}
if (self->rdk_conf) {
rd_kafka_conf_destroy(self->rdk_conf);
self->rdk_conf = NULL;
}
if (self->rdk_topic_conf) {
rd_kafka_topic_conf_destroy(self->rdk_topic_conf);
self->rdk_topic_conf = NULL;
}
Py_END_ALLOW_THREADS
Py_CLEAR(self->partition_ids);
if (RdkHandle_unlock(self)) return NULL;
Py_INCREF(Py_None);
return Py_None;
}
/**
* @brief Interceptors must not be copied automatically by conf_dup()
* unless the interceptors have added on_conf_dup().
* This behaviour makes sure an interceptor's instance
* is not duplicated without the interceptor's knowledge or
* assistance.
*/
static void do_test_conf_copy (const char *topic) {
rd_kafka_conf_t *conf, *conf2;
int i;
rd_kafka_t *rk;
TEST_SAY(_C_MAG "[ %s ]\n" _C_CLR, __FUNCTION__);
memset(&msgs[0], 0, sizeof(msgs));
test_conf_init(&conf, NULL, 0);
rd_kafka_conf_interceptor_add_on_new(conf, "on_new_conf_copy",
on_new_producer, NULL);
/* Now copy the configuration to verify that interceptors are
* NOT copied. */
conf2 = conf;
conf = rd_kafka_conf_dup(conf2);
rd_kafka_conf_destroy(conf2);
/* Create producer */
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
for (i = 0 ; i < msgcnt-1 ; i++)
do_test_produce(rk, topic, RD_KAFKA_PARTITION_UA, i, 0, 0);
/* Wait for messages to be delivered */
test_flush(rk, -1);
/* Verify acks */
for (i = 0 ; i < msgcnt ; i++) {
struct msg_state *msg = &msgs[i];
msg_verify_ic_cnt(msg, "on_ack", msg->bits[_ON_ACK], 0);
}
rd_kafka_destroy(rk);
}
int main_0004_conf (int argc, char **argv) {
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *ignore_conf, *conf, *conf2;
rd_kafka_topic_conf_t *ignore_topic_conf, *tconf, *tconf2;
char errstr[512];
const char **arr_orig, **arr_dup;
size_t cnt_orig, cnt_dup;
int i;
const char *topic;
static const char *gconfs[] = {
"message.max.bytes", "12345", /* int property */
"client.id", "my id", /* string property */
"debug", "topic,metadata", /* S2F property */
"topic.blacklist", "__.*", /* #778 */
"auto.offset.reset", "earliest", /* Global->Topic fallthru */
#if WITH_ZLIB
"compression.codec", "gzip", /* S2I property */
#endif
NULL
};
static const char *tconfs[] = {
"request.required.acks", "-1", /* int */
"auto.commit.enable", "false", /* bool */
"auto.offset.reset", "error", /* S2I */
"offset.store.path", "my/path", /* string */
NULL
};
test_conf_init(&ignore_conf, &ignore_topic_conf, 10);
rd_kafka_conf_destroy(ignore_conf);
rd_kafka_topic_conf_destroy(ignore_topic_conf);
topic = test_mk_topic_name("0004", 0);
/* Set up a global config object */
conf = rd_kafka_conf_new();
rd_kafka_conf_set_dr_cb(conf, dr_cb);
rd_kafka_conf_set_error_cb(conf, error_cb);
for (i = 0 ; gconfs[i] ; i += 2) {
if (rd_kafka_conf_set(conf, gconfs[i], gconfs[i+1],
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK)
TEST_FAIL("%s\n", errstr);
}
/* Set up a topic config object */
tconf = rd_kafka_topic_conf_new();
rd_kafka_topic_conf_set_partitioner_cb(tconf, partitioner);
rd_kafka_topic_conf_set_opaque(tconf, (void *)0xbeef);
for (i = 0 ; tconfs[i] ; i += 2) {
if (rd_kafka_topic_conf_set(tconf, tconfs[i], tconfs[i+1],
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK)
TEST_FAIL("%s\n", errstr);
}
/* Verify global config */
arr_orig = rd_kafka_conf_dump(conf, &cnt_orig);
conf_verify(__LINE__, arr_orig, cnt_orig, gconfs);
/* Verify copied global config */
conf2 = rd_kafka_conf_dup(conf);
arr_dup = rd_kafka_conf_dump(conf2, &cnt_dup);
conf_verify(__LINE__, arr_dup, cnt_dup, gconfs);
conf_cmp("global", arr_orig, cnt_orig, arr_dup, cnt_dup);
rd_kafka_conf_dump_free(arr_orig, cnt_orig);
rd_kafka_conf_dump_free(arr_dup, cnt_dup);
/* Verify topic config */
arr_orig = rd_kafka_topic_conf_dump(tconf, &cnt_orig);
conf_verify(__LINE__, arr_orig, cnt_orig, tconfs);
/* Verify copied topic config */
tconf2 = rd_kafka_topic_conf_dup(tconf);
arr_dup = rd_kafka_topic_conf_dump(tconf2, &cnt_dup);
conf_verify(__LINE__, arr_dup, cnt_dup, tconfs);
conf_cmp("topic", arr_orig, cnt_orig, arr_dup, cnt_dup);
rd_kafka_conf_dump_free(arr_orig, cnt_orig);
rd_kafka_conf_dump_free(arr_dup, cnt_dup);
/*
* Create kafka instances using original and copied confs
*/
/* original */
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
rkt = rd_kafka_topic_new(rk, topic, tconf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
rd_strerror(errno));
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
/* copied */
rk = test_create_handle(RD_KAFKA_PRODUCER, conf2);
rkt = rd_kafka_topic_new(rk, topic, tconf2);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
rd_strerror(errno));
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
/* Incremental S2F property.
* NOTE: The order of fields returned in get() is hardcoded here. */
{
static const char *s2fs[] = {
"generic,broker,queue,cgrp",
"generic,broker,queue,cgrp",
"-broker,+queue,topic",
"generic,topic,queue,cgrp",
"-all,security,-fetch,+metadata",
"metadata,security",
NULL
};
TEST_SAY("Incremental S2F tests\n");
conf = rd_kafka_conf_new();
for (i = 0 ; s2fs[i] ; i += 2) {
const char *val;
TEST_SAY(" Set: %s\n", s2fs[i]);
test_conf_set(conf, "debug", s2fs[i]);
val = test_conf_get(conf, "debug");
TEST_SAY(" Now: %s\n", val);
if (strcmp(val, s2fs[i+1]))
TEST_FAIL_LATER("\n"
"Expected: %s\n"
" Got: %s",
s2fs[i+1], val);
}
rd_kafka_conf_destroy(conf);
}
/* Canonical int values, aliases, s2i-verified strings */
{
static const struct {
const char *prop;
const char *val;
const char *exp;
int is_global;
} props[] = {
{ "request.required.acks", "0", "0" },
{ "request.required.acks", "-1", "-1" },
{ "request.required.acks", "1", "1" },
{ "acks", "3", "3" }, /* alias test */
{ "request.required.acks", "393", "393" },
{ "request.required.acks", "bad", NULL },
{ "request.required.acks", "all", "-1" },
{ "request.required.acks", "all", "-1", 1/*fallthru*/ },
{ "acks", "0", "0" }, /* alias test */
#if WITH_SASL
{ "sasl.mechanisms", "GSSAPI", "GSSAPI", 1 },
{ "sasl.mechanisms", "PLAIN", "PLAIN", 1 },
{ "sasl.mechanisms", "GSSAPI,PLAIN", NULL, 1 },
{ "sasl.mechanisms", "", NULL, 1 },
#endif
{ NULL }
};
TEST_SAY("Canonical tests\n");
tconf = rd_kafka_topic_conf_new();
conf = rd_kafka_conf_new();
for (i = 0 ; props[i].prop ; i++) {
char dest[64];
size_t destsz;
rd_kafka_conf_res_t res;
TEST_SAY(" Set: %s=%s expect %s (%s)\n",
props[i].prop, props[i].val, props[i].exp,
props[i].is_global ? "global":"topic");
/* Set value */
if (props[i].is_global)
res = rd_kafka_conf_set(conf,
props[i].prop,
props[i].val,
errstr, sizeof(errstr));
else
res = rd_kafka_topic_conf_set(tconf,
props[i].prop,
props[i].val,
errstr,
sizeof(errstr));
if ((res == RD_KAFKA_CONF_OK ? 1:0) !=
(props[i].exp ? 1:0))
TEST_FAIL("Expected %s, got %s",
props[i].exp ? "success" : "failure",
(res == RD_KAFKA_CONF_OK ? "OK" :
(res == RD_KAFKA_CONF_INVALID ? "INVALID" :
"UNKNOWN")));
if (!props[i].exp)
continue;
/* Get value and compare to expected result */
destsz = sizeof(dest);
if (props[i].is_global)
res = rd_kafka_conf_get(conf,
props[i].prop,
dest, &destsz);
else
res = rd_kafka_topic_conf_get(tconf,
props[i].prop,
dest, &destsz);
TEST_ASSERT(res == RD_KAFKA_CONF_OK,
".._conf_get(%s) returned %d",
props[i].prop, res);
TEST_ASSERT(!strcmp(props[i].exp, dest),
"Expected \"%s\", got \"%s\"",
props[i].exp, dest);
}
rd_kafka_topic_conf_destroy(tconf);
rd_kafka_conf_destroy(conf);
}
return 0;
}
int main (int argc, char **argv) {
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *ignore_conf, *conf, *conf2;
rd_kafka_topic_conf_t *ignore_topic_conf, *tconf, *tconf2;
char errstr[512];
const char **arr_orig, **arr_dup;
size_t cnt_orig, cnt_dup;
int i;
const char *topic;
static const char *gconfs[] = {
"message.max.bytes", "12345", /* int property */
"client.id", "my id", /* string property */
"debug", "topic,metadata", /* S2F property */
"compression.codec", "gzip", /* S2I property */
NULL
};
static const char *tconfs[] = {
"request.required.acks", "-1", /* int */
"auto.commit.enable", "false", /* bool */
"auto.offset.reset", "error", /* S2I */
"offset.store.path", "my/path", /* string */
NULL
};
test_conf_init(&ignore_conf, &ignore_topic_conf, 10);
rd_kafka_conf_destroy(ignore_conf);
rd_kafka_topic_conf_destroy(ignore_topic_conf);
topic = test_mk_topic_name("generic", 0);
/* Set up a global config object */
conf = rd_kafka_conf_new();
rd_kafka_conf_set_dr_cb(conf, dr_cb);
rd_kafka_conf_set_error_cb(conf, error_cb);
for (i = 0 ; gconfs[i] ; i += 2) {
if (rd_kafka_conf_set(conf, gconfs[i], gconfs[i+1],
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK)
TEST_FAIL("%s\n", errstr);
}
/* Set up a topic config object */
tconf = rd_kafka_topic_conf_new();
rd_kafka_topic_conf_set_partitioner_cb(tconf, partitioner);
rd_kafka_topic_conf_set_opaque(tconf, (void *)0xbeef);
for (i = 0 ; tconfs[i] ; i += 2) {
if (rd_kafka_topic_conf_set(tconf, tconfs[i], tconfs[i+1],
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK)
TEST_FAIL("%s\n", errstr);
}
/* Verify global config */
arr_orig = rd_kafka_conf_dump(conf, &cnt_orig);
conf_verify(__LINE__, arr_orig, cnt_orig, gconfs);
/* Verify copied global config */
conf2 = rd_kafka_conf_dup(conf);
arr_dup = rd_kafka_conf_dump(conf2, &cnt_dup);
conf_verify(__LINE__, arr_dup, cnt_dup, gconfs);
conf_cmp("global", arr_orig, cnt_orig, arr_dup, cnt_dup);
rd_kafka_conf_dump_free(arr_orig, cnt_orig);
rd_kafka_conf_dump_free(arr_dup, cnt_dup);
/* Verify topic config */
arr_orig = rd_kafka_topic_conf_dump(tconf, &cnt_orig);
conf_verify(__LINE__, arr_orig, cnt_orig, tconfs);
/* Verify copied topic config */
tconf2 = rd_kafka_topic_conf_dup(tconf);
arr_dup = rd_kafka_topic_conf_dump(tconf2, &cnt_dup);
conf_verify(__LINE__, arr_dup, cnt_dup, tconfs);
conf_cmp("topic", arr_orig, cnt_orig, arr_dup, cnt_dup);
rd_kafka_conf_dump_free(arr_orig, cnt_orig);
rd_kafka_conf_dump_free(arr_dup, cnt_dup);
/*
* Create kafka instances using original and copied confs
*/
/* original */
rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr));
if (!rk)
TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr);
rkt = rd_kafka_topic_new(rk, topic, tconf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
strerror(errno));
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
/* copied */
rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf2,
errstr, sizeof(errstr));
if (!rk)
TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr);
rkt = rd_kafka_topic_new(rk, topic, tconf2);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
strerror(errno));
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
/* Wait for everything to be cleaned up since broker destroys are
* handled in its own thread. */
test_wait_exit(2);
/* If we havent failed at this point then
* there were no threads leaked */
return 0;
}
static int kafka_config(oconfig_item_t *ci) /* {{{ */
{
int i;
oconfig_item_t *child;
rd_kafka_conf_t *conf;
rd_kafka_conf_t *cloned;
rd_kafka_conf_res_t ret;
char errbuf[1024];
if ((conf = rd_kafka_conf_new()) == NULL) {
WARNING("cannot allocate kafka configuration.");
return -1;
}
for (i = 0; i < ci->children_num; i++) {
child = &ci->children[i];
if (strcasecmp("Topic", child->key) == 0) {
if ((cloned = rd_kafka_conf_dup(conf)) == NULL) {
WARNING("write_kafka plugin: cannot allocate memory for kafka config");
goto errout;
}
kafka_config_topic (cloned, child);
} else if (strcasecmp(child->key, "Property") == 0) {
char *key = NULL;
char *val = NULL;
if (child->values_num != 2) {
WARNING("kafka properties need both a key and a value.");
goto errout;
}
if (child->values[0].type != OCONFIG_TYPE_STRING ||
child->values[1].type != OCONFIG_TYPE_STRING) {
WARNING("kafka properties needs string arguments.");
goto errout;
}
if ((key = strdup(child->values[0].value.string)) == NULL) {
WARNING("cannot allocate memory for attribute key.");
goto errout;
}
if ((val = strdup(child->values[1].value.string)) == NULL) {
WARNING("cannot allocate memory for attribute value.");
goto errout;
}
ret = rd_kafka_conf_set(conf, key, val, errbuf, sizeof(errbuf));
if (ret != RD_KAFKA_CONF_OK) {
WARNING("cannot set kafka property %s to %s: %s",
key, val, errbuf);
goto errout;
}
sfree(key);
sfree(val);
} else {
WARNING ("write_kafka plugin: Ignoring unknown "
"configuration option \"%s\" at top level.",
child->key);
}
}
if (conf != NULL)
rd_kafka_conf_destroy(conf);
return (0);
errout:
if (conf != NULL)
rd_kafka_conf_destroy(conf);
return -1;
} /* }}} int kafka_config */
static void kafka_config_topic(rd_kafka_conf_t *conf, oconfig_item_t *ci) /* {{{ */
{
int status;
int i;
struct kafka_topic_context *tctx;
char *key;
char *val;
char callback_name[DATA_MAX_NAME_LEN];
char errbuf[1024];
user_data_t ud;
oconfig_item_t *child;
rd_kafka_conf_res_t ret;
if ((tctx = calloc(1, sizeof (*tctx))) == NULL) {
ERROR ("write_kafka plugin: calloc failed.");
return;
}
tctx->escape_char = '.';
tctx->store_rates = 1;
rd_kafka_conf_set_log_cb(conf, kafka_log);
if ((tctx->kafka = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errbuf, sizeof(errbuf))) == NULL) {
sfree(tctx);
ERROR("write_kafka plugin: cannot create kafka handle.");
return;
}
conf = NULL;
if ((tctx->conf = rd_kafka_topic_conf_new()) == NULL) {
rd_kafka_destroy(tctx->kafka);
sfree(tctx);
ERROR ("write_kafka plugin: cannot create topic configuration.");
return;
}
if (ci->values_num != 1) {
WARNING("kafka topic name needed.");
goto errout;
}
if (ci->values[0].type != OCONFIG_TYPE_STRING) {
WARNING("kafka topic needs a string argument.");
goto errout;
}
if ((tctx->topic_name = strdup(ci->values[0].value.string)) == NULL) {
ERROR("write_kafka plugin: cannot copy topic name.");
goto errout;
}
for (i = 0; i < ci->children_num; i++) {
/*
* The code here could be simplified but makes room
* for easy adding of new options later on.
*/
child = &ci->children[i];
status = 0;
if (strcasecmp ("Property", child->key) == 0) {
if (child->values_num != 2) {
WARNING("kafka properties need both a key and a value.");
goto errout;
}
if (child->values[0].type != OCONFIG_TYPE_STRING ||
child->values[1].type != OCONFIG_TYPE_STRING) {
WARNING("kafka properties needs string arguments.");
goto errout;
}
key = child->values[0].value.string;
val = child->values[0].value.string;
ret = rd_kafka_topic_conf_set(tctx->conf,key, val,
errbuf, sizeof(errbuf));
if (ret != RD_KAFKA_CONF_OK) {
WARNING("cannot set kafka topic property %s to %s: %s.",
key, val, errbuf);
goto errout;
}
} else if (strcasecmp ("Key", child->key) == 0) {
char *tmp_buf = NULL;
status = cf_util_get_string(child, &tmp_buf);
if (status != 0) {
WARNING("write_kafka plugin: invalid key supplied");
break;
}
if (strcasecmp(tmp_buf, "Random") != 0) {
tctx->has_key = 1;
tctx->key = crc32_buffer((u_char *)tmp_buf, strlen(tmp_buf));
}
sfree(tmp_buf);
} else if (strcasecmp ("Format", child->key) == 0) {
status = cf_util_get_string(child, &key);
if (status != 0)
goto errout;
assert(key != NULL);
if (strcasecmp(key, "Command") == 0) {
tctx->format = KAFKA_FORMAT_COMMAND;
} else if (strcasecmp(key, "Graphite") == 0) {
tctx->format = KAFKA_FORMAT_GRAPHITE;
} else if (strcasecmp(key, "Json") == 0) {
tctx->format = KAFKA_FORMAT_JSON;
} else {
WARNING ("write_kafka plugin: Invalid format string: %s",
key);
}
sfree(key);
} else if (strcasecmp ("StoreRates", child->key) == 0) {
status = cf_util_get_boolean (child, &tctx->store_rates);
(void) cf_util_get_flag (child, &tctx->graphite_flags,
GRAPHITE_STORE_RATES);
} else if (strcasecmp ("GraphiteSeparateInstances", child->key) == 0) {
status = cf_util_get_flag (child, &tctx->graphite_flags,
GRAPHITE_SEPARATE_INSTANCES);
} else if (strcasecmp ("GraphiteAlwaysAppendDS", child->key) == 0) {
status = cf_util_get_flag (child, &tctx->graphite_flags,
GRAPHITE_ALWAYS_APPEND_DS);
} else if (strcasecmp ("GraphitePrefix", child->key) == 0) {
status = cf_util_get_string (child, &tctx->prefix);
} else if (strcasecmp ("GraphitePostfix", child->key) == 0) {
status = cf_util_get_string (child, &tctx->postfix);
} else if (strcasecmp ("GraphiteEscapeChar", child->key) == 0) {
char *tmp_buff = NULL;
status = cf_util_get_string (child, &tmp_buff);
if (strlen (tmp_buff) > 1)
WARNING ("write_kafka plugin: The option \"GraphiteEscapeChar\" handles "
"only one character. Others will be ignored.");
tctx->escape_char = tmp_buff[0];
sfree (tmp_buff);
} else {
WARNING ("write_kafka plugin: Invalid directive: %s.", child->key);
}
if (status != 0)
break;
}
rd_kafka_topic_conf_set_partitioner_cb(tctx->conf, kafka_partition);
rd_kafka_topic_conf_set_opaque(tctx->conf, tctx);
if ((tctx->topic = rd_kafka_topic_new(tctx->kafka, tctx->topic_name,
tctx->conf)) == NULL) {
ERROR("write_kafka plugin: cannot create topic.");
goto errout;
}
tctx->conf = NULL;
ssnprintf(callback_name, sizeof(callback_name),
"write_kafka/%s", tctx->topic_name);
ud.data = tctx;
ud.free_func = kafka_topic_context_free;
status = plugin_register_write (callback_name, kafka_write, &ud);
if (status != 0) {
WARNING ("write_kafka plugin: plugin_register_write (\"%s\") "
"failed with status %i.",
callback_name, status);
goto errout;
}
return;
errout:
if (conf != NULL)
rd_kafka_conf_destroy(conf);
if (tctx->kafka != NULL)
rd_kafka_destroy(tctx->kafka);
if (tctx->topic != NULL)
rd_kafka_topic_destroy(tctx->topic);
if (tctx->topic_name != NULL)
free(tctx->topic_name);
if (tctx->conf != NULL)
rd_kafka_topic_conf_destroy(tctx->conf);
sfree(tctx);
} /* }}} int kafka_config_topic */
rd_kafka_t *kafka_get_connection(kafka_connection_params params, const char *brokers)
{
rd_kafka_t *r = NULL;
char errstr[512];
rd_kafka_conf_t *conf = rd_kafka_conf_new();
//set error callback
rd_kafka_conf_set_error_cb(conf, kafka_err_cb);
if (params.type == RD_KAFKA_CONSUMER)
{
if (params.queue_buffer)
rd_kafka_conf_set(conf, "queued.min.messages", params.queue_buffer, NULL, 0);
r = rd_kafka_new(params.type, conf, errstr, sizeof errstr);
if (!r)
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "Failed to connect to kafka: %s", errstr);
}
//destroy config, no connection to use it...
rd_kafka_conf_destroy(conf);
return NULL;
}
if (!rd_kafka_brokers_add(r, brokers))
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "Failed to connect to brokers %s", brokers);
}
rd_kafka_destroy(r);
return NULL;
}
return r;
}
if (params.compression)
{
rd_kafka_conf_res_t result = rd_kafka_conf_set(
conf, "compression.codec",params.compression, errstr, sizeof errstr
);
if (result != RD_KAFKA_CONF_OK)
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ALERT, "Failed to set compression %s: %s", params.compression, errstr);
}
rd_kafka_conf_destroy(conf);
return NULL;
}
}
if (params.retry_count)
{
rd_kafka_conf_res_t result = rd_kafka_conf_set(
conf, "message.send.max.retries",params.retry_count, errstr, sizeof errstr
);
if (result != RD_KAFKA_CONF_OK)
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ALERT, "Failed to set compression %s: %s", params.compression, errstr);
}
rd_kafka_conf_destroy(conf);
return NULL;
}
}
if (params.retry_interval)
{
rd_kafka_conf_res_t result = rd_kafka_conf_set(
conf, "retry.backoff.ms",params.retry_interval, errstr, sizeof errstr
);
if (result != RD_KAFKA_CONF_OK)
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ALERT, "Failed to set compression %s: %s", params.compression, errstr);
}
rd_kafka_conf_destroy(conf);
return NULL;
}
}
if (params.reporting == 1)
rd_kafka_conf_set_dr_cb(conf, kafka_produce_cb_simple);
else if (params.reporting == 2)
rd_kafka_conf_set_dr_msg_cb(conf, kafka_produce_detailed_cb);
r = rd_kafka_new(params.type, conf, errstr, sizeof errstr);
if (!r)
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "Failed to connect to kafka: %s", errstr);
}
//destroy config, no connection to use it...
rd_kafka_conf_destroy(conf);
return NULL;
}
if (!rd_kafka_brokers_add(r, brokers))
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "Failed to connect to brokers %s", brokers);
}
rd_kafka_destroy(r);
return NULL;
}
return r;
}
int main_0006_symbols (int argc, char **argv) {
if (argc < 0 /* always false */) {
rd_kafka_version();
rd_kafka_version_str();
rd_kafka_get_debug_contexts();
rd_kafka_get_err_descs(NULL, NULL);
rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR);
rd_kafka_err2name(RD_KAFKA_RESP_ERR_NO_ERROR);
rd_kafka_errno2err(EINVAL);
rd_kafka_errno();
rd_kafka_last_error();
rd_kafka_conf_new();
rd_kafka_conf_destroy(NULL);
rd_kafka_conf_dup(NULL);
rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_conf_set_dr_cb(NULL, NULL);
rd_kafka_conf_set_dr_msg_cb(NULL, NULL);
rd_kafka_conf_set_error_cb(NULL, NULL);
rd_kafka_conf_set_stats_cb(NULL, NULL);
rd_kafka_conf_set_log_cb(NULL, NULL);
rd_kafka_conf_set_socket_cb(NULL, NULL);
rd_kafka_conf_set_rebalance_cb(NULL, NULL);
rd_kafka_conf_set_offset_commit_cb(NULL, NULL);
rd_kafka_conf_set_throttle_cb(NULL, NULL);
rd_kafka_conf_set_default_topic_conf(NULL, NULL);
rd_kafka_conf_get(NULL, NULL, NULL, NULL);
#ifndef _MSC_VER
rd_kafka_conf_set_open_cb(NULL, NULL);
#endif
rd_kafka_conf_set_opaque(NULL, NULL);
rd_kafka_opaque(NULL);
rd_kafka_conf_dump(NULL, NULL);
rd_kafka_topic_conf_dump(NULL, NULL);
rd_kafka_conf_dump_free(NULL, 0);
rd_kafka_conf_properties_show(NULL);
rd_kafka_topic_conf_new();
rd_kafka_topic_conf_dup(NULL);
rd_kafka_topic_conf_destroy(NULL);
rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_topic_conf_set_opaque(NULL, NULL);
rd_kafka_topic_conf_get(NULL, NULL, NULL, NULL);
rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL);
rd_kafka_topic_partition_available(NULL, 0);
rd_kafka_topic_opaque(NULL);
rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_msg_partitioner_consistent(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_msg_partitioner_consistent_random(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_new(0, NULL, NULL, 0);
rd_kafka_destroy(NULL);
rd_kafka_name(NULL);
rd_kafka_memberid(NULL);
rd_kafka_topic_new(NULL, NULL, NULL);
rd_kafka_topic_destroy(NULL);
rd_kafka_topic_name(NULL);
rd_kafka_message_destroy(NULL);
rd_kafka_message_errstr(NULL);
rd_kafka_message_timestamp(NULL, NULL);
rd_kafka_consume_start(NULL, 0, 0);
rd_kafka_consume_stop(NULL, 0);
rd_kafka_consume(NULL, 0, 0);
rd_kafka_consume_batch(NULL, 0, 0, NULL, 0);
rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL);
rd_kafka_offset_store(NULL, 0, 0);
rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL);
rd_kafka_produce_batch(NULL, 0, 0, NULL, 0);
rd_kafka_poll(NULL, 0);
rd_kafka_brokers_add(NULL, NULL);
/* DEPRECATED: rd_kafka_set_logger(NULL, NULL); */
rd_kafka_set_log_level(NULL, 0);
rd_kafka_log_print(NULL, 0, NULL, NULL);
#ifndef _MSC_VER
rd_kafka_log_syslog(NULL, 0, NULL, NULL);
#endif
rd_kafka_outq_len(NULL);
rd_kafka_dump(NULL, NULL);
rd_kafka_thread_cnt();
rd_kafka_wait_destroyed(0);
rd_kafka_metadata(NULL, 0, NULL, NULL, 0);
rd_kafka_metadata_destroy(NULL);
rd_kafka_queue_destroy(NULL);
rd_kafka_consume_start_queue(NULL, 0, 0, NULL);
rd_kafka_consume_queue(NULL, 0);
rd_kafka_consume_batch_queue(NULL, 0, NULL, 0);
rd_kafka_consume_callback_queue(NULL, 0, NULL, NULL);
rd_kafka_seek(NULL, 0, 0, 0);
rd_kafka_yield(NULL);
rd_kafka_mem_free(NULL, NULL);
rd_kafka_list_groups(NULL, NULL, NULL, 0);
rd_kafka_group_list_destroy(NULL);
/* KafkaConsumer API */
rd_kafka_subscribe(NULL, NULL);
rd_kafka_unsubscribe(NULL);
rd_kafka_subscription(NULL, NULL);
rd_kafka_consumer_poll(NULL, 0);
rd_kafka_consumer_close(NULL);
rd_kafka_assign(NULL, NULL);
rd_kafka_assignment(NULL, NULL);
rd_kafka_commit(NULL, NULL, 0);
rd_kafka_commit_message(NULL, NULL, 0);
rd_kafka_committed(NULL, NULL, 0);
rd_kafka_position(NULL, NULL);
/* TopicPartition */
rd_kafka_topic_partition_list_new(0);
rd_kafka_topic_partition_list_destroy(NULL);
rd_kafka_topic_partition_list_add(NULL, NULL, 0);
rd_kafka_topic_partition_list_add_range(NULL, NULL, 0, 0);
rd_kafka_topic_partition_list_del(NULL, NULL, 0);
rd_kafka_topic_partition_list_del_by_idx(NULL, 0);
rd_kafka_topic_partition_list_copy(NULL);
rd_kafka_topic_partition_list_set_offset(NULL, NULL, 0, 0);
rd_kafka_topic_partition_list_find(NULL, NULL, 0);
rd_kafka_query_watermark_offsets(NULL, NULL, 0, NULL, NULL, 0);
rd_kafka_get_watermark_offsets(NULL, NULL, 0, NULL, NULL);
}
return 0;
}
