static int Producer_init (PyObject *selfobj, PyObject *args, PyObject *kwargs) { Handle *self = (Handle *)selfobj; char errstr[256]; rd_kafka_conf_t *conf; if (self->rk) { PyErr_SetString(PyExc_RuntimeError, "Producer already __init__:ialized"); return -1; } self->type = RD_KAFKA_PRODUCER; if (!(conf = common_conf_setup(RD_KAFKA_PRODUCER, self, args, kwargs))) return -1; rd_kafka_conf_set_dr_msg_cb(conf, dr_msg_cb); self->rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); if (!self->rk) { cfl_PyErr_Format(rd_kafka_last_error(), "Failed to create producer: %s", errstr); rd_kafka_conf_destroy(conf); return -1; } /* Forward log messages to poll queue */ if (self->logger) rd_kafka_set_log_queue(self->rk, NULL); return 0; }
int main (int argc, char **argv) { if (argc < 0 /* always false */) { rd_kafka_version(); rd_kafka_version_str(); rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR); rd_kafka_errno2err(EINVAL); rd_kafka_conf_new(); rd_kafka_conf_destroy(NULL); rd_kafka_conf_dup(NULL); rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0); rd_kafka_conf_set_dr_cb(NULL, NULL); rd_kafka_conf_set_error_cb(NULL, NULL); rd_kafka_conf_set_stats_cb(NULL, NULL); rd_kafka_conf_set_opaque(NULL, NULL); rd_kafka_conf_dump(NULL, NULL); rd_kafka_topic_conf_dump(NULL, NULL); rd_kafka_conf_dump_free(NULL, 0); rd_kafka_conf_properties_show(NULL); rd_kafka_topic_conf_new(); rd_kafka_topic_conf_dup(NULL); rd_kafka_topic_conf_destroy(NULL); rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0); rd_kafka_topic_conf_set_opaque(NULL, NULL); rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL); rd_kafka_topic_partition_available(NULL, 0); rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL); rd_kafka_new(0, NULL, NULL, 0); rd_kafka_destroy(NULL); rd_kafka_name(NULL); rd_kafka_topic_new(NULL, NULL, NULL); rd_kafka_topic_destroy(NULL); rd_kafka_topic_name(NULL); rd_kafka_message_destroy(NULL); rd_kafka_message_errstr(NULL); rd_kafka_consume_start(NULL, 0, 0); rd_kafka_consume_stop(NULL, 0); rd_kafka_consume(NULL, 0, 0); rd_kafka_consume_batch(NULL, 0, 0, NULL, 0); rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL); rd_kafka_offset_store(NULL, 0, 0); rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL); rd_kafka_poll(NULL, 0); rd_kafka_brokers_add(NULL, NULL); rd_kafka_set_logger(NULL, NULL); rd_kafka_set_log_level(NULL, 0); rd_kafka_log_print(NULL, 0, NULL, NULL); rd_kafka_log_syslog(NULL, 0, NULL, NULL); rd_kafka_outq_len(NULL); rd_kafka_dump(NULL, NULL); rd_kafka_thread_cnt(); rd_kafka_wait_destroyed(0); } return 0; }
static int kafka_handle(struct kafka_topic_context *ctx) /* {{{ */ { char errbuf[1024]; rd_kafka_conf_t *conf; rd_kafka_topic_conf_t *topic_conf; if (ctx->kafka != NULL && ctx->topic != NULL) return(0); if (ctx->kafka == NULL) { if ((conf = rd_kafka_conf_dup(ctx->kafka_conf)) == NULL) { ERROR("write_kafka plugin: cannot duplicate kafka config"); return(1); } if ((ctx->kafka = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errbuf, sizeof(errbuf))) == NULL) { ERROR("write_kafka plugin: cannot create kafka handle."); return 1; } rd_kafka_conf_destroy(ctx->kafka_conf); ctx->kafka_conf = NULL; INFO ("write_kafka plugin: created KAFKA handle : %s", rd_kafka_name(ctx->kafka)); #if defined(HAVE_LIBRDKAFKA_LOGGER) && !defined(HAVE_LIBRDKAFKA_LOG_CB) rd_kafka_set_logger(ctx->kafka, kafka_log); #endif } if (ctx->topic == NULL ) { if ((topic_conf = rd_kafka_topic_conf_dup(ctx->conf)) == NULL) { ERROR("write_kafka plugin: cannot duplicate kafka topic config"); return 1; } if ((ctx->topic = rd_kafka_topic_new(ctx->kafka, ctx->topic_name, topic_conf)) == NULL) { ERROR("write_kafka plugin: cannot create topic : %s\n", rd_kafka_err2str(rd_kafka_errno2err(errno))); return errno; } rd_kafka_topic_conf_destroy(ctx->conf); ctx->conf = NULL; INFO ("write_kafka plugin: handle created for topic : %s", rd_kafka_topic_name(ctx->topic)); } return(0); } /* }}} int kafka_handle */
static void kafka_topic_context_free(void *p) /* {{{ */ { struct kafka_topic_context *ctx = p; if (ctx == NULL) return; if (ctx->topic_name != NULL) sfree(ctx->topic_name); if (ctx->topic != NULL) rd_kafka_topic_destroy(ctx->topic); if (ctx->conf != NULL) rd_kafka_topic_conf_destroy(ctx->conf); if (ctx->kafka_conf != NULL) rd_kafka_conf_destroy(ctx->kafka_conf); if (ctx->kafka != NULL) rd_kafka_destroy(ctx->kafka); sfree(ctx); } /* }}} void kafka_topic_context_free */
static int Consumer_init (PyObject *selfobj, PyObject *args, PyObject *kwargs) { Handle *self = (Handle *)selfobj; char errstr[256]; rd_kafka_conf_t *conf; if (self->rk) { PyErr_SetString(PyExc_RuntimeError, "Consumer already initialized"); return -1; } self->type = RD_KAFKA_CONSUMER; if (!(conf = common_conf_setup(RD_KAFKA_CONSUMER, self, args, kwargs))) return -1; /* Exception raised by ..conf_setup() */ rd_kafka_conf_set_rebalance_cb(conf, Consumer_rebalance_cb); rd_kafka_conf_set_offset_commit_cb(conf, Consumer_offset_commit_cb); self->rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr)); if (!self->rk) { cfl_PyErr_Format(rd_kafka_last_error(), "Failed to create consumer: %s", errstr); rd_kafka_conf_destroy(conf); return -1; } /* Forward log messages to main queue which is then forwarded * to the consumer queue */ if (self->logger) rd_kafka_set_log_queue(self->rk, NULL); rd_kafka_poll_set_consumer(self->rk); self->u.Consumer.rkqu = rd_kafka_queue_get_consumer(self->rk); assert(self->u.Consumer.rkqu); return 0; }
static PyObject * RdkHandle_stop(RdkHandle *self) { /* We'll only ever get a locking error if we programmed ourselves into a * deadlock. We'd have to admit defeat, abort, and leak this RdkHandle */ if (RdkHandle_excl_lock(self)) return NULL; Py_BEGIN_ALLOW_THREADS /* avoid callbacks deadlocking */ if (self->rdk_queue_handle) { rd_kafka_queue_destroy(self->rdk_queue_handle); self->rdk_queue_handle = NULL; } if (self->rdk_topic_handle) { rd_kafka_topic_destroy(self->rdk_topic_handle); self->rdk_topic_handle = NULL; } if (self->rdk_handle) { PyObject *opaque = (PyObject *)rd_kafka_opaque(self->rdk_handle); Py_XDECREF(opaque); rd_kafka_destroy(self->rdk_handle); self->rdk_handle = NULL; } if (self->rdk_conf) { rd_kafka_conf_destroy(self->rdk_conf); self->rdk_conf = NULL; } if (self->rdk_topic_conf) { rd_kafka_topic_conf_destroy(self->rdk_topic_conf); self->rdk_topic_conf = NULL; } Py_END_ALLOW_THREADS Py_CLEAR(self->partition_ids); if (RdkHandle_unlock(self)) return NULL; Py_INCREF(Py_None); return Py_None; }
/** * @brief Interceptors must not be copied automatically by conf_dup() * unless the interceptors have added on_conf_dup(). * This behaviour makes sure an interceptor's instance * is not duplicated without the interceptor's knowledge or * assistance. */ static void do_test_conf_copy (const char *topic) { rd_kafka_conf_t *conf, *conf2; int i; rd_kafka_t *rk; TEST_SAY(_C_MAG "[ %s ]\n" _C_CLR, __FUNCTION__); memset(&msgs[0], 0, sizeof(msgs)); test_conf_init(&conf, NULL, 0); rd_kafka_conf_interceptor_add_on_new(conf, "on_new_conf_copy", on_new_producer, NULL); /* Now copy the configuration to verify that interceptors are * NOT copied. */ conf2 = conf; conf = rd_kafka_conf_dup(conf2); rd_kafka_conf_destroy(conf2); /* Create producer */ rk = test_create_handle(RD_KAFKA_PRODUCER, conf); for (i = 0 ; i < msgcnt-1 ; i++) do_test_produce(rk, topic, RD_KAFKA_PARTITION_UA, i, 0, 0); /* Wait for messages to be delivered */ test_flush(rk, -1); /* Verify acks */ for (i = 0 ; i < msgcnt ; i++) { struct msg_state *msg = &msgs[i]; msg_verify_ic_cnt(msg, "on_ack", msg->bits[_ON_ACK], 0); } rd_kafka_destroy(rk); }
int main_0004_conf (int argc, char **argv) { rd_kafka_t *rk; rd_kafka_topic_t *rkt; rd_kafka_conf_t *ignore_conf, *conf, *conf2; rd_kafka_topic_conf_t *ignore_topic_conf, *tconf, *tconf2; char errstr[512]; const char **arr_orig, **arr_dup; size_t cnt_orig, cnt_dup; int i; const char *topic; static const char *gconfs[] = { "message.max.bytes", "12345", /* int property */ "client.id", "my id", /* string property */ "debug", "topic,metadata", /* S2F property */ "topic.blacklist", "__.*", /* #778 */ "auto.offset.reset", "earliest", /* Global->Topic fallthru */ #if WITH_ZLIB "compression.codec", "gzip", /* S2I property */ #endif NULL }; static const char *tconfs[] = { "request.required.acks", "-1", /* int */ "auto.commit.enable", "false", /* bool */ "auto.offset.reset", "error", /* S2I */ "offset.store.path", "my/path", /* string */ NULL }; test_conf_init(&ignore_conf, &ignore_topic_conf, 10); rd_kafka_conf_destroy(ignore_conf); rd_kafka_topic_conf_destroy(ignore_topic_conf); topic = test_mk_topic_name("0004", 0); /* Set up a global config object */ conf = rd_kafka_conf_new(); rd_kafka_conf_set_dr_cb(conf, dr_cb); rd_kafka_conf_set_error_cb(conf, error_cb); for (i = 0 ; gconfs[i] ; i += 2) { if (rd_kafka_conf_set(conf, gconfs[i], gconfs[i+1], errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) TEST_FAIL("%s\n", errstr); } /* Set up a topic config object */ tconf = rd_kafka_topic_conf_new(); rd_kafka_topic_conf_set_partitioner_cb(tconf, partitioner); rd_kafka_topic_conf_set_opaque(tconf, (void *)0xbeef); for (i = 0 ; tconfs[i] ; i += 2) { if (rd_kafka_topic_conf_set(tconf, tconfs[i], tconfs[i+1], errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) TEST_FAIL("%s\n", errstr); } /* Verify global config */ arr_orig = rd_kafka_conf_dump(conf, &cnt_orig); conf_verify(__LINE__, arr_orig, cnt_orig, gconfs); /* Verify copied global config */ conf2 = rd_kafka_conf_dup(conf); arr_dup = rd_kafka_conf_dump(conf2, &cnt_dup); conf_verify(__LINE__, arr_dup, cnt_dup, gconfs); conf_cmp("global", arr_orig, cnt_orig, arr_dup, cnt_dup); rd_kafka_conf_dump_free(arr_orig, cnt_orig); rd_kafka_conf_dump_free(arr_dup, cnt_dup); /* Verify topic config */ arr_orig = rd_kafka_topic_conf_dump(tconf, &cnt_orig); conf_verify(__LINE__, arr_orig, cnt_orig, tconfs); /* Verify copied topic config */ tconf2 = rd_kafka_topic_conf_dup(tconf); arr_dup = rd_kafka_topic_conf_dump(tconf2, &cnt_dup); conf_verify(__LINE__, arr_dup, cnt_dup, tconfs); conf_cmp("topic", arr_orig, cnt_orig, arr_dup, cnt_dup); rd_kafka_conf_dump_free(arr_orig, cnt_orig); rd_kafka_conf_dump_free(arr_dup, cnt_dup); /* * Create kafka instances using original and copied confs */ /* original */ rk = test_create_handle(RD_KAFKA_PRODUCER, conf); rkt = rd_kafka_topic_new(rk, topic, tconf); if (!rkt) TEST_FAIL("Failed to create topic: %s\n", rd_strerror(errno)); rd_kafka_topic_destroy(rkt); rd_kafka_destroy(rk); /* copied */ rk = test_create_handle(RD_KAFKA_PRODUCER, conf2); rkt = rd_kafka_topic_new(rk, topic, tconf2); if (!rkt) TEST_FAIL("Failed to create topic: %s\n", rd_strerror(errno)); rd_kafka_topic_destroy(rkt); rd_kafka_destroy(rk); /* Incremental S2F property. * NOTE: The order of fields returned in get() is hardcoded here. */ { static const char *s2fs[] = { "generic,broker,queue,cgrp", "generic,broker,queue,cgrp", "-broker,+queue,topic", "generic,topic,queue,cgrp", "-all,security,-fetch,+metadata", "metadata,security", NULL }; TEST_SAY("Incremental S2F tests\n"); conf = rd_kafka_conf_new(); for (i = 0 ; s2fs[i] ; i += 2) { const char *val; TEST_SAY(" Set: %s\n", s2fs[i]); test_conf_set(conf, "debug", s2fs[i]); val = test_conf_get(conf, "debug"); TEST_SAY(" Now: %s\n", val); if (strcmp(val, s2fs[i+1])) TEST_FAIL_LATER("\n" "Expected: %s\n" " Got: %s", s2fs[i+1], val); } rd_kafka_conf_destroy(conf); } /* Canonical int values, aliases, s2i-verified strings */ { static const struct { const char *prop; const char *val; const char *exp; int is_global; } props[] = { { "request.required.acks", "0", "0" }, { "request.required.acks", "-1", "-1" }, { "request.required.acks", "1", "1" }, { "acks", "3", "3" }, /* alias test */ { "request.required.acks", "393", "393" }, { "request.required.acks", "bad", NULL }, { "request.required.acks", "all", "-1" }, { "request.required.acks", "all", "-1", 1/*fallthru*/ }, { "acks", "0", "0" }, /* alias test */ #if WITH_SASL { "sasl.mechanisms", "GSSAPI", "GSSAPI", 1 }, { "sasl.mechanisms", "PLAIN", "PLAIN", 1 }, { "sasl.mechanisms", "GSSAPI,PLAIN", NULL, 1 }, { "sasl.mechanisms", "", NULL, 1 }, #endif { NULL } }; TEST_SAY("Canonical tests\n"); tconf = rd_kafka_topic_conf_new(); conf = rd_kafka_conf_new(); for (i = 0 ; props[i].prop ; i++) { char dest[64]; size_t destsz; rd_kafka_conf_res_t res; TEST_SAY(" Set: %s=%s expect %s (%s)\n", props[i].prop, props[i].val, props[i].exp, props[i].is_global ? "global":"topic"); /* Set value */ if (props[i].is_global) res = rd_kafka_conf_set(conf, props[i].prop, props[i].val, errstr, sizeof(errstr)); else res = rd_kafka_topic_conf_set(tconf, props[i].prop, props[i].val, errstr, sizeof(errstr)); if ((res == RD_KAFKA_CONF_OK ? 1:0) != (props[i].exp ? 1:0)) TEST_FAIL("Expected %s, got %s", props[i].exp ? "success" : "failure", (res == RD_KAFKA_CONF_OK ? "OK" : (res == RD_KAFKA_CONF_INVALID ? "INVALID" : "UNKNOWN"))); if (!props[i].exp) continue; /* Get value and compare to expected result */ destsz = sizeof(dest); if (props[i].is_global) res = rd_kafka_conf_get(conf, props[i].prop, dest, &destsz); else res = rd_kafka_topic_conf_get(tconf, props[i].prop, dest, &destsz); TEST_ASSERT(res == RD_KAFKA_CONF_OK, ".._conf_get(%s) returned %d", props[i].prop, res); TEST_ASSERT(!strcmp(props[i].exp, dest), "Expected \"%s\", got \"%s\"", props[i].exp, dest); } rd_kafka_topic_conf_destroy(tconf); rd_kafka_conf_destroy(conf); } return 0; }
int main (int argc, char **argv) { rd_kafka_t *rk; rd_kafka_topic_t *rkt; rd_kafka_conf_t *ignore_conf, *conf, *conf2; rd_kafka_topic_conf_t *ignore_topic_conf, *tconf, *tconf2; char errstr[512]; const char **arr_orig, **arr_dup; size_t cnt_orig, cnt_dup; int i; const char *topic; static const char *gconfs[] = { "message.max.bytes", "12345", /* int property */ "client.id", "my id", /* string property */ "debug", "topic,metadata", /* S2F property */ "compression.codec", "gzip", /* S2I property */ NULL }; static const char *tconfs[] = { "request.required.acks", "-1", /* int */ "auto.commit.enable", "false", /* bool */ "auto.offset.reset", "error", /* S2I */ "offset.store.path", "my/path", /* string */ NULL }; test_conf_init(&ignore_conf, &ignore_topic_conf, 10); rd_kafka_conf_destroy(ignore_conf); rd_kafka_topic_conf_destroy(ignore_topic_conf); topic = test_mk_topic_name("generic", 0); /* Set up a global config object */ conf = rd_kafka_conf_new(); rd_kafka_conf_set_dr_cb(conf, dr_cb); rd_kafka_conf_set_error_cb(conf, error_cb); for (i = 0 ; gconfs[i] ; i += 2) { if (rd_kafka_conf_set(conf, gconfs[i], gconfs[i+1], errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) TEST_FAIL("%s\n", errstr); } /* Set up a topic config object */ tconf = rd_kafka_topic_conf_new(); rd_kafka_topic_conf_set_partitioner_cb(tconf, partitioner); rd_kafka_topic_conf_set_opaque(tconf, (void *)0xbeef); for (i = 0 ; tconfs[i] ; i += 2) { if (rd_kafka_topic_conf_set(tconf, tconfs[i], tconfs[i+1], errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) TEST_FAIL("%s\n", errstr); } /* Verify global config */ arr_orig = rd_kafka_conf_dump(conf, &cnt_orig); conf_verify(__LINE__, arr_orig, cnt_orig, gconfs); /* Verify copied global config */ conf2 = rd_kafka_conf_dup(conf); arr_dup = rd_kafka_conf_dump(conf2, &cnt_dup); conf_verify(__LINE__, arr_dup, cnt_dup, gconfs); conf_cmp("global", arr_orig, cnt_orig, arr_dup, cnt_dup); rd_kafka_conf_dump_free(arr_orig, cnt_orig); rd_kafka_conf_dump_free(arr_dup, cnt_dup); /* Verify topic config */ arr_orig = rd_kafka_topic_conf_dump(tconf, &cnt_orig); conf_verify(__LINE__, arr_orig, cnt_orig, tconfs); /* Verify copied topic config */ tconf2 = rd_kafka_topic_conf_dup(tconf); arr_dup = rd_kafka_topic_conf_dump(tconf2, &cnt_dup); conf_verify(__LINE__, arr_dup, cnt_dup, tconfs); conf_cmp("topic", arr_orig, cnt_orig, arr_dup, cnt_dup); rd_kafka_conf_dump_free(arr_orig, cnt_orig); rd_kafka_conf_dump_free(arr_dup, cnt_dup); /* * Create kafka instances using original and copied confs */ /* original */ rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); if (!rk) TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr); rkt = rd_kafka_topic_new(rk, topic, tconf); if (!rkt) TEST_FAIL("Failed to create topic: %s\n", strerror(errno)); rd_kafka_topic_destroy(rkt); rd_kafka_destroy(rk); /* copied */ rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf2, errstr, sizeof(errstr)); if (!rk) TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr); rkt = rd_kafka_topic_new(rk, topic, tconf2); if (!rkt) TEST_FAIL("Failed to create topic: %s\n", strerror(errno)); rd_kafka_topic_destroy(rkt); rd_kafka_destroy(rk); /* Wait for everything to be cleaned up since broker destroys are * handled in its own thread. */ test_wait_exit(2); /* If we havent failed at this point then * there were no threads leaked */ return 0; }
static int kafka_config(oconfig_item_t *ci) /* {{{ */ { int i; oconfig_item_t *child; rd_kafka_conf_t *conf; rd_kafka_conf_t *cloned; rd_kafka_conf_res_t ret; char errbuf[1024]; if ((conf = rd_kafka_conf_new()) == NULL) { WARNING("cannot allocate kafka configuration."); return -1; } for (i = 0; i < ci->children_num; i++) { child = &ci->children[i]; if (strcasecmp("Topic", child->key) == 0) { if ((cloned = rd_kafka_conf_dup(conf)) == NULL) { WARNING("write_kafka plugin: cannot allocate memory for kafka config"); goto errout; } kafka_config_topic (cloned, child); } else if (strcasecmp(child->key, "Property") == 0) { char *key = NULL; char *val = NULL; if (child->values_num != 2) { WARNING("kafka properties need both a key and a value."); goto errout; } if (child->values[0].type != OCONFIG_TYPE_STRING || child->values[1].type != OCONFIG_TYPE_STRING) { WARNING("kafka properties needs string arguments."); goto errout; } if ((key = strdup(child->values[0].value.string)) == NULL) { WARNING("cannot allocate memory for attribute key."); goto errout; } if ((val = strdup(child->values[1].value.string)) == NULL) { WARNING("cannot allocate memory for attribute value."); goto errout; } ret = rd_kafka_conf_set(conf, key, val, errbuf, sizeof(errbuf)); if (ret != RD_KAFKA_CONF_OK) { WARNING("cannot set kafka property %s to %s: %s", key, val, errbuf); goto errout; } sfree(key); sfree(val); } else { WARNING ("write_kafka plugin: Ignoring unknown " "configuration option \"%s\" at top level.", child->key); } } if (conf != NULL) rd_kafka_conf_destroy(conf); return (0); errout: if (conf != NULL) rd_kafka_conf_destroy(conf); return -1; } /* }}} int kafka_config */
static void kafka_config_topic(rd_kafka_conf_t *conf, oconfig_item_t *ci) /* {{{ */ { int status; int i; struct kafka_topic_context *tctx; char *key; char *val; char callback_name[DATA_MAX_NAME_LEN]; char errbuf[1024]; user_data_t ud; oconfig_item_t *child; rd_kafka_conf_res_t ret; if ((tctx = calloc(1, sizeof (*tctx))) == NULL) { ERROR ("write_kafka plugin: calloc failed."); return; } tctx->escape_char = '.'; tctx->store_rates = 1; rd_kafka_conf_set_log_cb(conf, kafka_log); if ((tctx->kafka = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errbuf, sizeof(errbuf))) == NULL) { sfree(tctx); ERROR("write_kafka plugin: cannot create kafka handle."); return; } conf = NULL; if ((tctx->conf = rd_kafka_topic_conf_new()) == NULL) { rd_kafka_destroy(tctx->kafka); sfree(tctx); ERROR ("write_kafka plugin: cannot create topic configuration."); return; } if (ci->values_num != 1) { WARNING("kafka topic name needed."); goto errout; } if (ci->values[0].type != OCONFIG_TYPE_STRING) { WARNING("kafka topic needs a string argument."); goto errout; } if ((tctx->topic_name = strdup(ci->values[0].value.string)) == NULL) { ERROR("write_kafka plugin: cannot copy topic name."); goto errout; } for (i = 0; i < ci->children_num; i++) { /* * The code here could be simplified but makes room * for easy adding of new options later on. */ child = &ci->children[i]; status = 0; if (strcasecmp ("Property", child->key) == 0) { if (child->values_num != 2) { WARNING("kafka properties need both a key and a value."); goto errout; } if (child->values[0].type != OCONFIG_TYPE_STRING || child->values[1].type != OCONFIG_TYPE_STRING) { WARNING("kafka properties needs string arguments."); goto errout; } key = child->values[0].value.string; val = child->values[0].value.string; ret = rd_kafka_topic_conf_set(tctx->conf,key, val, errbuf, sizeof(errbuf)); if (ret != RD_KAFKA_CONF_OK) { WARNING("cannot set kafka topic property %s to %s: %s.", key, val, errbuf); goto errout; } } else if (strcasecmp ("Key", child->key) == 0) { char *tmp_buf = NULL; status = cf_util_get_string(child, &tmp_buf); if (status != 0) { WARNING("write_kafka plugin: invalid key supplied"); break; } if (strcasecmp(tmp_buf, "Random") != 0) { tctx->has_key = 1; tctx->key = crc32_buffer((u_char *)tmp_buf, strlen(tmp_buf)); } sfree(tmp_buf); } else if (strcasecmp ("Format", child->key) == 0) { status = cf_util_get_string(child, &key); if (status != 0) goto errout; assert(key != NULL); if (strcasecmp(key, "Command") == 0) { tctx->format = KAFKA_FORMAT_COMMAND; } else if (strcasecmp(key, "Graphite") == 0) { tctx->format = KAFKA_FORMAT_GRAPHITE; } else if (strcasecmp(key, "Json") == 0) { tctx->format = KAFKA_FORMAT_JSON; } else { WARNING ("write_kafka plugin: Invalid format string: %s", key); } sfree(key); } else if (strcasecmp ("StoreRates", child->key) == 0) { status = cf_util_get_boolean (child, &tctx->store_rates); (void) cf_util_get_flag (child, &tctx->graphite_flags, GRAPHITE_STORE_RATES); } else if (strcasecmp ("GraphiteSeparateInstances", child->key) == 0) { status = cf_util_get_flag (child, &tctx->graphite_flags, GRAPHITE_SEPARATE_INSTANCES); } else if (strcasecmp ("GraphiteAlwaysAppendDS", child->key) == 0) { status = cf_util_get_flag (child, &tctx->graphite_flags, GRAPHITE_ALWAYS_APPEND_DS); } else if (strcasecmp ("GraphitePrefix", child->key) == 0) { status = cf_util_get_string (child, &tctx->prefix); } else if (strcasecmp ("GraphitePostfix", child->key) == 0) { status = cf_util_get_string (child, &tctx->postfix); } else if (strcasecmp ("GraphiteEscapeChar", child->key) == 0) { char *tmp_buff = NULL; status = cf_util_get_string (child, &tmp_buff); if (strlen (tmp_buff) > 1) WARNING ("write_kafka plugin: The option \"GraphiteEscapeChar\" handles " "only one character. Others will be ignored."); tctx->escape_char = tmp_buff[0]; sfree (tmp_buff); } else { WARNING ("write_kafka plugin: Invalid directive: %s.", child->key); } if (status != 0) break; } rd_kafka_topic_conf_set_partitioner_cb(tctx->conf, kafka_partition); rd_kafka_topic_conf_set_opaque(tctx->conf, tctx); if ((tctx->topic = rd_kafka_topic_new(tctx->kafka, tctx->topic_name, tctx->conf)) == NULL) { ERROR("write_kafka plugin: cannot create topic."); goto errout; } tctx->conf = NULL; ssnprintf(callback_name, sizeof(callback_name), "write_kafka/%s", tctx->topic_name); ud.data = tctx; ud.free_func = kafka_topic_context_free; status = plugin_register_write (callback_name, kafka_write, &ud); if (status != 0) { WARNING ("write_kafka plugin: plugin_register_write (\"%s\") " "failed with status %i.", callback_name, status); goto errout; } return; errout: if (conf != NULL) rd_kafka_conf_destroy(conf); if (tctx->kafka != NULL) rd_kafka_destroy(tctx->kafka); if (tctx->topic != NULL) rd_kafka_topic_destroy(tctx->topic); if (tctx->topic_name != NULL) free(tctx->topic_name); if (tctx->conf != NULL) rd_kafka_topic_conf_destroy(tctx->conf); sfree(tctx); } /* }}} int kafka_config_topic */
rd_kafka_t *kafka_get_connection(kafka_connection_params params, const char *brokers) { rd_kafka_t *r = NULL; char errstr[512]; rd_kafka_conf_t *conf = rd_kafka_conf_new(); //set error callback rd_kafka_conf_set_error_cb(conf, kafka_err_cb); if (params.type == RD_KAFKA_CONSUMER) { if (params.queue_buffer) rd_kafka_conf_set(conf, "queued.min.messages", params.queue_buffer, NULL, 0); r = rd_kafka_new(params.type, conf, errstr, sizeof errstr); if (!r) { if (params.log_level) { openlog("phpkafka", 0, LOG_USER); syslog(LOG_ERR, "Failed to connect to kafka: %s", errstr); } //destroy config, no connection to use it... rd_kafka_conf_destroy(conf); return NULL; } if (!rd_kafka_brokers_add(r, brokers)) { if (params.log_level) { openlog("phpkafka", 0, LOG_USER); syslog(LOG_ERR, "Failed to connect to brokers %s", brokers); } rd_kafka_destroy(r); return NULL; } return r; } if (params.compression) { rd_kafka_conf_res_t result = rd_kafka_conf_set( conf, "compression.codec",params.compression, errstr, sizeof errstr ); if (result != RD_KAFKA_CONF_OK) { if (params.log_level) { openlog("phpkafka", 0, LOG_USER); syslog(LOG_ALERT, "Failed to set compression %s: %s", params.compression, errstr); } rd_kafka_conf_destroy(conf); return NULL; } } if (params.retry_count) { rd_kafka_conf_res_t result = rd_kafka_conf_set( conf, "message.send.max.retries",params.retry_count, errstr, sizeof errstr ); if (result != RD_KAFKA_CONF_OK) { if (params.log_level) { openlog("phpkafka", 0, LOG_USER); syslog(LOG_ALERT, "Failed to set compression %s: %s", params.compression, errstr); } rd_kafka_conf_destroy(conf); return NULL; } } if (params.retry_interval) { rd_kafka_conf_res_t result = rd_kafka_conf_set( conf, "retry.backoff.ms",params.retry_interval, errstr, sizeof errstr ); if (result != RD_KAFKA_CONF_OK) { if (params.log_level) { openlog("phpkafka", 0, LOG_USER); syslog(LOG_ALERT, "Failed to set compression %s: %s", params.compression, errstr); } rd_kafka_conf_destroy(conf); return NULL; } } if (params.reporting == 1) rd_kafka_conf_set_dr_cb(conf, kafka_produce_cb_simple); else if (params.reporting == 2) rd_kafka_conf_set_dr_msg_cb(conf, kafka_produce_detailed_cb); r = rd_kafka_new(params.type, conf, errstr, sizeof errstr); if (!r) { if (params.log_level) { openlog("phpkafka", 0, LOG_USER); syslog(LOG_ERR, "Failed to connect to kafka: %s", errstr); } //destroy config, no connection to use it... rd_kafka_conf_destroy(conf); return NULL; } if (!rd_kafka_brokers_add(r, brokers)) { if (params.log_level) { openlog("phpkafka", 0, LOG_USER); syslog(LOG_ERR, "Failed to connect to brokers %s", brokers); } rd_kafka_destroy(r); return NULL; } return r; }
int main_0006_symbols (int argc, char **argv) { if (argc < 0 /* always false */) { rd_kafka_version(); rd_kafka_version_str(); rd_kafka_get_debug_contexts(); rd_kafka_get_err_descs(NULL, NULL); rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR); rd_kafka_err2name(RD_KAFKA_RESP_ERR_NO_ERROR); rd_kafka_errno2err(EINVAL); rd_kafka_errno(); rd_kafka_last_error(); rd_kafka_conf_new(); rd_kafka_conf_destroy(NULL); rd_kafka_conf_dup(NULL); rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0); rd_kafka_conf_set_dr_cb(NULL, NULL); rd_kafka_conf_set_dr_msg_cb(NULL, NULL); rd_kafka_conf_set_error_cb(NULL, NULL); rd_kafka_conf_set_stats_cb(NULL, NULL); rd_kafka_conf_set_log_cb(NULL, NULL); rd_kafka_conf_set_socket_cb(NULL, NULL); rd_kafka_conf_set_rebalance_cb(NULL, NULL); rd_kafka_conf_set_offset_commit_cb(NULL, NULL); rd_kafka_conf_set_throttle_cb(NULL, NULL); rd_kafka_conf_set_default_topic_conf(NULL, NULL); rd_kafka_conf_get(NULL, NULL, NULL, NULL); #ifndef _MSC_VER rd_kafka_conf_set_open_cb(NULL, NULL); #endif rd_kafka_conf_set_opaque(NULL, NULL); rd_kafka_opaque(NULL); rd_kafka_conf_dump(NULL, NULL); rd_kafka_topic_conf_dump(NULL, NULL); rd_kafka_conf_dump_free(NULL, 0); rd_kafka_conf_properties_show(NULL); rd_kafka_topic_conf_new(); rd_kafka_topic_conf_dup(NULL); rd_kafka_topic_conf_destroy(NULL); rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0); rd_kafka_topic_conf_set_opaque(NULL, NULL); rd_kafka_topic_conf_get(NULL, NULL, NULL, NULL); rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL); rd_kafka_topic_partition_available(NULL, 0); rd_kafka_topic_opaque(NULL); rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL); rd_kafka_msg_partitioner_consistent(NULL, NULL, 0, 0, NULL, NULL); rd_kafka_msg_partitioner_consistent_random(NULL, NULL, 0, 0, NULL, NULL); rd_kafka_new(0, NULL, NULL, 0); rd_kafka_destroy(NULL); rd_kafka_name(NULL); rd_kafka_memberid(NULL); rd_kafka_topic_new(NULL, NULL, NULL); rd_kafka_topic_destroy(NULL); rd_kafka_topic_name(NULL); rd_kafka_message_destroy(NULL); rd_kafka_message_errstr(NULL); rd_kafka_message_timestamp(NULL, NULL); rd_kafka_consume_start(NULL, 0, 0); rd_kafka_consume_stop(NULL, 0); rd_kafka_consume(NULL, 0, 0); rd_kafka_consume_batch(NULL, 0, 0, NULL, 0); rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL); rd_kafka_offset_store(NULL, 0, 0); rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL); rd_kafka_produce_batch(NULL, 0, 0, NULL, 0); rd_kafka_poll(NULL, 0); rd_kafka_brokers_add(NULL, NULL); /* DEPRECATED: rd_kafka_set_logger(NULL, NULL); */ rd_kafka_set_log_level(NULL, 0); rd_kafka_log_print(NULL, 0, NULL, NULL); #ifndef _MSC_VER rd_kafka_log_syslog(NULL, 0, NULL, NULL); #endif rd_kafka_outq_len(NULL); rd_kafka_dump(NULL, NULL); rd_kafka_thread_cnt(); rd_kafka_wait_destroyed(0); rd_kafka_metadata(NULL, 0, NULL, NULL, 0); rd_kafka_metadata_destroy(NULL); rd_kafka_queue_destroy(NULL); rd_kafka_consume_start_queue(NULL, 0, 0, NULL); rd_kafka_consume_queue(NULL, 0); rd_kafka_consume_batch_queue(NULL, 0, NULL, 0); rd_kafka_consume_callback_queue(NULL, 0, NULL, NULL); rd_kafka_seek(NULL, 0, 0, 0); rd_kafka_yield(NULL); rd_kafka_mem_free(NULL, NULL); rd_kafka_list_groups(NULL, NULL, NULL, 0); rd_kafka_group_list_destroy(NULL); /* KafkaConsumer API */ rd_kafka_subscribe(NULL, NULL); rd_kafka_unsubscribe(NULL); rd_kafka_subscription(NULL, NULL); rd_kafka_consumer_poll(NULL, 0); rd_kafka_consumer_close(NULL); rd_kafka_assign(NULL, NULL); rd_kafka_assignment(NULL, NULL); rd_kafka_commit(NULL, NULL, 0); rd_kafka_commit_message(NULL, NULL, 0); rd_kafka_committed(NULL, NULL, 0); rd_kafka_position(NULL, NULL); /* TopicPartition */ rd_kafka_topic_partition_list_new(0); rd_kafka_topic_partition_list_destroy(NULL); rd_kafka_topic_partition_list_add(NULL, NULL, 0); rd_kafka_topic_partition_list_add_range(NULL, NULL, 0, 0); rd_kafka_topic_partition_list_del(NULL, NULL, 0); rd_kafka_topic_partition_list_del_by_idx(NULL, 0); rd_kafka_topic_partition_list_copy(NULL); rd_kafka_topic_partition_list_set_offset(NULL, NULL, 0, 0); rd_kafka_topic_partition_list_find(NULL, NULL, 0); rd_kafka_query_watermark_offsets(NULL, NULL, 0, NULL, NULL, 0); rd_kafka_get_watermark_offsets(NULL, NULL, 0, NULL, NULL); } return 0; }