Example #1
0
int main (int argc, char **argv) {

        if (argc < 0 /* always false */) {
                rd_kafka_version();
                rd_kafka_version_str();
                rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR);
                rd_kafka_errno2err(EINVAL);
                rd_kafka_conf_new();
                rd_kafka_conf_destroy(NULL);
                rd_kafka_conf_dup(NULL);
                rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0);
                rd_kafka_conf_set_dr_cb(NULL, NULL);
                rd_kafka_conf_set_error_cb(NULL, NULL);
                rd_kafka_conf_set_stats_cb(NULL, NULL);
                rd_kafka_conf_set_opaque(NULL, NULL);
                rd_kafka_conf_dump(NULL, NULL);
                rd_kafka_topic_conf_dump(NULL, NULL);
                rd_kafka_conf_dump_free(NULL, 0);
                rd_kafka_conf_properties_show(NULL);
                rd_kafka_topic_conf_new();
                rd_kafka_topic_conf_dup(NULL);
                rd_kafka_topic_conf_destroy(NULL);
                rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0);
                rd_kafka_topic_conf_set_opaque(NULL, NULL);
                rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL);
                rd_kafka_topic_partition_available(NULL, 0);
                rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL);
                rd_kafka_new(0, NULL, NULL, 0);
                rd_kafka_destroy(NULL);
                rd_kafka_name(NULL);
                rd_kafka_topic_new(NULL, NULL, NULL);
                rd_kafka_topic_destroy(NULL);
                rd_kafka_topic_name(NULL);
                rd_kafka_message_destroy(NULL);
                rd_kafka_message_errstr(NULL);
                rd_kafka_consume_start(NULL, 0, 0);
                rd_kafka_consume_stop(NULL, 0);
                rd_kafka_consume(NULL, 0, 0);
                rd_kafka_consume_batch(NULL, 0, 0, NULL, 0);
                rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL);
                rd_kafka_offset_store(NULL, 0, 0);
                rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL);
                rd_kafka_poll(NULL, 0);
                rd_kafka_brokers_add(NULL, NULL);
                rd_kafka_set_logger(NULL, NULL);
                rd_kafka_set_log_level(NULL, 0);
                rd_kafka_log_print(NULL, 0, NULL, NULL);
                rd_kafka_log_syslog(NULL, 0, NULL, NULL);
                rd_kafka_outq_len(NULL);
                rd_kafka_dump(NULL, NULL);
                rd_kafka_thread_cnt();
                rd_kafka_wait_destroyed(0);
        }


	return 0;
}
Example #2
0
/**
 *  producer_init_kafka
 *
 *  Initialize the producer instance, setting up the Kafka topic and context.
 *
 *  @param  self    VALUE Instance of the Producer Ruby object
 *  @param  config  HermannInstanceConfig*  the instance configuration associated with this producer.
 */
void producer_init_kafka(VALUE self, HermannInstanceConfig* config) {

	TRACER("initing (%p)\n", config);

	config->quiet = !isatty(STDIN_FILENO);

	/* Kafka configuration */
	config->conf = rd_kafka_conf_new();


	/* Add our `self` to the opaque pointer for error and logging callbacks
	 */
	rd_kafka_conf_set_opaque(config->conf, (void*)config);
	rd_kafka_conf_set_error_cb(config->conf, producer_error_callback);

	/* Topic configuration */
	config->topic_conf = rd_kafka_topic_conf_new();

	/* Set up a message delivery report callback.
	 * It will be called once for each message, either on successful
	 * delivery to broker, or upon failure to deliver to broker. */
	rd_kafka_conf_set_dr_msg_cb(config->conf, msg_delivered);

	/* Create Kafka handle */
	if (!(config->rk = rd_kafka_new(RD_KAFKA_PRODUCER,
									config->conf,
									config->errstr,
									sizeof(config->errstr)))) {
		/* TODO: Use proper logger */
		fprintf(stderr,
		"%% Failed to create new producer: %s\n", config->errstr);
		rb_raise(rb_eRuntimeError, "%% Failed to create new producer: %s\n", config->errstr);
	}

	/* Set logger */
	rd_kafka_set_logger(config->rk, logger);
	rd_kafka_set_log_level(config->rk, LOG_DEBUG);

	if (rd_kafka_brokers_add(config->rk, config->brokers) == 0) {
		/* TODO: Use proper logger */
		fprintf(stderr, "%% No valid brokers specified\n");
		rb_raise(rb_eRuntimeError, "No valid brokers specified");
		return;
	}

	/* Create topic */
	config->rkt = rd_kafka_topic_new(config->rk, config->topic, config->topic_conf);

	/* Set the partitioner callback */
	rd_kafka_topic_conf_set_partitioner_cb( config->topic_conf, producer_partitioner_callback);

	/* We're now initialized */
	config->isInitialized = 1;

	TRACER("completed kafka init\n");
}
Example #3
0
/** Creates a new topic handler using global configuration
    @param topic_name Topic name
    @param partitioner Partitioner function
    @return New topic handler */
rd_kafka_topic_t *new_rkt_global_config(const char *topic_name,
	rb_rd_kafka_partitioner_t partitioner,char *err,size_t errsize) {
	rd_kafka_topic_conf_t *template_config = global_config.kafka_topic_conf;
	rd_kafka_topic_conf_t *my_rkt_conf
		= rd_kafka_topic_conf_dup(template_config);

	if(NULL == my_rkt_conf) {
		rdlog(LOG_ERR,"Couldn't topic_conf_dup in topic %s",topic_name);
		return NULL;
	}

	rd_kafka_topic_conf_set_partitioner_cb(my_rkt_conf, partitioner);

	rd_kafka_topic_t *ret = rd_kafka_topic_new(global_config.rk, topic_name,
		my_rkt_conf);
	if (NULL == ret) {
		strerror_r(errno, err, errsize);
		rd_kafka_topic_conf_destroy(my_rkt_conf);
	}

	return ret;
}
Example #4
0
int main_0004_conf (int argc, char **argv) {
	rd_kafka_t *rk;
	rd_kafka_topic_t *rkt;
	rd_kafka_conf_t *ignore_conf, *conf, *conf2;
	rd_kafka_topic_conf_t *ignore_topic_conf, *tconf, *tconf2;
	char errstr[512];
	const char **arr_orig, **arr_dup;
	size_t cnt_orig, cnt_dup;
	int i;
        const char *topic;
	static const char *gconfs[] = {
		"message.max.bytes", "12345", /* int property */
		"client.id", "my id", /* string property */
		"debug", "topic,metadata", /* S2F property */
		"topic.blacklist", "__.*", /* #778 */
                "auto.offset.reset", "earliest", /* Global->Topic fallthru */
#if WITH_ZLIB
		"compression.codec", "gzip", /* S2I property */
#endif
		NULL
	};
	static const char *tconfs[] = {
		"request.required.acks", "-1", /* int */
		"auto.commit.enable", "false", /* bool */
		"auto.offset.reset", "error",  /* S2I */
		"offset.store.path", "my/path", /* string */
		NULL
	};

	test_conf_init(&ignore_conf, &ignore_topic_conf, 10);
	rd_kafka_conf_destroy(ignore_conf);
	rd_kafka_topic_conf_destroy(ignore_topic_conf);

        topic = test_mk_topic_name("0004", 0);

	/* Set up a global config object */
	conf = rd_kafka_conf_new();

	rd_kafka_conf_set_dr_cb(conf, dr_cb);
	rd_kafka_conf_set_error_cb(conf, error_cb);

	for (i = 0 ; gconfs[i] ; i += 2) {
		if (rd_kafka_conf_set(conf, gconfs[i], gconfs[i+1],
				      errstr, sizeof(errstr)) !=
		    RD_KAFKA_CONF_OK)
			TEST_FAIL("%s\n", errstr);
	}

	/* Set up a topic config object */
	tconf = rd_kafka_topic_conf_new();

	rd_kafka_topic_conf_set_partitioner_cb(tconf, partitioner);
	rd_kafka_topic_conf_set_opaque(tconf, (void *)0xbeef);

	for (i = 0 ; tconfs[i] ; i += 2) {
		if (rd_kafka_topic_conf_set(tconf, tconfs[i], tconfs[i+1],
				      errstr, sizeof(errstr)) !=
		    RD_KAFKA_CONF_OK)
			TEST_FAIL("%s\n", errstr);
	}


	/* Verify global config */
	arr_orig = rd_kafka_conf_dump(conf, &cnt_orig);
	conf_verify(__LINE__, arr_orig, cnt_orig, gconfs);

	/* Verify copied global config */
	conf2 = rd_kafka_conf_dup(conf);
	arr_dup = rd_kafka_conf_dump(conf2, &cnt_dup);
	conf_verify(__LINE__, arr_dup, cnt_dup, gconfs);
	conf_cmp("global", arr_orig, cnt_orig, arr_dup, cnt_dup);
	rd_kafka_conf_dump_free(arr_orig, cnt_orig);
	rd_kafka_conf_dump_free(arr_dup, cnt_dup);

	/* Verify topic config */
	arr_orig = rd_kafka_topic_conf_dump(tconf, &cnt_orig);
	conf_verify(__LINE__, arr_orig, cnt_orig, tconfs);

	/* Verify copied topic config */
	tconf2 = rd_kafka_topic_conf_dup(tconf);
	arr_dup = rd_kafka_topic_conf_dump(tconf2, &cnt_dup);
	conf_verify(__LINE__, arr_dup, cnt_dup, tconfs);
	conf_cmp("topic", arr_orig, cnt_orig, arr_dup, cnt_dup);
	rd_kafka_conf_dump_free(arr_orig, cnt_orig);
	rd_kafka_conf_dump_free(arr_dup, cnt_dup);


	/*
	 * Create kafka instances using original and copied confs
	 */

	/* original */
	rk = test_create_handle(RD_KAFKA_PRODUCER, conf);

	rkt = rd_kafka_topic_new(rk, topic, tconf);
	if (!rkt)
		TEST_FAIL("Failed to create topic: %s\n",
			  rd_strerror(errno));

	rd_kafka_topic_destroy(rkt);
	rd_kafka_destroy(rk);

	/* copied */
	rk = test_create_handle(RD_KAFKA_PRODUCER, conf2);

	rkt = rd_kafka_topic_new(rk, topic, tconf2);
	if (!rkt)
		TEST_FAIL("Failed to create topic: %s\n",
			  rd_strerror(errno));

	rd_kafka_topic_destroy(rkt);
	rd_kafka_destroy(rk);


	/* Incremental S2F property.
	 * NOTE: The order of fields returned in get() is hardcoded here. */
	{
		static const char *s2fs[] = {
			"generic,broker,queue,cgrp",
			"generic,broker,queue,cgrp",

			"-broker,+queue,topic",
			"generic,topic,queue,cgrp",

			"-all,security,-fetch,+metadata",
			"metadata,security",

			NULL
		};

		TEST_SAY("Incremental S2F tests\n");
		conf = rd_kafka_conf_new();

		for (i = 0 ; s2fs[i] ; i += 2) {
			const char *val;

			TEST_SAY("  Set: %s\n", s2fs[i]);
			test_conf_set(conf, "debug", s2fs[i]);
			val = test_conf_get(conf, "debug");
			TEST_SAY("  Now: %s\n", val);

			if (strcmp(val, s2fs[i+1]))
				TEST_FAIL_LATER("\n"
						"Expected: %s\n"
						"     Got: %s",
						s2fs[i+1], val);
		}
		rd_kafka_conf_destroy(conf);
	}

	/* Canonical int values, aliases, s2i-verified strings */
	{
		static const struct {
			const char *prop;
			const char *val;
			const char *exp;
			int is_global;
		} props[] = {
			{ "request.required.acks", "0", "0" },
			{ "request.required.acks", "-1", "-1" },
			{ "request.required.acks", "1", "1" },
			{ "acks", "3", "3" }, /* alias test */
			{ "request.required.acks", "393", "393" },
			{ "request.required.acks", "bad", NULL },
			{ "request.required.acks", "all", "-1" },
                        { "request.required.acks", "all", "-1", 1/*fallthru*/ },
			{ "acks", "0", "0" }, /* alias test */
#if WITH_SASL
			{ "sasl.mechanisms", "GSSAPI", "GSSAPI", 1 },
			{ "sasl.mechanisms", "PLAIN", "PLAIN", 1  },
			{ "sasl.mechanisms", "GSSAPI,PLAIN", NULL, 1  },
			{ "sasl.mechanisms", "", NULL, 1  },
#endif
			{ NULL }
		};

		TEST_SAY("Canonical tests\n");
		tconf = rd_kafka_topic_conf_new();
		conf = rd_kafka_conf_new();

		for (i = 0 ; props[i].prop ; i++) {
			char dest[64];
			size_t destsz;
			rd_kafka_conf_res_t res;

			TEST_SAY("  Set: %s=%s expect %s (%s)\n",
				 props[i].prop, props[i].val, props[i].exp,
                                 props[i].is_global ? "global":"topic");


			/* Set value */
			if (props[i].is_global)
				res = rd_kafka_conf_set(conf,
						      props[i].prop,
						      props[i].val,
						      errstr, sizeof(errstr));
			else
				res = rd_kafka_topic_conf_set(tconf,
							      props[i].prop,
							      props[i].val,
							      errstr,
							      sizeof(errstr));
			if ((res == RD_KAFKA_CONF_OK ? 1:0) !=
			    (props[i].exp ? 1:0))
				TEST_FAIL("Expected %s, got %s",
					  props[i].exp ? "success" : "failure",
					  (res == RD_KAFKA_CONF_OK ? "OK" :
					   (res == RD_KAFKA_CONF_INVALID ? "INVALID" :
					    "UNKNOWN")));

			if (!props[i].exp)
				continue;

			/* Get value and compare to expected result */
			destsz = sizeof(dest);
			if (props[i].is_global)
				res = rd_kafka_conf_get(conf,
							props[i].prop,
							dest, &destsz);
			else
				res = rd_kafka_topic_conf_get(tconf,
							      props[i].prop,
							      dest, &destsz);
			TEST_ASSERT(res == RD_KAFKA_CONF_OK,
				    ".._conf_get(%s) returned %d",
                                    props[i].prop, res);

			TEST_ASSERT(!strcmp(props[i].exp, dest),
				    "Expected \"%s\", got \"%s\"",
				    props[i].exp, dest);
		}
		rd_kafka_topic_conf_destroy(tconf);
		rd_kafka_conf_destroy(conf);
	}

	return 0;
}
Example #5
0
int main (int argc, char **argv) {
    rd_kafka_t *rk;
    rd_kafka_topic_t *rkt;
    rd_kafka_conf_t *ignore_conf, *conf, *conf2;
    rd_kafka_topic_conf_t *ignore_topic_conf, *tconf, *tconf2;
    char errstr[512];
    const char **arr_orig, **arr_dup;
    size_t cnt_orig, cnt_dup;
    int i;
    const char *topic;
    static const char *gconfs[] = {
        "message.max.bytes", "12345", /* int property */
        "client.id", "my id", /* string property */
        "debug", "topic,metadata", /* S2F property */
        "compression.codec", "gzip", /* S2I property */
        NULL
    };
    static const char *tconfs[] = {
        "request.required.acks", "-1", /* int */
        "auto.commit.enable", "false", /* bool */
        "auto.offset.reset", "error",  /* S2I */
        "offset.store.path", "my/path", /* string */
        NULL
    };

    test_conf_init(&ignore_conf, &ignore_topic_conf, 10);
    rd_kafka_conf_destroy(ignore_conf);
    rd_kafka_topic_conf_destroy(ignore_topic_conf);

    topic = test_mk_topic_name("generic", 0);

    /* Set up a global config object */
    conf = rd_kafka_conf_new();

    rd_kafka_conf_set_dr_cb(conf, dr_cb);
    rd_kafka_conf_set_error_cb(conf, error_cb);

    for (i = 0 ; gconfs[i] ; i += 2) {
        if (rd_kafka_conf_set(conf, gconfs[i], gconfs[i+1],
                              errstr, sizeof(errstr)) !=
                RD_KAFKA_CONF_OK)
            TEST_FAIL("%s\n", errstr);
    }

    /* Set up a topic config object */
    tconf = rd_kafka_topic_conf_new();

    rd_kafka_topic_conf_set_partitioner_cb(tconf, partitioner);
    rd_kafka_topic_conf_set_opaque(tconf, (void *)0xbeef);

    for (i = 0 ; tconfs[i] ; i += 2) {
        if (rd_kafka_topic_conf_set(tconf, tconfs[i], tconfs[i+1],
                                    errstr, sizeof(errstr)) !=
                RD_KAFKA_CONF_OK)
            TEST_FAIL("%s\n", errstr);
    }


    /* Verify global config */
    arr_orig = rd_kafka_conf_dump(conf, &cnt_orig);
    conf_verify(__LINE__, arr_orig, cnt_orig, gconfs);

    /* Verify copied global config */
    conf2 = rd_kafka_conf_dup(conf);
    arr_dup = rd_kafka_conf_dump(conf2, &cnt_dup);
    conf_verify(__LINE__, arr_dup, cnt_dup, gconfs);
    conf_cmp("global", arr_orig, cnt_orig, arr_dup, cnt_dup);
    rd_kafka_conf_dump_free(arr_orig, cnt_orig);
    rd_kafka_conf_dump_free(arr_dup, cnt_dup);

    /* Verify topic config */
    arr_orig = rd_kafka_topic_conf_dump(tconf, &cnt_orig);
    conf_verify(__LINE__, arr_orig, cnt_orig, tconfs);

    /* Verify copied topic config */
    tconf2 = rd_kafka_topic_conf_dup(tconf);
    arr_dup = rd_kafka_topic_conf_dump(tconf2, &cnt_dup);
    conf_verify(__LINE__, arr_dup, cnt_dup, tconfs);
    conf_cmp("topic", arr_orig, cnt_orig, arr_dup, cnt_dup);
    rd_kafka_conf_dump_free(arr_orig, cnt_orig);
    rd_kafka_conf_dump_free(arr_dup, cnt_dup);


    /*
     * Create kafka instances using original and copied confs
     */

    /* original */
    rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
                      errstr, sizeof(errstr));
    if (!rk)
        TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr);

    rkt = rd_kafka_topic_new(rk, topic, tconf);
    if (!rkt)
        TEST_FAIL("Failed to create topic: %s\n",
                  strerror(errno));

    rd_kafka_topic_destroy(rkt);
    rd_kafka_destroy(rk);

    /* copied */
    rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf2,
                      errstr, sizeof(errstr));
    if (!rk)
        TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr);

    rkt = rd_kafka_topic_new(rk, topic, tconf2);
    if (!rkt)
        TEST_FAIL("Failed to create topic: %s\n",
                  strerror(errno));

    rd_kafka_topic_destroy(rkt);
    rd_kafka_destroy(rk);


    /* Wait for everything to be cleaned up since broker destroys are
     * handled in its own thread. */
    test_wait_exit(2);

    /* If we havent failed at this point then
     * there were no threads leaked */
    return 0;
}
Example #6
0
static void kafka_config_topic(rd_kafka_conf_t *conf, oconfig_item_t *ci) /* {{{ */
{
    int                          status;
    int                          i;
    struct kafka_topic_context  *tctx;
    char                        *key;
    char                        *val;
    char                         callback_name[DATA_MAX_NAME_LEN];
    char                         errbuf[1024];
    user_data_t                  ud;
	oconfig_item_t              *child;
    rd_kafka_conf_res_t          ret;

	if ((tctx = calloc(1, sizeof (*tctx))) == NULL) {
		ERROR ("write_kafka plugin: calloc failed.");
        return;
	}

    tctx->escape_char = '.';
    tctx->store_rates = 1;

    rd_kafka_conf_set_log_cb(conf, kafka_log);
    if ((tctx->kafka = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
                                    errbuf, sizeof(errbuf))) == NULL) {
        sfree(tctx);
        ERROR("write_kafka plugin: cannot create kafka handle.");
        return;
    }
    conf = NULL;

    if ((tctx->conf = rd_kafka_topic_conf_new()) == NULL) {
        rd_kafka_destroy(tctx->kafka);
        sfree(tctx);
        ERROR ("write_kafka plugin: cannot create topic configuration.");
        return;
    }

    if (ci->values_num != 1) {
        WARNING("kafka topic name needed.");
        goto errout;
    }

    if (ci->values[0].type != OCONFIG_TYPE_STRING) {
        WARNING("kafka topic needs a string argument.");
        goto errout;
    }

    if ((tctx->topic_name = strdup(ci->values[0].value.string)) == NULL) {
        ERROR("write_kafka plugin: cannot copy topic name.");
        goto errout;
    }

	for (i = 0; i < ci->children_num; i++) {
		/*
		 * The code here could be simplified but makes room
		 * for easy adding of new options later on.
		 */
		child = &ci->children[i];
		status = 0;

		if (strcasecmp ("Property", child->key) == 0) {
			if (child->values_num != 2) {
				WARNING("kafka properties need both a key and a value.");
                goto errout;
			}
			if (child->values[0].type != OCONFIG_TYPE_STRING ||
			    child->values[1].type != OCONFIG_TYPE_STRING) {
				WARNING("kafka properties needs string arguments.");
                goto errout;
			}
            key = child->values[0].value.string;
            val = child->values[0].value.string;
            ret = rd_kafka_topic_conf_set(tctx->conf,key, val,
                                          errbuf, sizeof(errbuf));
            if (ret != RD_KAFKA_CONF_OK) {
				WARNING("cannot set kafka topic property %s to %s: %s.",
                        key, val, errbuf);
                goto errout;
			}

        } else if (strcasecmp ("Key", child->key) == 0)  {
            char *tmp_buf = NULL;
            status = cf_util_get_string(child, &tmp_buf);
            if (status != 0) {
                WARNING("write_kafka plugin: invalid key supplied");
                break;
            }

            if (strcasecmp(tmp_buf, "Random") != 0) {
                tctx->has_key = 1;
                tctx->key = crc32_buffer((u_char *)tmp_buf, strlen(tmp_buf));
            }
            sfree(tmp_buf);

        } else if (strcasecmp ("Format", child->key) == 0) {
            status = cf_util_get_string(child, &key);
            if (status != 0)
                goto errout;

            assert(key != NULL);

            if (strcasecmp(key, "Command") == 0) {

                tctx->format = KAFKA_FORMAT_COMMAND;

            } else if (strcasecmp(key, "Graphite") == 0) {
                tctx->format = KAFKA_FORMAT_GRAPHITE;

            } else if (strcasecmp(key, "Json") == 0) {
                tctx->format = KAFKA_FORMAT_JSON;

            } else {
                WARNING ("write_kafka plugin: Invalid format string: %s",
                         key);
            }
            sfree(key);

        } else if (strcasecmp ("StoreRates", child->key) == 0) {
            status = cf_util_get_boolean (child, &tctx->store_rates);
            (void) cf_util_get_flag (child, &tctx->graphite_flags,
                                     GRAPHITE_STORE_RATES);

        } else if (strcasecmp ("GraphiteSeparateInstances", child->key) == 0) {
            status = cf_util_get_flag (child, &tctx->graphite_flags,
                                       GRAPHITE_SEPARATE_INSTANCES);

        } else if (strcasecmp ("GraphiteAlwaysAppendDS", child->key) == 0) {
            status = cf_util_get_flag (child, &tctx->graphite_flags,
                                       GRAPHITE_ALWAYS_APPEND_DS);

        } else if (strcasecmp ("GraphitePrefix", child->key) == 0) {
            status = cf_util_get_string (child, &tctx->prefix);
        } else if (strcasecmp ("GraphitePostfix", child->key) == 0) {
            status = cf_util_get_string (child, &tctx->postfix);
        } else if (strcasecmp ("GraphiteEscapeChar", child->key) == 0) {
            char *tmp_buff = NULL;
            status = cf_util_get_string (child, &tmp_buff);
            if (strlen (tmp_buff) > 1)
                WARNING ("write_kafka plugin: The option \"GraphiteEscapeChar\" handles "
                        "only one character. Others will be ignored.");
            tctx->escape_char = tmp_buff[0];
            sfree (tmp_buff);
        } else {
            WARNING ("write_kafka plugin: Invalid directive: %s.", child->key);
        }

        if (status != 0)
            break;
    }

    rd_kafka_topic_conf_set_partitioner_cb(tctx->conf, kafka_partition);
    rd_kafka_topic_conf_set_opaque(tctx->conf, tctx);

    if ((tctx->topic = rd_kafka_topic_new(tctx->kafka, tctx->topic_name,
                                       tctx->conf)) == NULL) {
        ERROR("write_kafka plugin: cannot create topic.");
        goto errout;
    }
    tctx->conf = NULL;

    ssnprintf(callback_name, sizeof(callback_name),
              "write_kafka/%s", tctx->topic_name);

    ud.data = tctx;
    ud.free_func = kafka_topic_context_free;

	status = plugin_register_write (callback_name, kafka_write, &ud);
	if (status != 0) {
		WARNING ("write_kafka plugin: plugin_register_write (\"%s\") "
				"failed with status %i.",
				callback_name, status);
        goto errout;
    }
    return;
 errout:
    if (conf != NULL)
        rd_kafka_conf_destroy(conf);
    if (tctx->kafka != NULL)
        rd_kafka_destroy(tctx->kafka);
    if (tctx->topic != NULL)
        rd_kafka_topic_destroy(tctx->topic);
    if (tctx->topic_name != NULL)
        free(tctx->topic_name);
    if (tctx->conf != NULL)
        rd_kafka_topic_conf_destroy(tctx->conf);
    sfree(tctx);
} /* }}} int kafka_config_topic */
Example #7
0
void p_kafka_set_dynamic_partitioner(struct p_kafka_host *kafka_host)
{
  rd_kafka_topic_conf_set_partitioner_cb(kafka_host->topic_cfg, &rd_kafka_msg_partitioner_consistent_random);
}
Example #8
0
int main_0006_symbols (int argc, char **argv) {

        if (argc < 0 /* always false */) {
                rd_kafka_version();
                rd_kafka_version_str();
		rd_kafka_get_debug_contexts();
		rd_kafka_get_err_descs(NULL, NULL);
                rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR);
		rd_kafka_err2name(RD_KAFKA_RESP_ERR_NO_ERROR);
                rd_kafka_errno2err(EINVAL);
                rd_kafka_errno();
		rd_kafka_last_error();
                rd_kafka_conf_new();
                rd_kafka_conf_destroy(NULL);
                rd_kafka_conf_dup(NULL);
                rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0);
                rd_kafka_conf_set_dr_cb(NULL, NULL);
                rd_kafka_conf_set_dr_msg_cb(NULL, NULL);
                rd_kafka_conf_set_error_cb(NULL, NULL);
                rd_kafka_conf_set_stats_cb(NULL, NULL);
                rd_kafka_conf_set_log_cb(NULL, NULL);
                rd_kafka_conf_set_socket_cb(NULL, NULL);
		rd_kafka_conf_set_rebalance_cb(NULL, NULL);
		rd_kafka_conf_set_offset_commit_cb(NULL, NULL);
		rd_kafka_conf_set_throttle_cb(NULL, NULL);
		rd_kafka_conf_set_default_topic_conf(NULL, NULL);
		rd_kafka_conf_get(NULL, NULL, NULL, NULL);
#ifndef _MSC_VER
		rd_kafka_conf_set_open_cb(NULL, NULL);
#endif
		rd_kafka_conf_set_opaque(NULL, NULL);
                rd_kafka_opaque(NULL);
                rd_kafka_conf_dump(NULL, NULL);
                rd_kafka_topic_conf_dump(NULL, NULL);
                rd_kafka_conf_dump_free(NULL, 0);
                rd_kafka_conf_properties_show(NULL);
                rd_kafka_topic_conf_new();
                rd_kafka_topic_conf_dup(NULL);
                rd_kafka_topic_conf_destroy(NULL);
                rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0);
                rd_kafka_topic_conf_set_opaque(NULL, NULL);
		rd_kafka_topic_conf_get(NULL, NULL, NULL, NULL);
                rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL);
                rd_kafka_topic_partition_available(NULL, 0);
		rd_kafka_topic_opaque(NULL);
                rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL);
                rd_kafka_msg_partitioner_consistent(NULL, NULL, 0, 0, NULL, NULL);
                rd_kafka_msg_partitioner_consistent_random(NULL, NULL, 0, 0, NULL, NULL);
                rd_kafka_new(0, NULL, NULL, 0);
                rd_kafka_destroy(NULL);
                rd_kafka_name(NULL);
		rd_kafka_memberid(NULL);
                rd_kafka_topic_new(NULL, NULL, NULL);
                rd_kafka_topic_destroy(NULL);
                rd_kafka_topic_name(NULL);
                rd_kafka_message_destroy(NULL);
                rd_kafka_message_errstr(NULL);
		rd_kafka_message_timestamp(NULL, NULL);
                rd_kafka_consume_start(NULL, 0, 0);
                rd_kafka_consume_stop(NULL, 0);
                rd_kafka_consume(NULL, 0, 0);
                rd_kafka_consume_batch(NULL, 0, 0, NULL, 0);
                rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL);
                rd_kafka_offset_store(NULL, 0, 0);
                rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL);
                rd_kafka_produce_batch(NULL, 0, 0, NULL, 0);
                rd_kafka_poll(NULL, 0);
                rd_kafka_brokers_add(NULL, NULL);
                /* DEPRECATED: rd_kafka_set_logger(NULL, NULL); */
                rd_kafka_set_log_level(NULL, 0);
                rd_kafka_log_print(NULL, 0, NULL, NULL);
#ifndef _MSC_VER
                rd_kafka_log_syslog(NULL, 0, NULL, NULL);
#endif
                rd_kafka_outq_len(NULL);
                rd_kafka_dump(NULL, NULL);
                rd_kafka_thread_cnt();
                rd_kafka_wait_destroyed(0);
                rd_kafka_metadata(NULL, 0, NULL, NULL, 0);
                rd_kafka_metadata_destroy(NULL);
                rd_kafka_queue_destroy(NULL);
                rd_kafka_consume_start_queue(NULL, 0, 0, NULL);
                rd_kafka_consume_queue(NULL, 0);
                rd_kafka_consume_batch_queue(NULL, 0, NULL, 0);
                rd_kafka_consume_callback_queue(NULL, 0, NULL, NULL);
                rd_kafka_seek(NULL, 0, 0, 0);
                rd_kafka_yield(NULL);
                rd_kafka_mem_free(NULL, NULL);
                rd_kafka_list_groups(NULL, NULL, NULL, 0);
                rd_kafka_group_list_destroy(NULL);

		/* KafkaConsumer API */
		rd_kafka_subscribe(NULL, NULL);
		rd_kafka_unsubscribe(NULL);
		rd_kafka_subscription(NULL, NULL);
		rd_kafka_consumer_poll(NULL, 0);
		rd_kafka_consumer_close(NULL);
		rd_kafka_assign(NULL, NULL);
		rd_kafka_assignment(NULL, NULL);
		rd_kafka_commit(NULL, NULL, 0);
		rd_kafka_commit_message(NULL, NULL, 0);
                rd_kafka_committed(NULL, NULL, 0);
		rd_kafka_position(NULL, NULL);

		/* TopicPartition */
		rd_kafka_topic_partition_list_new(0);
		rd_kafka_topic_partition_list_destroy(NULL);
		rd_kafka_topic_partition_list_add(NULL, NULL, 0);
		rd_kafka_topic_partition_list_add_range(NULL, NULL, 0, 0);
		rd_kafka_topic_partition_list_del(NULL, NULL, 0);
		rd_kafka_topic_partition_list_del_by_idx(NULL, 0);
		rd_kafka_topic_partition_list_copy(NULL);
		rd_kafka_topic_partition_list_set_offset(NULL, NULL, 0, 0);
		rd_kafka_topic_partition_list_find(NULL, NULL, 0);
		rd_kafka_query_watermark_offsets(NULL, NULL, 0, NULL, NULL, 0);
		rd_kafka_get_watermark_offsets(NULL, NULL, 0, NULL, NULL);
        }


	return 0;
}