static int Producer_init (PyObject *selfobj, PyObject *args, PyObject *kwargs) {
        Handle *self = (Handle *)selfobj;
        char errstr[256];
        rd_kafka_conf_t *conf;

        if (self->rk) {
                PyErr_SetString(PyExc_RuntimeError,
                                "Producer already __init__:ialized");
                return -1;
        }

        self->type = RD_KAFKA_PRODUCER;

        if (!(conf = common_conf_setup(RD_KAFKA_PRODUCER, self,
                                       args, kwargs)))
                return -1;

        rd_kafka_conf_set_dr_msg_cb(conf, dr_msg_cb);

        self->rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
                                errstr, sizeof(errstr));
        if (!self->rk) {
                cfl_PyErr_Format(rd_kafka_last_error(),
                                 "Failed to create producer: %s", errstr);
                rd_kafka_conf_destroy(conf);
                return -1;
        }

        /* Forward log messages to poll queue */
        if (self->logger)
                rd_kafka_set_log_queue(self->rk, NULL);

        return 0;
}
Beispiel #2
0
int main (int argc, char **argv) {

        if (argc < 0 /* always false */) {
                rd_kafka_version();
                rd_kafka_version_str();
                rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR);
                rd_kafka_errno2err(EINVAL);
                rd_kafka_conf_new();
                rd_kafka_conf_destroy(NULL);
                rd_kafka_conf_dup(NULL);
                rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0);
                rd_kafka_conf_set_dr_cb(NULL, NULL);
                rd_kafka_conf_set_error_cb(NULL, NULL);
                rd_kafka_conf_set_stats_cb(NULL, NULL);
                rd_kafka_conf_set_opaque(NULL, NULL);
                rd_kafka_conf_dump(NULL, NULL);
                rd_kafka_topic_conf_dump(NULL, NULL);
                rd_kafka_conf_dump_free(NULL, 0);
                rd_kafka_conf_properties_show(NULL);
                rd_kafka_topic_conf_new();
                rd_kafka_topic_conf_dup(NULL);
                rd_kafka_topic_conf_destroy(NULL);
                rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0);
                rd_kafka_topic_conf_set_opaque(NULL, NULL);
                rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL);
                rd_kafka_topic_partition_available(NULL, 0);
                rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL);
                rd_kafka_new(0, NULL, NULL, 0);
                rd_kafka_destroy(NULL);
                rd_kafka_name(NULL);
                rd_kafka_topic_new(NULL, NULL, NULL);
                rd_kafka_topic_destroy(NULL);
                rd_kafka_topic_name(NULL);
                rd_kafka_message_destroy(NULL);
                rd_kafka_message_errstr(NULL);
                rd_kafka_consume_start(NULL, 0, 0);
                rd_kafka_consume_stop(NULL, 0);
                rd_kafka_consume(NULL, 0, 0);
                rd_kafka_consume_batch(NULL, 0, 0, NULL, 0);
                rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL);
                rd_kafka_offset_store(NULL, 0, 0);
                rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL);
                rd_kafka_poll(NULL, 0);
                rd_kafka_brokers_add(NULL, NULL);
                rd_kafka_set_logger(NULL, NULL);
                rd_kafka_set_log_level(NULL, 0);
                rd_kafka_log_print(NULL, 0, NULL, NULL);
                rd_kafka_log_syslog(NULL, 0, NULL, NULL);
                rd_kafka_outq_len(NULL);
                rd_kafka_dump(NULL, NULL);
                rd_kafka_thread_cnt();
                rd_kafka_wait_destroyed(0);
        }


	return 0;
}
Beispiel #3
0
static int kafka_handle(struct kafka_topic_context *ctx) /* {{{ */
{
    char                         errbuf[1024];
    rd_kafka_conf_t             *conf;
    rd_kafka_topic_conf_t       *topic_conf;

    if (ctx->kafka != NULL && ctx->topic != NULL)
        return(0);

    if (ctx->kafka == NULL) {
        if ((conf = rd_kafka_conf_dup(ctx->kafka_conf)) == NULL) {
            ERROR("write_kafka plugin: cannot duplicate kafka config");
            return(1);
        }

        if ((ctx->kafka = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
                                    errbuf, sizeof(errbuf))) == NULL) {
            ERROR("write_kafka plugin: cannot create kafka handle.");
            return 1;
        }

        rd_kafka_conf_destroy(ctx->kafka_conf);
        ctx->kafka_conf = NULL;

        INFO ("write_kafka plugin: created KAFKA handle : %s", rd_kafka_name(ctx->kafka));

#if defined(HAVE_LIBRDKAFKA_LOGGER) && !defined(HAVE_LIBRDKAFKA_LOG_CB)
        rd_kafka_set_logger(ctx->kafka, kafka_log);
#endif
    }

    if (ctx->topic == NULL ) {
        if ((topic_conf = rd_kafka_topic_conf_dup(ctx->conf)) == NULL) {
            ERROR("write_kafka plugin: cannot duplicate kafka topic config");
            return 1;
        }

        if ((ctx->topic = rd_kafka_topic_new(ctx->kafka, ctx->topic_name,
                                            topic_conf)) == NULL) {
            ERROR("write_kafka plugin: cannot create topic : %s\n",
            rd_kafka_err2str(rd_kafka_errno2err(errno)));
            return errno;
        }

        rd_kafka_topic_conf_destroy(ctx->conf);
        ctx->conf = NULL;

        INFO ("write_kafka plugin: handle created for topic : %s", rd_kafka_topic_name(ctx->topic));
    }

    return(0);

} /* }}} int kafka_handle */
Beispiel #4
0
static void kafka_topic_context_free(void *p) /* {{{ */
{
    struct kafka_topic_context *ctx = p;

    if (ctx == NULL)
        return;

    if (ctx->topic_name != NULL)
        sfree(ctx->topic_name);
    if (ctx->topic != NULL)
        rd_kafka_topic_destroy(ctx->topic);
    if (ctx->conf != NULL)
        rd_kafka_topic_conf_destroy(ctx->conf);
    if (ctx->kafka_conf != NULL)
        rd_kafka_conf_destroy(ctx->kafka_conf);
    if (ctx->kafka != NULL)
        rd_kafka_destroy(ctx->kafka);

    sfree(ctx);
} /* }}} void kafka_topic_context_free */
static int Consumer_init (PyObject *selfobj, PyObject *args, PyObject *kwargs) {
        Handle *self = (Handle *)selfobj;
        char errstr[256];
        rd_kafka_conf_t *conf;

        if (self->rk) {
                PyErr_SetString(PyExc_RuntimeError,
                                "Consumer already initialized");
                return -1;
        }

        self->type = RD_KAFKA_CONSUMER;

        if (!(conf = common_conf_setup(RD_KAFKA_CONSUMER, self,
                                       args, kwargs)))
                return -1; /* Exception raised by ..conf_setup() */

        rd_kafka_conf_set_rebalance_cb(conf, Consumer_rebalance_cb);
        rd_kafka_conf_set_offset_commit_cb(conf, Consumer_offset_commit_cb);

        self->rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf,
                                errstr, sizeof(errstr));
        if (!self->rk) {
                cfl_PyErr_Format(rd_kafka_last_error(),
                                 "Failed to create consumer: %s", errstr);
                rd_kafka_conf_destroy(conf);
                return -1;
        }

        /* Forward log messages to main queue which is then forwarded
         * to the consumer queue */
        if (self->logger)
                rd_kafka_set_log_queue(self->rk, NULL);

        rd_kafka_poll_set_consumer(self->rk);

        self->u.Consumer.rkqu = rd_kafka_queue_get_consumer(self->rk);
        assert(self->u.Consumer.rkqu);

        return 0;
}
Beispiel #6
0
static PyObject *
RdkHandle_stop(RdkHandle *self)
{
    /* We'll only ever get a locking error if we programmed ourselves into a
     * deadlock.  We'd have to admit defeat, abort, and leak this RdkHandle */
    if (RdkHandle_excl_lock(self)) return NULL;

    Py_BEGIN_ALLOW_THREADS  /* avoid callbacks deadlocking */
        if (self->rdk_queue_handle) {
            rd_kafka_queue_destroy(self->rdk_queue_handle);
            self->rdk_queue_handle = NULL;
        }
        if (self->rdk_topic_handle) {
            rd_kafka_topic_destroy(self->rdk_topic_handle);
            self->rdk_topic_handle = NULL;
        }
        if (self->rdk_handle) {
            PyObject *opaque = (PyObject *)rd_kafka_opaque(self->rdk_handle);
            Py_XDECREF(opaque);
            rd_kafka_destroy(self->rdk_handle);
            self->rdk_handle = NULL;
        }
        if (self->rdk_conf) {
            rd_kafka_conf_destroy(self->rdk_conf);
            self->rdk_conf = NULL;
        }
        if (self->rdk_topic_conf) {
            rd_kafka_topic_conf_destroy(self->rdk_topic_conf);
            self->rdk_topic_conf = NULL;
        }
    Py_END_ALLOW_THREADS

    Py_CLEAR(self->partition_ids);

    if (RdkHandle_unlock(self)) return NULL;
    Py_INCREF(Py_None);
    return Py_None;
}
Beispiel #7
0
/**
 * @brief Interceptors must not be copied automatically by conf_dup()
 *        unless the interceptors have added on_conf_dup().
 *        This behaviour makes sure an interceptor's instance
 *        is not duplicated without the interceptor's knowledge or
 *        assistance.
 */
static void do_test_conf_copy (const char *topic) {
        rd_kafka_conf_t *conf, *conf2;
        int i;
        rd_kafka_t *rk;

        TEST_SAY(_C_MAG "[ %s ]\n" _C_CLR, __FUNCTION__);

        memset(&msgs[0], 0, sizeof(msgs));

        test_conf_init(&conf, NULL, 0);

        rd_kafka_conf_interceptor_add_on_new(conf, "on_new_conf_copy",
                                             on_new_producer, NULL);

        /* Now copy the configuration to verify that interceptors are
         * NOT copied. */
        conf2 = conf;
        conf = rd_kafka_conf_dup(conf2);
        rd_kafka_conf_destroy(conf2);

        /* Create producer */
        rk = test_create_handle(RD_KAFKA_PRODUCER, conf);

        for (i = 0 ; i < msgcnt-1 ; i++)
                do_test_produce(rk, topic, RD_KAFKA_PARTITION_UA, i, 0, 0);

        /* Wait for messages to be delivered */
        test_flush(rk, -1);

        /* Verify acks */
        for (i = 0 ; i < msgcnt ; i++) {
                struct msg_state *msg = &msgs[i];
                msg_verify_ic_cnt(msg, "on_ack", msg->bits[_ON_ACK], 0);
        }

        rd_kafka_destroy(rk);
}
Beispiel #8
0
int main_0004_conf (int argc, char **argv) {
	rd_kafka_t *rk;
	rd_kafka_topic_t *rkt;
	rd_kafka_conf_t *ignore_conf, *conf, *conf2;
	rd_kafka_topic_conf_t *ignore_topic_conf, *tconf, *tconf2;
	char errstr[512];
	const char **arr_orig, **arr_dup;
	size_t cnt_orig, cnt_dup;
	int i;
        const char *topic;
	static const char *gconfs[] = {
		"message.max.bytes", "12345", /* int property */
		"client.id", "my id", /* string property */
		"debug", "topic,metadata", /* S2F property */
		"topic.blacklist", "__.*", /* #778 */
                "auto.offset.reset", "earliest", /* Global->Topic fallthru */
#if WITH_ZLIB
		"compression.codec", "gzip", /* S2I property */
#endif
		NULL
	};
	static const char *tconfs[] = {
		"request.required.acks", "-1", /* int */
		"auto.commit.enable", "false", /* bool */
		"auto.offset.reset", "error",  /* S2I */
		"offset.store.path", "my/path", /* string */
		NULL
	};

	test_conf_init(&ignore_conf, &ignore_topic_conf, 10);
	rd_kafka_conf_destroy(ignore_conf);
	rd_kafka_topic_conf_destroy(ignore_topic_conf);

        topic = test_mk_topic_name("0004", 0);

	/* Set up a global config object */
	conf = rd_kafka_conf_new();

	rd_kafka_conf_set_dr_cb(conf, dr_cb);
	rd_kafka_conf_set_error_cb(conf, error_cb);

	for (i = 0 ; gconfs[i] ; i += 2) {
		if (rd_kafka_conf_set(conf, gconfs[i], gconfs[i+1],
				      errstr, sizeof(errstr)) !=
		    RD_KAFKA_CONF_OK)
			TEST_FAIL("%s\n", errstr);
	}

	/* Set up a topic config object */
	tconf = rd_kafka_topic_conf_new();

	rd_kafka_topic_conf_set_partitioner_cb(tconf, partitioner);
	rd_kafka_topic_conf_set_opaque(tconf, (void *)0xbeef);

	for (i = 0 ; tconfs[i] ; i += 2) {
		if (rd_kafka_topic_conf_set(tconf, tconfs[i], tconfs[i+1],
				      errstr, sizeof(errstr)) !=
		    RD_KAFKA_CONF_OK)
			TEST_FAIL("%s\n", errstr);
	}


	/* Verify global config */
	arr_orig = rd_kafka_conf_dump(conf, &cnt_orig);
	conf_verify(__LINE__, arr_orig, cnt_orig, gconfs);

	/* Verify copied global config */
	conf2 = rd_kafka_conf_dup(conf);
	arr_dup = rd_kafka_conf_dump(conf2, &cnt_dup);
	conf_verify(__LINE__, arr_dup, cnt_dup, gconfs);
	conf_cmp("global", arr_orig, cnt_orig, arr_dup, cnt_dup);
	rd_kafka_conf_dump_free(arr_orig, cnt_orig);
	rd_kafka_conf_dump_free(arr_dup, cnt_dup);

	/* Verify topic config */
	arr_orig = rd_kafka_topic_conf_dump(tconf, &cnt_orig);
	conf_verify(__LINE__, arr_orig, cnt_orig, tconfs);

	/* Verify copied topic config */
	tconf2 = rd_kafka_topic_conf_dup(tconf);
	arr_dup = rd_kafka_topic_conf_dump(tconf2, &cnt_dup);
	conf_verify(__LINE__, arr_dup, cnt_dup, tconfs);
	conf_cmp("topic", arr_orig, cnt_orig, arr_dup, cnt_dup);
	rd_kafka_conf_dump_free(arr_orig, cnt_orig);
	rd_kafka_conf_dump_free(arr_dup, cnt_dup);


	/*
	 * Create kafka instances using original and copied confs
	 */

	/* original */
	rk = test_create_handle(RD_KAFKA_PRODUCER, conf);

	rkt = rd_kafka_topic_new(rk, topic, tconf);
	if (!rkt)
		TEST_FAIL("Failed to create topic: %s\n",
			  rd_strerror(errno));

	rd_kafka_topic_destroy(rkt);
	rd_kafka_destroy(rk);

	/* copied */
	rk = test_create_handle(RD_KAFKA_PRODUCER, conf2);

	rkt = rd_kafka_topic_new(rk, topic, tconf2);
	if (!rkt)
		TEST_FAIL("Failed to create topic: %s\n",
			  rd_strerror(errno));

	rd_kafka_topic_destroy(rkt);
	rd_kafka_destroy(rk);


	/* Incremental S2F property.
	 * NOTE: The order of fields returned in get() is hardcoded here. */
	{
		static const char *s2fs[] = {
			"generic,broker,queue,cgrp",
			"generic,broker,queue,cgrp",

			"-broker,+queue,topic",
			"generic,topic,queue,cgrp",

			"-all,security,-fetch,+metadata",
			"metadata,security",

			NULL
		};

		TEST_SAY("Incremental S2F tests\n");
		conf = rd_kafka_conf_new();

		for (i = 0 ; s2fs[i] ; i += 2) {
			const char *val;

			TEST_SAY("  Set: %s\n", s2fs[i]);
			test_conf_set(conf, "debug", s2fs[i]);
			val = test_conf_get(conf, "debug");
			TEST_SAY("  Now: %s\n", val);

			if (strcmp(val, s2fs[i+1]))
				TEST_FAIL_LATER("\n"
						"Expected: %s\n"
						"     Got: %s",
						s2fs[i+1], val);
		}
		rd_kafka_conf_destroy(conf);
	}

	/* Canonical int values, aliases, s2i-verified strings */
	{
		static const struct {
			const char *prop;
			const char *val;
			const char *exp;
			int is_global;
		} props[] = {
			{ "request.required.acks", "0", "0" },
			{ "request.required.acks", "-1", "-1" },
			{ "request.required.acks", "1", "1" },
			{ "acks", "3", "3" }, /* alias test */
			{ "request.required.acks", "393", "393" },
			{ "request.required.acks", "bad", NULL },
			{ "request.required.acks", "all", "-1" },
                        { "request.required.acks", "all", "-1", 1/*fallthru*/ },
			{ "acks", "0", "0" }, /* alias test */
#if WITH_SASL
			{ "sasl.mechanisms", "GSSAPI", "GSSAPI", 1 },
			{ "sasl.mechanisms", "PLAIN", "PLAIN", 1  },
			{ "sasl.mechanisms", "GSSAPI,PLAIN", NULL, 1  },
			{ "sasl.mechanisms", "", NULL, 1  },
#endif
			{ NULL }
		};

		TEST_SAY("Canonical tests\n");
		tconf = rd_kafka_topic_conf_new();
		conf = rd_kafka_conf_new();

		for (i = 0 ; props[i].prop ; i++) {
			char dest[64];
			size_t destsz;
			rd_kafka_conf_res_t res;

			TEST_SAY("  Set: %s=%s expect %s (%s)\n",
				 props[i].prop, props[i].val, props[i].exp,
                                 props[i].is_global ? "global":"topic");


			/* Set value */
			if (props[i].is_global)
				res = rd_kafka_conf_set(conf,
						      props[i].prop,
						      props[i].val,
						      errstr, sizeof(errstr));
			else
				res = rd_kafka_topic_conf_set(tconf,
							      props[i].prop,
							      props[i].val,
							      errstr,
							      sizeof(errstr));
			if ((res == RD_KAFKA_CONF_OK ? 1:0) !=
			    (props[i].exp ? 1:0))
				TEST_FAIL("Expected %s, got %s",
					  props[i].exp ? "success" : "failure",
					  (res == RD_KAFKA_CONF_OK ? "OK" :
					   (res == RD_KAFKA_CONF_INVALID ? "INVALID" :
					    "UNKNOWN")));

			if (!props[i].exp)
				continue;

			/* Get value and compare to expected result */
			destsz = sizeof(dest);
			if (props[i].is_global)
				res = rd_kafka_conf_get(conf,
							props[i].prop,
							dest, &destsz);
			else
				res = rd_kafka_topic_conf_get(tconf,
							      props[i].prop,
							      dest, &destsz);
			TEST_ASSERT(res == RD_KAFKA_CONF_OK,
				    ".._conf_get(%s) returned %d",
                                    props[i].prop, res);

			TEST_ASSERT(!strcmp(props[i].exp, dest),
				    "Expected \"%s\", got \"%s\"",
				    props[i].exp, dest);
		}
		rd_kafka_topic_conf_destroy(tconf);
		rd_kafka_conf_destroy(conf);
	}

	return 0;
}
Beispiel #9
0
int main (int argc, char **argv) {
    rd_kafka_t *rk;
    rd_kafka_topic_t *rkt;
    rd_kafka_conf_t *ignore_conf, *conf, *conf2;
    rd_kafka_topic_conf_t *ignore_topic_conf, *tconf, *tconf2;
    char errstr[512];
    const char **arr_orig, **arr_dup;
    size_t cnt_orig, cnt_dup;
    int i;
    const char *topic;
    static const char *gconfs[] = {
        "message.max.bytes", "12345", /* int property */
        "client.id", "my id", /* string property */
        "debug", "topic,metadata", /* S2F property */
        "compression.codec", "gzip", /* S2I property */
        NULL
    };
    static const char *tconfs[] = {
        "request.required.acks", "-1", /* int */
        "auto.commit.enable", "false", /* bool */
        "auto.offset.reset", "error",  /* S2I */
        "offset.store.path", "my/path", /* string */
        NULL
    };

    test_conf_init(&ignore_conf, &ignore_topic_conf, 10);
    rd_kafka_conf_destroy(ignore_conf);
    rd_kafka_topic_conf_destroy(ignore_topic_conf);

    topic = test_mk_topic_name("generic", 0);

    /* Set up a global config object */
    conf = rd_kafka_conf_new();

    rd_kafka_conf_set_dr_cb(conf, dr_cb);
    rd_kafka_conf_set_error_cb(conf, error_cb);

    for (i = 0 ; gconfs[i] ; i += 2) {
        if (rd_kafka_conf_set(conf, gconfs[i], gconfs[i+1],
                              errstr, sizeof(errstr)) !=
                RD_KAFKA_CONF_OK)
            TEST_FAIL("%s\n", errstr);
    }

    /* Set up a topic config object */
    tconf = rd_kafka_topic_conf_new();

    rd_kafka_topic_conf_set_partitioner_cb(tconf, partitioner);
    rd_kafka_topic_conf_set_opaque(tconf, (void *)0xbeef);

    for (i = 0 ; tconfs[i] ; i += 2) {
        if (rd_kafka_topic_conf_set(tconf, tconfs[i], tconfs[i+1],
                                    errstr, sizeof(errstr)) !=
                RD_KAFKA_CONF_OK)
            TEST_FAIL("%s\n", errstr);
    }


    /* Verify global config */
    arr_orig = rd_kafka_conf_dump(conf, &cnt_orig);
    conf_verify(__LINE__, arr_orig, cnt_orig, gconfs);

    /* Verify copied global config */
    conf2 = rd_kafka_conf_dup(conf);
    arr_dup = rd_kafka_conf_dump(conf2, &cnt_dup);
    conf_verify(__LINE__, arr_dup, cnt_dup, gconfs);
    conf_cmp("global", arr_orig, cnt_orig, arr_dup, cnt_dup);
    rd_kafka_conf_dump_free(arr_orig, cnt_orig);
    rd_kafka_conf_dump_free(arr_dup, cnt_dup);

    /* Verify topic config */
    arr_orig = rd_kafka_topic_conf_dump(tconf, &cnt_orig);
    conf_verify(__LINE__, arr_orig, cnt_orig, tconfs);

    /* Verify copied topic config */
    tconf2 = rd_kafka_topic_conf_dup(tconf);
    arr_dup = rd_kafka_topic_conf_dump(tconf2, &cnt_dup);
    conf_verify(__LINE__, arr_dup, cnt_dup, tconfs);
    conf_cmp("topic", arr_orig, cnt_orig, arr_dup, cnt_dup);
    rd_kafka_conf_dump_free(arr_orig, cnt_orig);
    rd_kafka_conf_dump_free(arr_dup, cnt_dup);


    /*
     * Create kafka instances using original and copied confs
     */

    /* original */
    rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
                      errstr, sizeof(errstr));
    if (!rk)
        TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr);

    rkt = rd_kafka_topic_new(rk, topic, tconf);
    if (!rkt)
        TEST_FAIL("Failed to create topic: %s\n",
                  strerror(errno));

    rd_kafka_topic_destroy(rkt);
    rd_kafka_destroy(rk);

    /* copied */
    rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf2,
                      errstr, sizeof(errstr));
    if (!rk)
        TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr);

    rkt = rd_kafka_topic_new(rk, topic, tconf2);
    if (!rkt)
        TEST_FAIL("Failed to create topic: %s\n",
                  strerror(errno));

    rd_kafka_topic_destroy(rkt);
    rd_kafka_destroy(rk);


    /* Wait for everything to be cleaned up since broker destroys are
     * handled in its own thread. */
    test_wait_exit(2);

    /* If we havent failed at this point then
     * there were no threads leaked */
    return 0;
}
Beispiel #10
0
static int kafka_config(oconfig_item_t *ci) /* {{{ */
{
	int                          i;
	oconfig_item_t              *child;
    rd_kafka_conf_t             *conf;
    rd_kafka_conf_t             *cloned;
    rd_kafka_conf_res_t          ret;
    char                         errbuf[1024];

    if ((conf = rd_kafka_conf_new()) == NULL) {
        WARNING("cannot allocate kafka configuration.");
        return -1;
    }

	for (i = 0; i < ci->children_num; i++)  {
		child = &ci->children[i];

		if (strcasecmp("Topic", child->key) == 0) {
            if ((cloned = rd_kafka_conf_dup(conf)) == NULL) {
                WARNING("write_kafka plugin: cannot allocate memory for kafka config");
                goto errout;
            }
			kafka_config_topic (cloned, child);
		} else if (strcasecmp(child->key, "Property") == 0) {
			char *key = NULL;
			char *val = NULL;

			if (child->values_num != 2) {
				WARNING("kafka properties need both a key and a value.");
                goto errout;
			}
			if (child->values[0].type != OCONFIG_TYPE_STRING ||
			    child->values[1].type != OCONFIG_TYPE_STRING) {
				WARNING("kafka properties needs string arguments.");
                goto errout;
			}
			if ((key = strdup(child->values[0].value.string)) == NULL) {
				WARNING("cannot allocate memory for attribute key.");
                goto errout;
			}
			if ((val = strdup(child->values[1].value.string)) == NULL) {
				WARNING("cannot allocate memory for attribute value.");
                goto errout;
			}
            ret = rd_kafka_conf_set(conf, key, val, errbuf, sizeof(errbuf));
            if (ret != RD_KAFKA_CONF_OK) {
                WARNING("cannot set kafka property %s to %s: %s",
                        key, val, errbuf);
                goto errout;
            }
			sfree(key);
			sfree(val);
		} else {
			WARNING ("write_kafka plugin: Ignoring unknown "
				 "configuration option \"%s\" at top level.",
				 child->key);
		}
	}
    if (conf != NULL)
        rd_kafka_conf_destroy(conf);
	return (0);
 errout:
    if (conf != NULL)
        rd_kafka_conf_destroy(conf);
    return -1;
} /* }}} int kafka_config */
Beispiel #11
0
static void kafka_config_topic(rd_kafka_conf_t *conf, oconfig_item_t *ci) /* {{{ */
{
    int                          status;
    int                          i;
    struct kafka_topic_context  *tctx;
    char                        *key;
    char                        *val;
    char                         callback_name[DATA_MAX_NAME_LEN];
    char                         errbuf[1024];
    user_data_t                  ud;
	oconfig_item_t              *child;
    rd_kafka_conf_res_t          ret;

	if ((tctx = calloc(1, sizeof (*tctx))) == NULL) {
		ERROR ("write_kafka plugin: calloc failed.");
        return;
	}

    tctx->escape_char = '.';
    tctx->store_rates = 1;

    rd_kafka_conf_set_log_cb(conf, kafka_log);
    if ((tctx->kafka = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
                                    errbuf, sizeof(errbuf))) == NULL) {
        sfree(tctx);
        ERROR("write_kafka plugin: cannot create kafka handle.");
        return;
    }
    conf = NULL;

    if ((tctx->conf = rd_kafka_topic_conf_new()) == NULL) {
        rd_kafka_destroy(tctx->kafka);
        sfree(tctx);
        ERROR ("write_kafka plugin: cannot create topic configuration.");
        return;
    }

    if (ci->values_num != 1) {
        WARNING("kafka topic name needed.");
        goto errout;
    }

    if (ci->values[0].type != OCONFIG_TYPE_STRING) {
        WARNING("kafka topic needs a string argument.");
        goto errout;
    }

    if ((tctx->topic_name = strdup(ci->values[0].value.string)) == NULL) {
        ERROR("write_kafka plugin: cannot copy topic name.");
        goto errout;
    }

	for (i = 0; i < ci->children_num; i++) {
		/*
		 * The code here could be simplified but makes room
		 * for easy adding of new options later on.
		 */
		child = &ci->children[i];
		status = 0;

		if (strcasecmp ("Property", child->key) == 0) {
			if (child->values_num != 2) {
				WARNING("kafka properties need both a key and a value.");
                goto errout;
			}
			if (child->values[0].type != OCONFIG_TYPE_STRING ||
			    child->values[1].type != OCONFIG_TYPE_STRING) {
				WARNING("kafka properties needs string arguments.");
                goto errout;
			}
            key = child->values[0].value.string;
            val = child->values[0].value.string;
            ret = rd_kafka_topic_conf_set(tctx->conf,key, val,
                                          errbuf, sizeof(errbuf));
            if (ret != RD_KAFKA_CONF_OK) {
				WARNING("cannot set kafka topic property %s to %s: %s.",
                        key, val, errbuf);
                goto errout;
			}

        } else if (strcasecmp ("Key", child->key) == 0)  {
            char *tmp_buf = NULL;
            status = cf_util_get_string(child, &tmp_buf);
            if (status != 0) {
                WARNING("write_kafka plugin: invalid key supplied");
                break;
            }

            if (strcasecmp(tmp_buf, "Random") != 0) {
                tctx->has_key = 1;
                tctx->key = crc32_buffer((u_char *)tmp_buf, strlen(tmp_buf));
            }
            sfree(tmp_buf);

        } else if (strcasecmp ("Format", child->key) == 0) {
            status = cf_util_get_string(child, &key);
            if (status != 0)
                goto errout;

            assert(key != NULL);

            if (strcasecmp(key, "Command") == 0) {

                tctx->format = KAFKA_FORMAT_COMMAND;

            } else if (strcasecmp(key, "Graphite") == 0) {
                tctx->format = KAFKA_FORMAT_GRAPHITE;

            } else if (strcasecmp(key, "Json") == 0) {
                tctx->format = KAFKA_FORMAT_JSON;

            } else {
                WARNING ("write_kafka plugin: Invalid format string: %s",
                         key);
            }
            sfree(key);

        } else if (strcasecmp ("StoreRates", child->key) == 0) {
            status = cf_util_get_boolean (child, &tctx->store_rates);
            (void) cf_util_get_flag (child, &tctx->graphite_flags,
                                     GRAPHITE_STORE_RATES);

        } else if (strcasecmp ("GraphiteSeparateInstances", child->key) == 0) {
            status = cf_util_get_flag (child, &tctx->graphite_flags,
                                       GRAPHITE_SEPARATE_INSTANCES);

        } else if (strcasecmp ("GraphiteAlwaysAppendDS", child->key) == 0) {
            status = cf_util_get_flag (child, &tctx->graphite_flags,
                                       GRAPHITE_ALWAYS_APPEND_DS);

        } else if (strcasecmp ("GraphitePrefix", child->key) == 0) {
            status = cf_util_get_string (child, &tctx->prefix);
        } else if (strcasecmp ("GraphitePostfix", child->key) == 0) {
            status = cf_util_get_string (child, &tctx->postfix);
        } else if (strcasecmp ("GraphiteEscapeChar", child->key) == 0) {
            char *tmp_buff = NULL;
            status = cf_util_get_string (child, &tmp_buff);
            if (strlen (tmp_buff) > 1)
                WARNING ("write_kafka plugin: The option \"GraphiteEscapeChar\" handles "
                        "only one character. Others will be ignored.");
            tctx->escape_char = tmp_buff[0];
            sfree (tmp_buff);
        } else {
            WARNING ("write_kafka plugin: Invalid directive: %s.", child->key);
        }

        if (status != 0)
            break;
    }

    rd_kafka_topic_conf_set_partitioner_cb(tctx->conf, kafka_partition);
    rd_kafka_topic_conf_set_opaque(tctx->conf, tctx);

    if ((tctx->topic = rd_kafka_topic_new(tctx->kafka, tctx->topic_name,
                                       tctx->conf)) == NULL) {
        ERROR("write_kafka plugin: cannot create topic.");
        goto errout;
    }
    tctx->conf = NULL;

    ssnprintf(callback_name, sizeof(callback_name),
              "write_kafka/%s", tctx->topic_name);

    ud.data = tctx;
    ud.free_func = kafka_topic_context_free;

	status = plugin_register_write (callback_name, kafka_write, &ud);
	if (status != 0) {
		WARNING ("write_kafka plugin: plugin_register_write (\"%s\") "
				"failed with status %i.",
				callback_name, status);
        goto errout;
    }
    return;
 errout:
    if (conf != NULL)
        rd_kafka_conf_destroy(conf);
    if (tctx->kafka != NULL)
        rd_kafka_destroy(tctx->kafka);
    if (tctx->topic != NULL)
        rd_kafka_topic_destroy(tctx->topic);
    if (tctx->topic_name != NULL)
        free(tctx->topic_name);
    if (tctx->conf != NULL)
        rd_kafka_topic_conf_destroy(tctx->conf);
    sfree(tctx);
} /* }}} int kafka_config_topic */
Beispiel #12
0
rd_kafka_t *kafka_get_connection(kafka_connection_params params, const char *brokers)
{
    rd_kafka_t *r = NULL;
    char errstr[512];
    rd_kafka_conf_t *conf = rd_kafka_conf_new();
    //set error callback
    rd_kafka_conf_set_error_cb(conf, kafka_err_cb);
    if (params.type == RD_KAFKA_CONSUMER)
    {
        if (params.queue_buffer)
            rd_kafka_conf_set(conf, "queued.min.messages", params.queue_buffer, NULL, 0);
        r = rd_kafka_new(params.type, conf, errstr, sizeof errstr);
        if (!r)
        {
            if (params.log_level)
            {
                openlog("phpkafka", 0, LOG_USER);
                syslog(LOG_ERR, "Failed to connect to kafka: %s", errstr);
            }
            //destroy config, no connection to use it...
            rd_kafka_conf_destroy(conf);
            return NULL;
        }
        if (!rd_kafka_brokers_add(r, brokers))
        {
            if (params.log_level)
            {
                openlog("phpkafka", 0, LOG_USER);
                syslog(LOG_ERR, "Failed to connect to brokers %s", brokers);
            }
            rd_kafka_destroy(r);
            return NULL;
        }
        return r;
    }
    if (params.compression)
    {
        rd_kafka_conf_res_t result = rd_kafka_conf_set(
            conf, "compression.codec",params.compression, errstr, sizeof errstr
        );
        if (result != RD_KAFKA_CONF_OK)
        {
            if (params.log_level)
            {
                openlog("phpkafka", 0, LOG_USER);
                syslog(LOG_ALERT, "Failed to set compression %s: %s", params.compression, errstr);
            }
            rd_kafka_conf_destroy(conf);
            return NULL;
        }
    }
    if (params.retry_count)
    {
        rd_kafka_conf_res_t result = rd_kafka_conf_set(
            conf, "message.send.max.retries",params.retry_count, errstr, sizeof errstr
        );
        if (result != RD_KAFKA_CONF_OK)
        {
            if (params.log_level)
            {
                openlog("phpkafka", 0, LOG_USER);
                syslog(LOG_ALERT, "Failed to set compression %s: %s", params.compression, errstr);
            }
            rd_kafka_conf_destroy(conf);
            return NULL;
        }
    }
    if (params.retry_interval)
    {
        rd_kafka_conf_res_t result = rd_kafka_conf_set(
            conf, "retry.backoff.ms",params.retry_interval, errstr, sizeof errstr
        );
        if (result != RD_KAFKA_CONF_OK)
        {
            if (params.log_level)
            {
                openlog("phpkafka", 0, LOG_USER);
                syslog(LOG_ALERT, "Failed to set compression %s: %s", params.compression, errstr);
            }
            rd_kafka_conf_destroy(conf);
            return NULL;
        }
    }
    if (params.reporting == 1)
        rd_kafka_conf_set_dr_cb(conf, kafka_produce_cb_simple);
    else if (params.reporting == 2)
        rd_kafka_conf_set_dr_msg_cb(conf, kafka_produce_detailed_cb);
    r = rd_kafka_new(params.type, conf, errstr, sizeof errstr);
    if (!r)
    {
        if (params.log_level)
        {
            openlog("phpkafka", 0, LOG_USER);
            syslog(LOG_ERR, "Failed to connect to kafka: %s", errstr);
        }
        //destroy config, no connection to use it...
        rd_kafka_conf_destroy(conf);
        return NULL;
    }
    if (!rd_kafka_brokers_add(r, brokers))
    {
        if (params.log_level)
        {
            openlog("phpkafka", 0, LOG_USER);
            syslog(LOG_ERR, "Failed to connect to brokers %s", brokers);
        }
        rd_kafka_destroy(r);
        return NULL;
    }
    return r;
}
Beispiel #13
0
int main_0006_symbols (int argc, char **argv) {

        if (argc < 0 /* always false */) {
                rd_kafka_version();
                rd_kafka_version_str();
		rd_kafka_get_debug_contexts();
		rd_kafka_get_err_descs(NULL, NULL);
                rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR);
		rd_kafka_err2name(RD_KAFKA_RESP_ERR_NO_ERROR);
                rd_kafka_errno2err(EINVAL);
                rd_kafka_errno();
		rd_kafka_last_error();
                rd_kafka_conf_new();
                rd_kafka_conf_destroy(NULL);
                rd_kafka_conf_dup(NULL);
                rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0);
                rd_kafka_conf_set_dr_cb(NULL, NULL);
                rd_kafka_conf_set_dr_msg_cb(NULL, NULL);
                rd_kafka_conf_set_error_cb(NULL, NULL);
                rd_kafka_conf_set_stats_cb(NULL, NULL);
                rd_kafka_conf_set_log_cb(NULL, NULL);
                rd_kafka_conf_set_socket_cb(NULL, NULL);
		rd_kafka_conf_set_rebalance_cb(NULL, NULL);
		rd_kafka_conf_set_offset_commit_cb(NULL, NULL);
		rd_kafka_conf_set_throttle_cb(NULL, NULL);
		rd_kafka_conf_set_default_topic_conf(NULL, NULL);
		rd_kafka_conf_get(NULL, NULL, NULL, NULL);
#ifndef _MSC_VER
		rd_kafka_conf_set_open_cb(NULL, NULL);
#endif
		rd_kafka_conf_set_opaque(NULL, NULL);
                rd_kafka_opaque(NULL);
                rd_kafka_conf_dump(NULL, NULL);
                rd_kafka_topic_conf_dump(NULL, NULL);
                rd_kafka_conf_dump_free(NULL, 0);
                rd_kafka_conf_properties_show(NULL);
                rd_kafka_topic_conf_new();
                rd_kafka_topic_conf_dup(NULL);
                rd_kafka_topic_conf_destroy(NULL);
                rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0);
                rd_kafka_topic_conf_set_opaque(NULL, NULL);
		rd_kafka_topic_conf_get(NULL, NULL, NULL, NULL);
                rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL);
                rd_kafka_topic_partition_available(NULL, 0);
		rd_kafka_topic_opaque(NULL);
                rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL);
                rd_kafka_msg_partitioner_consistent(NULL, NULL, 0, 0, NULL, NULL);
                rd_kafka_msg_partitioner_consistent_random(NULL, NULL, 0, 0, NULL, NULL);
                rd_kafka_new(0, NULL, NULL, 0);
                rd_kafka_destroy(NULL);
                rd_kafka_name(NULL);
		rd_kafka_memberid(NULL);
                rd_kafka_topic_new(NULL, NULL, NULL);
                rd_kafka_topic_destroy(NULL);
                rd_kafka_topic_name(NULL);
                rd_kafka_message_destroy(NULL);
                rd_kafka_message_errstr(NULL);
		rd_kafka_message_timestamp(NULL, NULL);
                rd_kafka_consume_start(NULL, 0, 0);
                rd_kafka_consume_stop(NULL, 0);
                rd_kafka_consume(NULL, 0, 0);
                rd_kafka_consume_batch(NULL, 0, 0, NULL, 0);
                rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL);
                rd_kafka_offset_store(NULL, 0, 0);
                rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL);
                rd_kafka_produce_batch(NULL, 0, 0, NULL, 0);
                rd_kafka_poll(NULL, 0);
                rd_kafka_brokers_add(NULL, NULL);
                /* DEPRECATED: rd_kafka_set_logger(NULL, NULL); */
                rd_kafka_set_log_level(NULL, 0);
                rd_kafka_log_print(NULL, 0, NULL, NULL);
#ifndef _MSC_VER
                rd_kafka_log_syslog(NULL, 0, NULL, NULL);
#endif
                rd_kafka_outq_len(NULL);
                rd_kafka_dump(NULL, NULL);
                rd_kafka_thread_cnt();
                rd_kafka_wait_destroyed(0);
                rd_kafka_metadata(NULL, 0, NULL, NULL, 0);
                rd_kafka_metadata_destroy(NULL);
                rd_kafka_queue_destroy(NULL);
                rd_kafka_consume_start_queue(NULL, 0, 0, NULL);
                rd_kafka_consume_queue(NULL, 0);
                rd_kafka_consume_batch_queue(NULL, 0, NULL, 0);
                rd_kafka_consume_callback_queue(NULL, 0, NULL, NULL);
                rd_kafka_seek(NULL, 0, 0, 0);
                rd_kafka_yield(NULL);
                rd_kafka_mem_free(NULL, NULL);
                rd_kafka_list_groups(NULL, NULL, NULL, 0);
                rd_kafka_group_list_destroy(NULL);

		/* KafkaConsumer API */
		rd_kafka_subscribe(NULL, NULL);
		rd_kafka_unsubscribe(NULL);
		rd_kafka_subscription(NULL, NULL);
		rd_kafka_consumer_poll(NULL, 0);
		rd_kafka_consumer_close(NULL);
		rd_kafka_assign(NULL, NULL);
		rd_kafka_assignment(NULL, NULL);
		rd_kafka_commit(NULL, NULL, 0);
		rd_kafka_commit_message(NULL, NULL, 0);
                rd_kafka_committed(NULL, NULL, 0);
		rd_kafka_position(NULL, NULL);

		/* TopicPartition */
		rd_kafka_topic_partition_list_new(0);
		rd_kafka_topic_partition_list_destroy(NULL);
		rd_kafka_topic_partition_list_add(NULL, NULL, 0);
		rd_kafka_topic_partition_list_add_range(NULL, NULL, 0, 0);
		rd_kafka_topic_partition_list_del(NULL, NULL, 0);
		rd_kafka_topic_partition_list_del_by_idx(NULL, 0);
		rd_kafka_topic_partition_list_copy(NULL);
		rd_kafka_topic_partition_list_set_offset(NULL, NULL, 0, 0);
		rd_kafka_topic_partition_list_find(NULL, NULL, 0);
		rd_kafka_query_watermark_offsets(NULL, NULL, 0, NULL, NULL, 0);
		rd_kafka_get_watermark_offsets(NULL, NULL, 0, NULL, NULL);
        }


	return 0;
}