static void watcher(zhandle_t *zh, int type,
int state, const char *path, void *param)
{
char brokers[1024];
kafka_t* k = (kafka_t*) param;
rd_kafka_topic_conf_t *topic_conf;
if(k->conf == NULL) return;
char* topic = k->conf->topic[0];
if (k->no_brokers || type == ZOO_CHILD_EVENT && strncmp(
path, BROKER_PATH, sizeof(BROKER_PATH) - 1) == 0)
{
brokers[0] = '\0';
set_brokerlist_from_zookeeper(zh, brokers);
if (brokers[0] != '\0' && k->rk != NULL &&
server_list_add_once(&(k->broker_list), brokers))
{
rd_kafka_brokers_add(k->rk, brokers);
k->no_brokers = 0;
rd_kafka_poll(k->rk, 10);
topic_conf = rd_kafka_topic_conf_new();
k->rkt = rd_kafka_topic_new(k->rk, topic, topic_conf);
if(k->rkt == NULL)
printf("topic %s creation failed\n", topic);
}
}
}
int Http::kafka_consumer_::Init(const int partition, const char* topic, const char* brokers, MsgConsume msg_consume) {
char err_str[512];
partition_ = partition;
msg_consume_ = msg_consume;
rd_kafka_conf_t *conf = rd_kafka_conf_new();
if (NULL == conf) {
return -1;
}
rd_kafka_conf_set(conf, "batch.num.messages", "100", err_str, sizeof(err_str));
if (!(rk_ = rd_kafka_new(RD_KAFKA_CONSUMER, conf, err_str, sizeof(err_str)))) {
return -1;
}
rd_kafka_set_log_level(rk_, 1);
if (rd_kafka_brokers_add(rk_, brokers) == 0) {
return -1;
}
rd_kafka_topic_conf_t *topic_conf = rd_kafka_topic_conf_new();
rkt_ = rd_kafka_topic_new(rk_, topic, topic_conf);
if (NULL == rkt_) {
return -1;
}
//RD_KAFKA_OFFSET_BEGINNING，从partition消息队列的开始进行consume；
//RD_KAFKA_OFFSET_END：从partition中的将要produce的下一条信息开始（忽略即当前所有的消息)
if (rd_kafka_consume_start(this->rkt_, partition, RD_KAFKA_OFFSET_END) == -1) {
return -1;
}
return 1;
}
int main (int argc, char **argv) {
if (argc < 0 /* always false */) {
rd_kafka_version();
rd_kafka_version_str();
rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR);
rd_kafka_errno2err(EINVAL);
rd_kafka_conf_new();
rd_kafka_conf_destroy(NULL);
rd_kafka_conf_dup(NULL);
rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_conf_set_dr_cb(NULL, NULL);
rd_kafka_conf_set_error_cb(NULL, NULL);
rd_kafka_conf_set_stats_cb(NULL, NULL);
rd_kafka_conf_set_opaque(NULL, NULL);
rd_kafka_conf_dump(NULL, NULL);
rd_kafka_topic_conf_dump(NULL, NULL);
rd_kafka_conf_dump_free(NULL, 0);
rd_kafka_conf_properties_show(NULL);
rd_kafka_topic_conf_new();
rd_kafka_topic_conf_dup(NULL);
rd_kafka_topic_conf_destroy(NULL);
rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_topic_conf_set_opaque(NULL, NULL);
rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL);
rd_kafka_topic_partition_available(NULL, 0);
rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_new(0, NULL, NULL, 0);
rd_kafka_destroy(NULL);
rd_kafka_name(NULL);
rd_kafka_topic_new(NULL, NULL, NULL);
rd_kafka_topic_destroy(NULL);
rd_kafka_topic_name(NULL);
rd_kafka_message_destroy(NULL);
rd_kafka_message_errstr(NULL);
rd_kafka_consume_start(NULL, 0, 0);
rd_kafka_consume_stop(NULL, 0);
rd_kafka_consume(NULL, 0, 0);
rd_kafka_consume_batch(NULL, 0, 0, NULL, 0);
rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL);
rd_kafka_offset_store(NULL, 0, 0);
rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL);
rd_kafka_poll(NULL, 0);
rd_kafka_brokers_add(NULL, NULL);
rd_kafka_set_logger(NULL, NULL);
rd_kafka_set_log_level(NULL, 0);
rd_kafka_log_print(NULL, 0, NULL, NULL);
rd_kafka_log_syslog(NULL, 0, NULL, NULL);
rd_kafka_outq_len(NULL);
rd_kafka_dump(NULL, NULL);
rd_kafka_thread_cnt();
rd_kafka_wait_destroyed(0);
}
return 0;
}
int consumer_init(const int partition, const char* topic, const char* brokers, Consume_Data consume_data, wrapper_Info* producer_info)
{
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
rd_kafka_t *rk;
char errstr[512];
producer_info->start_offset = RD_KAFKA_OFFSET_END;
producer_info->partition = partition;
if (NULL != consume_data)
producer_info->func_consume_data = consume_data;
else
return CONSUMER_INIT_FAILED;
/* Kafka configuration */
conf = rd_kafka_conf_new();
if (NULL == conf)
return CONSUMER_INIT_FAILED;
if (RD_KAFKA_CONF_OK != rd_kafka_conf_set(conf, "group.id", "one", errstr, sizeof(errstr)))
return CONSUMER_INIT_FAILED;
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new consumer: %s\n",
errstr);
return CONSUMER_INIT_FAILED;
}
rd_kafka_set_log_level(rk, LOG_DEBUG);
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
return CONSUMER_INIT_FAILED;
}
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
/* Create topic */
producer_info->rkt = rd_kafka_topic_new(rk, topic, topic_conf);
producer_info->rk = rk;
/* Start consuming */
if (rd_kafka_consume_start(producer_info->rkt, partition, RD_KAFKA_OFFSET_END) == -1){
fprintf(stderr, "%% Failed to start consuming: %s\n",
rd_kafka_err2str(rd_kafka_errno2err(errno)));
return CONSUMER_INIT_FAILED;
}
return CONSUMER_INIT_SUCCESS;
}
/**
* producer_init_kafka
*
* Initialize the producer instance, setting up the Kafka topic and context.
*
* @param self VALUE Instance of the Producer Ruby object
* @param config HermannInstanceConfig* the instance configuration associated with this producer.
*/
void producer_init_kafka(VALUE self, HermannInstanceConfig* config) {
TRACER("initing (%p)\n", config);
config->quiet = !isatty(STDIN_FILENO);
/* Kafka configuration */
config->conf = rd_kafka_conf_new();
/* Add our `self` to the opaque pointer for error and logging callbacks
*/
rd_kafka_conf_set_opaque(config->conf, (void*)config);
rd_kafka_conf_set_error_cb(config->conf, producer_error_callback);
/* Topic configuration */
config->topic_conf = rd_kafka_topic_conf_new();
/* Set up a message delivery report callback.
* It will be called once for each message, either on successful
* delivery to broker, or upon failure to deliver to broker. */
rd_kafka_conf_set_dr_msg_cb(config->conf, msg_delivered);
/* Create Kafka handle */
if (!(config->rk = rd_kafka_new(RD_KAFKA_PRODUCER,
config->conf,
config->errstr,
sizeof(config->errstr)))) {
/* TODO: Use proper logger */
fprintf(stderr,
"%% Failed to create new producer: %s\n", config->errstr);
rb_raise(rb_eRuntimeError, "%% Failed to create new producer: %s\n", config->errstr);
}
/* Set logger */
rd_kafka_set_logger(config->rk, logger);
rd_kafka_set_log_level(config->rk, LOG_DEBUG);
if (rd_kafka_brokers_add(config->rk, config->brokers) == 0) {
/* TODO: Use proper logger */
fprintf(stderr, "%% No valid brokers specified\n");
rb_raise(rb_eRuntimeError, "No valid brokers specified");
return;
}
/* Create topic */
config->rkt = rd_kafka_topic_new(config->rk, config->topic, config->topic_conf);
/* Set the partitioner callback */
rd_kafka_topic_conf_set_partitioner_cb( config->topic_conf, producer_partitioner_callback);
/* We're now initialized */
config->isInitialized = 1;
TRACER("completed kafka init\n");
}
void p_kafka_set_broker(struct p_kafka_host *kafka_host, char *host, int port)
{
if (kafka_host && kafka_host->rk) {
if (host && port) snprintf(kafka_host->broker, SRVBUFLEN, "%s:%u", host, port);
if (rd_kafka_brokers_add(kafka_host->rk, kafka_host->broker) == 0) {
Log(LOG_WARNING, "WARN ( %s/%s ): Invalid 'kafka_broker_host' or 'kafka_broker_port' specified (%s).\n",
config.name, config.type, kafka_host->broker);
}
}
}
static void watcher(zhandle_t *zh, int type, int state, const char *path, void *watcherCtx)
{
char brokers[1024];
if (type == ZOO_CHILD_EVENT && strncmp(path, BROKER_PATH, sizeof(BROKER_PATH) - 1) == 0)
{
brokers[0] = '\0';
set_brokerlist_from_zookeeper(zh, brokers);
if (brokers[0] != '\0' && rk != NULL)
{
rd_kafka_brokers_add(rk, brokers);
rd_kafka_poll(rk, 10);
}
}
}
rd_kafka_t *kafka_set_connection(rd_kafka_type_t type, const char *b, int report_level, const char *compression)
{
rd_kafka_t *r = NULL;
char *tmp = brokers;
char errstr[512];
rd_kafka_conf_t *conf = rd_kafka_conf_new();
if (!(r = rd_kafka_new(type, conf, errstr, sizeof(errstr)))) {
if (log_level) {
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - failed to create new producer: %s", errstr);
}
exit(1);
}
/* Add brokers */
if (rd_kafka_brokers_add(r, b) == 0) {
if (log_level) {
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "php kafka - No valid brokers specified");
}
exit(1);
}
/* Set up a message delivery report callback.
* It will be called once for each message, either on successful
* delivery to broker, or upon failure to deliver to broker. */
if (type == RD_KAFKA_PRODUCER)
{
if (compression && !strcmp(compression, "none"))
{//silently fail on error ATM...
if (RD_KAFKA_CONF_OK != rd_kafka_conf_set(conf, "compression.codec", compression, errstr, sizeof errstr))
{
if (log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "Failed to set compression to %s", compression);
}
}
}
if (report_level == 1)
rd_kafka_conf_set_dr_cb(conf, kafka_produce_cb_simple);
else if (report_level == 2)
rd_kafka_conf_set_dr_msg_cb(conf, kafka_produce_detailed_cb);
}
rd_kafka_conf_set_error_cb(conf, kafka_err_cb);
if (log_level) {
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - using: %s", brokers);
}
return r;
}
/* Shared logic of Consumer_start and Producer_start */
static PyObject *
RdkHandle_start(RdkHandle *self,
rd_kafka_type_t rdk_type,
const char *brokers,
const char *topic_name)
{
if (RdkHandle_excl_lock(self)) return NULL;
if (self->rdk_handle) {
set_pykafka_error("RdKafkaException", "Already started!");
return RdkHandle_start_fail(self, RdkHandle_stop);
}
/* Configure and start rdk_handle */
char errstr[512];
Py_BEGIN_ALLOW_THREADS /* avoid callbacks deadlocking */
self->rdk_handle = rd_kafka_new(
rdk_type, self->rdk_conf, errstr, sizeof(errstr));
self->rdk_conf = NULL; /* deallocated by rd_kafka_new() */
Py_END_ALLOW_THREADS
if (! self->rdk_handle) {
set_pykafka_error("RdKafkaException", errstr);
return RdkHandle_start_fail(self, RdkHandle_stop);
}
/* Set brokers */
int brokers_added;
Py_BEGIN_ALLOW_THREADS /* avoid callbacks deadlocking */
brokers_added = rd_kafka_brokers_add(self->rdk_handle, brokers);
Py_END_ALLOW_THREADS
if (brokers_added == 0) {
set_pykafka_error("RdKafkaException", "adding brokers failed");
return RdkHandle_start_fail(self, RdkHandle_stop);
}
/* Configure and take out a topic handle */
Py_BEGIN_ALLOW_THREADS /* avoid callbacks deadlocking */
self->rdk_topic_handle = rd_kafka_topic_new(self->rdk_handle,
topic_name,
self->rdk_topic_conf);
self->rdk_topic_conf = NULL; /* deallocated by rd_kafka_topic_new() */
Py_END_ALLOW_THREADS
if (! self->rdk_topic_handle) {
set_pykafka_error_from_code(rd_kafka_errno2err(errno), NULL);
return RdkHandle_start_fail(self, RdkHandle_stop);
}
if (RdkHandle_unlock(self)) return NULL;
Py_INCREF(Py_None);
return Py_None;
}
int producer_init(const int partition, const char* topic, const char* brokers, Msg_Delivered func_msg_delivered, wrapper_Info* producer_info)
{
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
rd_kafka_t *rk;
char errstr[512];
producer_info->partition = partition;
strcpy(producer_info->topic, topic);
if (NULL != func_msg_delivered)
producer_info->func_msg_delivered = func_msg_delivered;
else
return PRODUCER_INIT_FAILED;
/* Kafka configuration */
conf = rd_kafka_conf_new();
if (RD_KAFKA_CONF_OK != rd_kafka_conf_set(conf, "queue.buffering.max.messages", "500000", NULL, 0))
return PRODUCER_INIT_FAILED;
/* Set logger */
rd_kafka_conf_set_log_cb(conf, logger);
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
rd_kafka_conf_set_dr_cb(conf, func_msg_delivered);
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new producer: %s\n",
errstr);
return PRODUCER_INIT_FAILED;
}
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
return PRODUCER_INIT_FAILED;
}
/* Create topic */
producer_info->rkt = rd_kafka_topic_new(rk, topic, topic_conf);
producer_info->rk = rk;
return PRODUCER_INIT_SUCCESS;
}
/**
* @brief setup_kafka initialises librdkafka based on the config
* wrapped in kafka_t
* @param k kafka configuration
**/
int setup_kafka(kafka_t* k)
{
char* brokers = "localhost:9092";
char* zookeepers = NULL;
char* topic = "bloh";
config* fk_conf = (config*) fuse_get_context()->private_data;
if(fk_conf->zookeepers_n > 0) zookeepers = fk_conf->zookeepers[0];
if(fk_conf->brokers_n > 0) brokers = fk_conf->brokers[0];
topic = fk_conf->topic[0];
rd_kafka_topic_conf_t *topic_conf;
rd_kafka_conf_t *conf;
conf = rd_kafka_conf_new();
rd_kafka_conf_set_dr_cb(conf, msg_delivered);
if(rd_kafka_conf_set(conf, "debug", "all",
errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK ||
rd_kafka_conf_set(conf, "batch.num.messages", "1",
errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) {
printf("%% Debug configuration failed: %s: %s\n",
errstr, "all");
return(1);
}
if (!(k->rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new producer: %s\n",
errstr);
return(1);
}
rd_kafka_set_logger(k->rk, logger);
rd_kafka_set_log_level(k->rk, 7);
if (zookeepers != NULL)
{
initialize_zookeeper(zookeepers, k);
return 0;
}
else
{
if (rd_kafka_brokers_add(k->rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
return(1);
}
topic_conf = rd_kafka_topic_conf_new();
k->rkt = rd_kafka_topic_new(k->rk, topic, topic_conf);
if(k->rkt == NULL)
printf("topic %s creation failed\n", topic);
return k->rkt == NULL;
}
}
static VALUE kafka_add_broker(VALUE self, VALUE broker)
{
char *value = StringValueCStr(broker);
int res;
if(!value) {
rb_raise(rb_eArgError, "invalid string");
}
res = rd_kafka_brokers_add(rk, value);
if (res == 0) {
rb_raise(rb_eStandardError, "failed to add any brokers!");
}
return Qnil;
}
int kafka_consumer_::Init(const int partition, const char* topic, const char* brokers, MsgConsume msg_consume)
{
char err_str[512];
partition_ = partition;
msg_consume_ = msg_consume;
printf("partition=%d, topic=%s, brokers=%s\n", partition, topic, brokers);
rd_kafka_conf_t *conf = rd_kafka_conf_new();
if (NULL == conf)
return CONSUMER_INIT_FAILED;
if (RD_KAFKA_CONF_OK != rd_kafka_conf_set(conf, "group.id", "one", err_str, sizeof(err_str)))
return CONSUMER_INIT_FAILED;
// rd_kafka_conf_set(conf, "queued.min.messages", "1000000", NULL, 0);
if (!(rk_ = rd_kafka_new(RD_KAFKA_CONSUMER, conf,
err_str, sizeof(err_str)))) {
printf("%% Failed to create new consumer: %s\n",err_str);
return CONSUMER_INIT_FAILED;
}
//rd_kafka_set_log_level(rk_, LOG_DEBUG);
if (rd_kafka_brokers_add(rk_, brokers) == 0) {
printf("%% No valid brokers specified\n");
return CONSUMER_INIT_FAILED;
}
rd_kafka_topic_conf_t *topic_conf = rd_kafka_topic_conf_new();
rkt_ = rd_kafka_topic_new(rk_, topic, topic_conf);
if (NULL == rkt_)
{
printf("topic creat failed\n");
return CONSUMER_INIT_FAILED;
}
printf("rkt_=%p,partition=%d\n", rkt_, partition);
if (rd_kafka_consume_start(this->rkt_, partition, RD_KAFKA_OFFSET_END) == -1){
printf("Failed to start consuming:");
return CONSUMER_INIT_FAILED;
}
return CONSUMER_INIT_SUCCESS;
}
ngx_int_t ngx_http_kafka_init_worker(ngx_cycle_t *cycle)
{
size_t n;
ngx_http_kafka_main_conf_t *main_conf;
main_conf = ngx_http_cycle_get_module_main_conf(cycle, ngx_http_kafka_module);
main_conf->rkc = rd_kafka_conf_new();
rd_kafka_conf_set_dr_cb(main_conf->rkc, kafka_callback_handler);
main_conf->rk = rd_kafka_new(RD_KAFKA_PRODUCER, main_conf->rkc, NULL, 0);
for (n = 0; n != main_conf->nbrokers; ++n) {
ngx_str_helper(&main_conf->brokers[n], ngx_str_push);
rd_kafka_brokers_add(main_conf->rk, (const char *)main_conf->brokers[n].data);
ngx_str_helper(&main_conf->brokers[n], ngx_str_pop);
}
return 0;
}
void p_kafka_set_broker(struct p_kafka_host *kafka_host, char *host, int port)
{
int ret, multiple_brokers = FALSE;
if (strchr(host, ',')) multiple_brokers = TRUE;
if (kafka_host && kafka_host->rk) {
/* if host is a comma-separated list of brokers, assume port is part of the definition */
if (multiple_brokers) snprintf(kafka_host->broker, SRVBUFLEN, "%s", host);
else {
if (host && port) snprintf(kafka_host->broker, SRVBUFLEN, "%s:%u", host, port);
}
if ((ret = rd_kafka_brokers_add(kafka_host->rk, kafka_host->broker)) == 0) {
Log(LOG_WARNING, "WARN ( %s/%s ): Invalid 'kafka_broker_host' or 'kafka_broker_port' specified (%s).\n",
config.name, config.type, kafka_host->broker);
}
else Log(LOG_DEBUG, "DEBUG ( %s/%s ): %u broker(s) successfully added.\n", config.name, config.type, ret);
}
}
//We're no longer relying on the global rk variable (not thread-safe)
static void kafka_init( rd_kafka_type_t type )
{
if (rk && type != rk_type)
{
rd_kafka_destroy(rk);
rk = NULL;
}
if (rk == NULL)
{
char errstr[512];
rd_kafka_conf_t *conf = rd_kafka_conf_new();
if (!(rk = rd_kafka_new(type, conf, errstr, sizeof(errstr)))) {
if (log_level) {
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - failed to create new producer: %s", errstr);
}
exit(1);
}
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
if (log_level) {
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "php kafka - No valid brokers specified");
}
exit(1);
}
/* Set up a message delivery report callback.
* It will be called once for each message, either on successful
* delivery to broker, or upon failure to deliver to broker. */
if (type == RD_KAFKA_PRODUCER)
rd_kafka_conf_set_dr_cb(conf, kafka_produce_cb_simple);
rd_kafka_conf_set_error_cb(conf, kafka_err_cb);
if (log_level) {
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - using: %s", brokers);
}
}
}
static void om_kafka_init(nx_module_t *module) {
log_debug("Kafka module init entrypoint");
char errstr[512];
nx_om_kafka_conf_t* modconf;
modconf = (nx_om_kafka_conf_t*) module->config;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
/* Kafka configuration */
conf = rd_kafka_conf_new();
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
rd_kafka_conf_set_dr_cb(conf, msg_delivered);
if (rd_kafka_conf_set(conf, "compression.codec", modconf->compression, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) {
log_error("Unable to set compression codec %s", modconf->compression);
} else {
log_info("Kafka compression set to %s", modconf->compression);
}
if (!(modconf->rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)))) {
log_error("Failed to create new producer: %s\n", errstr);
}
if (rd_kafka_brokers_add(modconf->rk, modconf->brokerlist) == 0) {
log_error("No valid brokers specified (%s)", modconf->brokerlist);
} else {
log_info("Kafka brokers set to %s", modconf->brokerlist);
}
modconf->rkt = rd_kafka_topic_new(modconf->rk, modconf->topic, topic_conf);
modconf->kafka_conf = conf;
modconf->topic_conf = topic_conf;
}
/**
* consumer_init_kafka
*
* Initialize the Kafka context and instantiate a consumer.
*
* @param config HermannInstanceConfig* pointer to the instance configuration for this producer or consumer
*/
void consumer_init_kafka(HermannInstanceConfig* config) {
TRACER("configuring rd_kafka\n");
config->quiet = !isatty(STDIN_FILENO);
/* Kafka configuration */
config->conf = rd_kafka_conf_new();
/* Topic configuration */
config->topic_conf = rd_kafka_topic_conf_new();
/* Create Kafka handle */
if (!(config->rk = rd_kafka_new(RD_KAFKA_CONSUMER, config->conf,
config->errstr, sizeof(config->errstr)))) {
fprintf(stderr, "%% Failed to create new consumer: %s\n", config->errstr);
rb_raise(rb_eRuntimeError, "%% Failed to create new consumer: %s\n", config->errstr);
}
/* Set logger */
rd_kafka_set_logger(config->rk, logger);
rd_kafka_set_log_level(config->rk, LOG_DEBUG);
/* Add brokers */
if (rd_kafka_brokers_add(config->rk, config->brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
rb_raise(rb_eRuntimeError, "No valid brokers specified");
return;
}
/* Create topic */
config->rkt = rd_kafka_topic_new(config->rk, config->topic, config->topic_conf);
/* We're now initialized */
config->isInitialized = 1;
}
/*
* kafka_consume_main
*
* Main function for Kafka consumers running as background workers
*/
void
kafka_consume_main(Datum arg)
{
char err_msg[512];
rd_kafka_topic_conf_t *topic_conf;
rd_kafka_t *kafka;
rd_kafka_topic_t *topic;
rd_kafka_message_t **messages;
const struct rd_kafka_metadata *meta;
struct rd_kafka_metadata_topic topic_meta;
rd_kafka_resp_err_t err;
bool found;
Oid id = (Oid) arg;
ListCell *lc;
KafkaConsumerProc *proc = hash_search(consumer_procs, &id, HASH_FIND, &found);
KafkaConsumer consumer;
CopyStmt *copy;
int valid_brokers = 0;
int i;
int my_partitions = 0;
if (!found)
elog(ERROR, "kafka consumer %d not found", id);
pqsignal(SIGTERM, kafka_consume_main_sigterm);
#define BACKTRACE_SEGFAULTS
#ifdef BACKTRACE_SEGFAULTS
pqsignal(SIGSEGV, debug_segfault);
#endif
/* we're now ready to receive signals */
BackgroundWorkerUnblockSignals();
/* give this proc access to the database */
BackgroundWorkerInitializeConnection(NameStr(proc->dbname), NULL);
/* load saved consumer state */
StartTransactionCommand();
load_consumer_state(proc->consumer_id, &consumer);
copy = get_copy_statement(&consumer);
topic_conf = rd_kafka_topic_conf_new();
kafka = rd_kafka_new(RD_KAFKA_CONSUMER, NULL, err_msg, sizeof(err_msg));
rd_kafka_set_logger(kafka, logger);
/*
* Add all brokers currently in pipeline_kafka_brokers
*/
if (consumer.brokers == NIL)
elog(ERROR, "no valid brokers were found");
foreach(lc, consumer.brokers)
valid_brokers += rd_kafka_brokers_add(kafka, lfirst(lc));
if (!valid_brokers)
elog(ERROR, "no valid brokers were found");
/*
* Set up our topic to read from
*/
topic = rd_kafka_topic_new(kafka, consumer.topic, topic_conf);
err = rd_kafka_metadata(kafka, false, topic, &meta, CONSUMER_TIMEOUT);
if (err != RD_KAFKA_RESP_ERR_NO_ERROR)
elog(ERROR, "failed to acquire metadata: %s", rd_kafka_err2str(err));
Assert(meta->topic_cnt == 1);
topic_meta = meta->topics[0];
load_consumer_offsets(&consumer, &topic_meta, proc->offset);
CommitTransactionCommand();
/*
* Begin consuming all partitions that this process is responsible for
*/
for (i = 0; i < topic_meta.partition_cnt; i++)
{
int partition = topic_meta.partitions[i].id;
Assert(partition <= consumer.num_partitions);
if (partition % consumer.parallelism != proc->partition_group)
continue;
elog(LOG, "[kafka consumer] %s <- %s consuming partition %d from offset %ld",
consumer.rel->relname, consumer.topic, partition, consumer.offsets[partition]);
if (rd_kafka_consume_start(topic, partition, consumer.offsets[partition]) == -1)
elog(ERROR, "failed to start consuming: %s", rd_kafka_err2str(rd_kafka_errno2err(errno)));
my_partitions++;
}
/*
* No point doing anything if we don't have any partitions assigned to us
*/
if (my_partitions == 0)
{
elog(LOG, "[kafka consumer] %s <- %s consumer %d doesn't have any partitions to read from",
consumer.rel->relname, consumer.topic, MyProcPid);
goto done;
}
messages = palloc0(sizeof(rd_kafka_message_t) * consumer.batch_size);
/*
* Consume messages until we are terminated
*/
while (!got_sigterm)
{
ssize_t num_consumed;
int i;
int messages_buffered = 0;
int partition;
StringInfoData buf;
bool xact = false;
for (partition = 0; partition < consumer.num_partitions; partition++)
{
if (partition % consumer.parallelism != proc->partition_group)
continue;
num_consumed = rd_kafka_consume_batch(topic, partition,
CONSUMER_TIMEOUT, messages, consumer.batch_size);
if (num_consumed <= 0)
continue;
if (!xact)
{
StartTransactionCommand();
xact = true;
}
initStringInfo(&buf);
for (i = 0; i < num_consumed; i++)
{
if (messages[i]->payload != NULL)
{
appendBinaryStringInfo(&buf, messages[i]->payload, messages[i]->len);
if (buf.len > 0 && buf.data[buf.len - 1] != '\n')
appendStringInfoChar(&buf, '\n');
messages_buffered++;
}
consumer.offsets[partition] = messages[i]->offset;
rd_kafka_message_destroy(messages[i]);
}
}
if (!xact)
{
pg_usleep(1 * 1000);
continue;
}
/* we don't want to die in the event of any errors */
PG_TRY();
{
if (messages_buffered)
execute_copy(copy, &buf);
}
PG_CATCH();
{
elog(LOG, "[kafka consumer] %s <- %s failed to process batch, dropped %d message%s:",
consumer.rel->relname, consumer.topic, (int) num_consumed, (num_consumed == 1 ? "" : "s"));
EmitErrorReport();
FlushErrorState();
AbortCurrentTransaction();
xact = false;
}
PG_END_TRY();
if (!xact)
StartTransactionCommand();
if (messages_buffered)
save_consumer_state(&consumer, proc->partition_group);
CommitTransactionCommand();
}
done:
hash_search(consumer_procs, &id, HASH_REMOVE, NULL);
rd_kafka_topic_destroy(topic);
rd_kafka_destroy(kafka);
rd_kafka_wait_destroyed(CONSUMER_TIMEOUT);
}
int main (int argc, char **argv) {
rd_kafka_topic_t *rkt;
char *brokers = "localhost:9092";
char mode = 'C';
char *topic = NULL;
int partition = RD_KAFKA_PARTITION_UA;
int opt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char errstr[512];
const char *debug = NULL;
int64_t start_offset = 0;
int report_offsets = 0;
int do_conf_dump = 0;
quiet = !isatty(STDIN_FILENO);
/* Kafka configuration */
conf = rd_kafka_conf_new();
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
while ((opt = getopt(argc, argv, "PCLt:p:b:z:qd:o:eX:A")) != -1) {
switch (opt) {
case 'P':
case 'C':
case 'L':
mode = opt;
break;
case 't':
topic = optarg;
break;
case 'p':
partition = atoi(optarg);
break;
case 'b':
brokers = optarg;
break;
case 'z':
if (rd_kafka_conf_set(conf, "compression.codec",
optarg,
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
break;
case 'o':
if (!strcmp(optarg, "end"))
start_offset = RD_KAFKA_OFFSET_END;
else if (!strcmp(optarg, "beginning"))
start_offset = RD_KAFKA_OFFSET_BEGINNING;
else if (!strcmp(optarg, "stored"))
start_offset = RD_KAFKA_OFFSET_STORED;
else if (!strcmp(optarg, "report"))
report_offsets = 1;
else
start_offset = strtoll(optarg, NULL, 10);
break;
case 'e':
exit_eof = 1;
break;
case 'd':
debug = optarg;
break;
case 'q':
quiet = 1;
break;
case 'A':
output = OUTPUT_RAW;
break;
case 'X':
{
char *name, *val;
rd_kafka_conf_res_t res;
if (!strcmp(optarg, "list") ||
!strcmp(optarg, "help")) {
rd_kafka_conf_properties_show(stdout);
exit(0);
}
if (!strcmp(optarg, "dump")) {
do_conf_dump = 1;
continue;
}
name = optarg;
if (!(val = strchr(name, '='))) {
fprintf(stderr, "%% Expected "
"-X property=value, not %s\n", name);
exit(1);
}
*val = '\0';
val++;
res = RD_KAFKA_CONF_UNKNOWN;
/* Try "topic." prefixed properties on topic
* conf first, and then fall through to global if
* it didnt match a topic configuration property. */
if (!strncmp(name, "topic.", strlen("topic.")))
res = rd_kafka_topic_conf_set(topic_conf,
name+
strlen("topic."),
val,
errstr,
sizeof(errstr));
if (res == RD_KAFKA_CONF_UNKNOWN)
res = rd_kafka_conf_set(conf, name, val,
errstr, sizeof(errstr));
if (res != RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
}
break;
default:
goto usage;
}
}
if (do_conf_dump) {
const char **arr;
size_t cnt;
int pass;
for (pass = 0 ; pass < 2 ; pass++) {
int i;
if (pass == 0) {
arr = rd_kafka_conf_dump(conf, &cnt);
printf("# Global config\n");
} else {
printf("# Topic config\n");
arr = rd_kafka_topic_conf_dump(topic_conf,
&cnt);
}
for (i = 0 ; i < cnt ; i += 2)
printf("%s = %s\n",
arr[i], arr[i+1]);
printf("\n");
rd_kafka_conf_dump_free(arr, cnt);
}
exit(0);
}
if (optind != argc || (mode != 'L' && !topic)) {
usage:
fprintf(stderr,
"Usage: %s -C|-P|-L -t <topic> "
"[-p <partition>] [-b <host1:port1,host2:port2,..>]\n"
"\n"
"librdkafka version %s (0x%08x)\n"
"\n"
" Options:\n"
" -C | -P Consumer or Producer mode\n"
" -L Metadata list mode\n"
" -t <topic> Topic to fetch / produce\n"
" -p <num> Partition (random partitioner)\n"
" -b <brokers> Broker address (localhost:9092)\n"
" -z <codec> Enable compression:\n"
" none|gzip|snappy\n"
" -o <offset> Start offset (consumer)\n"
" -o report Report message offsets (producer)\n"
" -e Exit consumer when last message\n"
" in partition has been received.\n"
" -d [facs..] Enable debugging contexts:\n"
" -q Be quiet\n"
" -A Raw payload output (consumer)\n"
" %s\n"
" -X <prop=name> Set arbitrary librdkafka "
"configuration property\n"
" Properties prefixed with \"topic.\" "
"will be set on topic object.\n"
" Use '-X list' to see the full list\n"
" of supported properties.\n"
"\n"
" In Consumer mode:\n"
" writes fetched messages to stdout\n"
" In Producer mode:\n"
" reads messages from stdin and sends to broker\n"
" In List mode:\n"
" queries broker for metadata information, "
"topic is optional.\n"
"\n"
"\n"
"\n",
argv[0],
rd_kafka_version_str(), rd_kafka_version(),
RD_KAFKA_DEBUG_CONTEXTS);
exit(1);
}
signal(SIGINT, stop);
signal(SIGUSR1, sig_usr1);
if (debug &&
rd_kafka_conf_set(conf, "debug", debug, errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% Debug configuration failed: %s: %s\n",
errstr, debug);
exit(1);
}
if (mode == 'P') {
/*
* Producer
*/
char buf[2048];
int sendcnt = 0;
/* Set up a message delivery report callback.
* It will be called once for each message, either on successful
* delivery to broker, or upon failure to deliver to broker. */
/* If offset reporting (-o report) is enabled, use the
* richer dr_msg_cb instead. */
if (report_offsets) {
rd_kafka_topic_conf_set(topic_conf,
"produce.offset.report",
"true", errstr, sizeof(errstr));
rd_kafka_conf_set_dr_msg_cb(conf, msg_delivered2);
} else
rd_kafka_conf_set_dr_cb(conf, msg_delivered);
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new producer: %s\n",
errstr);
exit(1);
}
/* Set logger */
rd_kafka_set_logger(rk, logger);
rd_kafka_set_log_level(rk, LOG_DEBUG);
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Create topic */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (!quiet)
fprintf(stderr,
"%% Type stuff and hit enter to send\n");
while (run && fgets(buf, sizeof(buf), stdin)) {
size_t len = strlen(buf);
if (buf[len-1] == '\n')
buf[--len] = '\0';
/* Send/Produce message. */
if (rd_kafka_produce(rkt, partition,
RD_KAFKA_MSG_F_COPY,
/* Payload and length */
buf, len,
/* Optional key and its length */
NULL, 0,
/* Message opaque, provided in
* delivery report callback as
* msg_opaque. */
NULL) == -1) {
fprintf(stderr,
"%% Failed to produce to topic %s "
"partition %i: %s\n",
rd_kafka_topic_name(rkt), partition,
rd_kafka_err2str(
rd_kafka_errno2err(errno)));
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
continue;
}
if (!quiet)
fprintf(stderr, "%% Sent %zd bytes to topic "
"%s partition %i\n",
len, rd_kafka_topic_name(rkt), partition);
sendcnt++;
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
}
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
/* Wait for messages to be delivered */
while (run && rd_kafka_outq_len(rk) > 0)
rd_kafka_poll(rk, 100);
/* Destroy the handle */
rd_kafka_destroy(rk);
} else if (mode == 'C') {
/*
* Consumer
*/
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new consumer: %s\n",
errstr);
exit(1);
}
/* Set logger */
rd_kafka_set_logger(rk, logger);
rd_kafka_set_log_level(rk, LOG_DEBUG);
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Create topic */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
/* Start consuming */
if (rd_kafka_consume_start(rkt, partition, start_offset) == -1){
fprintf(stderr, "%% Failed to start consuming: %s\n",
rd_kafka_err2str(rd_kafka_errno2err(errno)));
exit(1);
}
while (run) {
rd_kafka_message_t *rkmessage;
/* Consume single message.
* See rdkafka_performance.c for high speed
* consuming of messages. */
rkmessage = rd_kafka_consume(rkt, partition, 1000);
if (!rkmessage) /* timeout */
continue;
msg_consume(rkmessage, NULL);
/* Return message to rdkafka */
rd_kafka_message_destroy(rkmessage);
}
/* Stop consuming */
rd_kafka_consume_stop(rkt, partition);
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
} else if (mode == 'L') {
rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new producer: %s\n",
errstr);
exit(1);
}
/* Set logger */
rd_kafka_set_logger(rk, logger);
rd_kafka_set_log_level(rk, LOG_DEBUG);
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Create topic */
if (topic)
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
else
rkt = NULL;
while (run) {
const struct rd_kafka_metadata *metadata;
/* Fetch metadata */
err = rd_kafka_metadata(rk, rkt ? 0 : 1, rkt,
&metadata, 5000);
if (err != RD_KAFKA_RESP_ERR_NO_ERROR) {
fprintf(stderr,
"%% Failed to acquire metadata: %s\n",
rd_kafka_err2str(err));
run = 0;
break;
}
metadata_print(topic, metadata);
rd_kafka_metadata_destroy(metadata);
run = 0;
}
/* Destroy the handle */
rd_kafka_destroy(rk);
/* Exit right away, dont wait for background cleanup, we haven't
* done anything important anyway. */
exit(err ? 2 : 0);
}
/* Let background threads clean up and terminate cleanly. */
rd_kafka_wait_destroyed(2000);
return 0;
}
int main (int argc, char **argv) {
char *brokers = "localhost";
char mode = 'C';
char *topic = NULL;
const char *key = NULL;
int partition = RD_KAFKA_PARTITION_UA; /* random */
int opt;
int msgcnt = -1;
int sendflags = 0;
char *msgpattern = "librdkafka_performance testing!";
int msgsize = strlen(msgpattern);
const char *debug = NULL;
rd_ts_t now;
char errstr[512];
uint64_t seq = 0;
int seed = time(NULL);
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
const char *compression = "no";
int64_t start_offset = 0;
int batch_size = 0;
/* Kafka configuration */
conf = rd_kafka_conf_new();
rd_kafka_conf_set_error_cb(conf, err_cb);
rd_kafka_conf_set_dr_cb(conf, msg_delivered);
/* Producer config */
rd_kafka_conf_set(conf, "queue.buffering.max.messages", "500000",
NULL, 0);
rd_kafka_conf_set(conf, "message.send.max.retries", "3", NULL, 0);
rd_kafka_conf_set(conf, "retry.backoff.ms", "500", NULL, 0);
/* Consumer config */
/* Tell rdkafka to (try to) maintain 1M messages
* in its internal receive buffers. This is to avoid
* application -> rdkafka -> broker per-message ping-pong
* latency.
* The larger the local queue, the higher the performance.
* Try other values with: ... -X queued.min.messages=1000
*/
rd_kafka_conf_set(conf, "queued.min.messages", "1000000", NULL, 0);
/* Kafka topic configuration */
topic_conf = rd_kafka_topic_conf_new();
rd_kafka_topic_conf_set(topic_conf, "message.timeout.ms", "5000",
NULL, 0);
while ((opt =
getopt(argc, argv,
"PCt:p:b:s:k:c:fi:Dd:m:S:x:R:a:z:o:X:B:eT:q")) != -1) {
switch (opt) {
case 'P':
case 'C':
mode = opt;
break;
case 't':
topic = optarg;
break;
case 'p':
partition = atoi(optarg);
break;
case 'b':
brokers = optarg;
break;
case 's':
msgsize = atoi(optarg);
break;
case 'k':
key = optarg;
break;
case 'c':
msgcnt = atoi(optarg);
break;
case 'D':
sendflags |= RD_KAFKA_MSG_F_FREE;
break;
case 'i':
dispintvl = atoi(optarg);
break;
case 'm':
msgpattern = optarg;
break;
case 'S':
seq = strtoull(optarg, NULL, 10);
do_seq = 1;
break;
case 'x':
exit_after = atoi(optarg);
break;
case 'R':
seed = atoi(optarg);
break;
case 'a':
if (rd_kafka_topic_conf_set(topic_conf,
"request.required.acks",
optarg,
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
break;
case 'B':
batch_size = atoi(optarg);
break;
case 'z':
if (rd_kafka_conf_set(conf, "compression.codec",
optarg,
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
compression = optarg;
break;
case 'o':
start_offset = strtoll(optarg, NULL, 10);
break;
case 'e':
exit_eof = 1;
break;
case 'd':
debug = optarg;
break;
case 'X':
{
char *name, *val;
rd_kafka_conf_res_t res;
if (!strcmp(optarg, "list") ||
!strcmp(optarg, "help")) {
rd_kafka_conf_properties_show(stdout);
exit(0);
}
name = optarg;
if (!(val = strchr(name, '='))) {
fprintf(stderr, "%% Expected "
"-X property=value, not %s\n", name);
exit(1);
}
*val = '\0';
val++;
res = RD_KAFKA_CONF_UNKNOWN;
/* Try "topic." prefixed properties on topic
* conf first, and then fall through to global if
* it didnt match a topic configuration property. */
if (!strncmp(name, "topic.", strlen("topic.")))
res = rd_kafka_topic_conf_set(topic_conf,
name+
strlen("topic"),
val,
errstr,
sizeof(errstr));
if (res == RD_KAFKA_CONF_UNKNOWN)
res = rd_kafka_conf_set(conf, name, val,
errstr, sizeof(errstr));
if (res != RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
}
break;
case 'T':
if (rd_kafka_conf_set(conf, "statistics.interval.ms",
optarg, errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
rd_kafka_conf_set_stats_cb(conf, stats_cb);
break;
case 'q':
quiet = 1;
break;
default:
goto usage;
}
}
if (!topic || optind != argc) {
usage:
fprintf(stderr,
"Usage: %s [-C|-P] -t <topic> "
"[-p <partition>] [-b <broker,broker..>] [options..]\n"
"\n"
" Options:\n"
" -C | -P Consumer or Producer mode\n"
" -t <topic> Topic to fetch / produce\n"
" -p <num> Partition (defaults to random)\n"
" -b <brokers> Broker address list (host[:port],..)\n"
" -s <size> Message size (producer)\n"
" -k <key> Message key (producer)\n"
" -c <cnt> Messages to transmit/receive\n"
" -D Copy/Duplicate data buffer (producer)\n"
" -i <ms> Display interval\n"
" -m <msg> Message payload pattern\n"
" -S <start> Send a sequence number starting at "
"<start> as payload\n"
" -R <seed> Random seed value (defaults to time)\n"
" -a <acks> Required acks (producer): "
"-1, 0, 1, >1\n"
" -B <size> Consume batch size (# of msgs)\n"
" -z <codec> Enable compression:\n"
" none|gzip|snappy\n"
" -o <offset> Start offset (consumer)\n"
" -d [facs..] Enable debugging contexts:\n"
" %s\n"
" -X <prop=name> Set arbitrary librdkafka "
"configuration property\n"
" Properties prefixed with \"topic.\" "
"will be set on topic object.\n"
" Use '-X list' to see the full list\n"
" of supported properties.\n"
" -T <intvl> Enable statistics from librdkafka at "
"specified interval (ms)\n"
" -q Be more quiet\n"
"\n"
" In Consumer mode:\n"
" consumes messages and prints thruput\n"
" If -B <..> is supplied the batch consumer\n"
" mode is used, else the callback mode is used.\n"
"\n"
" In Producer mode:\n"
" writes messages of size -s <..> and prints thruput\n"
"\n",
argv[0],
RD_KAFKA_DEBUG_CONTEXTS);
exit(1);
}
dispintvl *= 1000; /* us */
printf("%% Using random seed %i\n", seed);
srand(seed);
signal(SIGINT, stop);
signal(SIGUSR1, sig_usr1);
if (debug &&
rd_kafka_conf_set(conf, "debug", debug, errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
printf("%% Debug configuration failed: %s: %s\n",
errstr, debug);
exit(1);
}
/* Socket hangups are gracefully handled in librdkafka on socket error
* without the use of signals, so SIGPIPE should be ignored by the
* calling program. */
signal(SIGPIPE, SIG_IGN);
if (msgcnt != -1)
forever = 0;
if (mode == 'P') {
/*
* Producer
*/
char *sbuf;
char *pbuf;
int outq;
int i;
int keylen = key ? strlen(key) : 0;
off_t rof = 0;
size_t plen = strlen(msgpattern);
if (do_seq) {
if (msgsize < strlen("18446744073709551615: ")+1)
msgsize = strlen("18446744073709551615: ")+1;
/* Force duplication of payload */
sendflags |= RD_KAFKA_MSG_F_FREE;
}
sbuf = malloc(msgsize);
/* Copy payload content to new buffer */
while (rof < msgsize) {
size_t xlen = RD_MIN(msgsize-rof, plen);
memcpy(sbuf+rof, msgpattern, xlen);
rof += xlen;
}
if (msgcnt == -1)
printf("%% Sending messages of size %i bytes\n",
msgsize);
else
printf("%% Sending %i messages of size %i bytes\n",
msgcnt, msgsize);
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create Kafka producer: %s\n",
errstr);
exit(1);
}
if (debug)
rd_kafka_set_log_level(rk, 7);
/* Add broker(s) */
if (rd_kafka_brokers_add(rk, brokers) < 1) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Explicitly create topic to avoid per-msg lookups. */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
cnt.t_start = rd_clock();
while (run && (msgcnt == -1 || cnt.msgs < msgcnt)) {
/* Send/Produce message. */
if (do_seq) {
snprintf(sbuf, msgsize-1, "%"PRIu64": ", seq);
seq++;
}
if (sendflags & RD_KAFKA_MSG_F_FREE) {
/* Duplicate memory */
pbuf = malloc(msgsize);
memcpy(pbuf, sbuf, msgsize);
} else
pbuf = sbuf;
cnt.tx++;
while (run &&
rd_kafka_produce(rkt, partition,
sendflags, pbuf, msgsize,
key, keylen, NULL) == -1) {
if (!quiet || errno != ENOBUFS)
printf("produce error: %s%s\n",
strerror(errno),
errno == ENOBUFS ?
" (backpressure)":"");
cnt.tx_err++;
if (errno != ENOBUFS) {
run = 0;
break;
}
now = rd_clock();
if (cnt.t_last + dispintvl <= now) {
printf("%% Backpressure %i "
"(tx %"PRIu64", "
"txerr %"PRIu64")\n",
rd_kafka_outq_len(rk),
cnt.tx, cnt.tx_err);
cnt.t_last = now;
}
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 10);
}
msgs_wait_cnt++;
cnt.msgs++;
cnt.bytes += msgsize;
print_stats(mode, 0, compression);
/* Must poll to handle delivery reports */
rd_kafka_poll(rk, 0);
}
forever = 0;
printf("All messages produced, "
"now waiting for %li deliveries\n",
msgs_wait_cnt);
rd_kafka_dump(stdout, rk);
/* Wait for messages to be delivered */
i = 0;
while (run && rd_kafka_poll(rk, 1000) != -1) {
if (!(i++ % (dispintvl/1000)))
printf("%% Waiting for %li, "
"%i messages in outq "
"to be sent. Abort with Ctrl-c\n",
msgs_wait_cnt,
rd_kafka_outq_len(rk));
}
outq = rd_kafka_outq_len(rk);
printf("%% %i messages in outq\n", outq);
cnt.msgs -= outq;
cnt.bytes -= msgsize * outq;
cnt.t_end = t_end;
if (cnt.tx_err > 0)
printf("%% %"PRIu64" backpressures for %"PRIu64
" produce calls: %.3f%% backpressure rate\n",
cnt.tx_err, cnt.tx,
((double)cnt.tx_err / (double)cnt.tx) * 100.0);
rd_kafka_dump(stdout, rk);
/* Destroy the handle */
rd_kafka_destroy(rk);
} else if (mode == 'C') {
/*
* Consumer
*/
rd_kafka_message_t **rkmessages = NULL;
#if 0 /* Future API */
/* The offset storage file is optional but its presence
* avoids starting all over from offset 0 again when
* the program restarts.
* ZooKeeper functionality will be implemented in future
* versions and then the offset will be stored there instead. */
conf.consumer.offset_file = "."; /* current directory */
/* Indicate to rdkafka that the application is responsible
* for storing the offset. This allows the application to
* successfully handle a message before storing the offset.
* If this flag is not set rdkafka will store the offset
* just prior to returning the message from rd_kafka_consume().
*/
conf.flags |= RD_KAFKA_CONF_F_APP_OFFSET_STORE;
#endif
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create Kafka producer: %s\n",
errstr);
exit(1);
}
if (debug)
rd_kafka_set_log_level(rk, 7);
/* Add broker(s) */
if (rd_kafka_brokers_add(rk, brokers) < 1) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Create topic to consume from */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
/* Batch consumer */
if (batch_size)
rkmessages = malloc(sizeof(*rkmessages) * batch_size);
/* Start consuming */
if (rd_kafka_consume_start(rkt, partition, start_offset) == -1){
fprintf(stderr, "%% Failed to start consuming: %s\n",
strerror(errno));
exit(1);
}
cnt.t_start = rd_clock();
while (run && (msgcnt == -1 || msgcnt > cnt.msgs)) {
/* Consume messages.
* A message may either be a real message, or
* an error signaling (if rkmessage->err is set).
*/
uint64_t latency;
int r;
latency = rd_clock();
if (batch_size) {
int i;
/* Batch fetch mode */
r = rd_kafka_consume_batch(rkt, partition,
1000,
rkmessages,
batch_size);
if (r != -1) {
for (i = 0 ; i < r ; i++) {
msg_consume(rkmessages[i],NULL);
rd_kafka_message_destroy(
rkmessages[i]);
}
}
} else {
/* Callback mode */
r = rd_kafka_consume_callback(rkt, partition,
1000/*timeout*/,
msg_consume,
NULL);
}
cnt.t_latency += rd_clock() - latency;
if (r == -1)
fprintf(stderr, "%% Error: %s\n",
strerror(errno));
print_stats(mode, 0, compression);
/* Poll to handle stats callbacks */
rd_kafka_poll(rk, 0);
}
cnt.t_end = rd_clock();
/* Stop consuming */
rd_kafka_consume_stop(rkt, partition);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
if (batch_size)
free(rkmessages);
/* Destroy the handle */
rd_kafka_destroy(rk);
}
print_stats(mode, 1, compression);
if (cnt.t_latency && cnt.msgs)
printf("%% Average application fetch latency: %"PRIu64"us\n",
cnt.t_latency / cnt.msgs);
/* Let background threads clean up and terminate cleanly. */
rd_kafka_wait_destroyed(2000);
return 0;
}
void kafka_consume(zval* return_value, char* topic, char* offset, int item_count)
{
int read_counter = 0;
if (strlen(offset) != 0) {
if (!strcmp(offset, "end"))
start_offset = RD_KAFKA_OFFSET_END;
else if (!strcmp(offset, "beginning"))
start_offset = RD_KAFKA_OFFSET_BEGINNING;
else if (!strcmp(offset, "stored"))
start_offset = RD_KAFKA_OFFSET_STORED;
else
start_offset = strtoll(offset, NULL, 10);
}
rd_kafka_topic_t *rkt;
char errstr[512];
rd_kafka_conf_t *conf;
/* Kafka configuration */
conf = rd_kafka_conf_new();
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr)))) {
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - failed to create new consumer: %s", errstr);
exit(1);
}
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "php kafka - No valid brokers specified");
exit(1);
}
rd_kafka_topic_conf_t *topic_conf;
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
/* Create topic */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - start_offset: %d and offset passed: %d", start_offset, offset);
/* Start consuming */
if (rd_kafka_consume_start(rkt, partition, start_offset) == -1) {
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - %% Failed to start consuming: %s",
rd_kafka_err2str(rd_kafka_errno2err(errno)));
exit(1);
}
if (item_count != 0) {
read_counter = item_count;
}
while (run) {
if (item_count != 0 && read_counter >= 0) {
read_counter--;
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - read_counter: %d", read_counter);
if (read_counter == -1) {
run = 0;
continue;
}
}
rd_kafka_message_t *rkmessage;
/* Consume single message.
* See rdkafka_performance.c for high speed
* consuming of messages. */
rkmessage = rd_kafka_consume(rkt, partition, 1000);
if (!rkmessage) /* timeout */
continue;
rd_kafka_message_t *rkmessage_return;
rkmessage_return = msg_consume(rkmessage, NULL);
char payload[(int)rkmessage_return->len];
sprintf(payload, "%.*s", (int)rkmessage_return->len, (char *)rkmessage_return->payload);
add_index_string(return_value, (int)rkmessage_return->offset, payload, 1);
/* Return message to rdkafka */
rd_kafka_message_destroy(rkmessage);
}
/* Stop consuming */
rd_kafka_consume_stop(rkt, partition);
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
}
void kafka_produce(char* topic, char* msg, int msg_len)
{
signal(SIGINT, kafka_stop);
signal(SIGPIPE, kafka_stop);
rd_kafka_topic_t *rkt;
int partition = RD_KAFKA_PARTITION_UA;
rd_kafka_topic_conf_t *topic_conf;
if(rk == NULL) {
char errstr[512];
rd_kafka_conf_t *conf;
/* Kafka configuration */
conf = rd_kafka_conf_new();
if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)))) {
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - failed to create new producer: %s", errstr);
exit(1);
}
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "php kafka - No valid brokers specified");
exit(1);
}
/* Set up a message delivery report callback.
* It will be called once for each message, either on successful
* delivery to broker, or upon failure to deliver to broker. */
rd_kafka_conf_set_dr_cb(conf, kafka_msg_delivered);
rd_kafka_conf_set_error_cb(conf, kafka_err_cb);
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - using: %s", brokers);
}
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
/* Create topic */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (rd_kafka_produce(rkt, partition,
RD_KAFKA_MSG_F_COPY,
/* Payload and length */
msg, msg_len,
/* Optional key and its length */
NULL, 0,
/* Message opaque, provided in
* delivery report callback as
* msg_opaque. */
NULL) == -1) {
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_INFO, "phpkafka - %% Failed to produce to topic %s "
"partition %i: %s",
rd_kafka_topic_name(rkt), partition,
rd_kafka_err2str(
rd_kafka_errno2err(errno)));
rd_kafka_poll(rk, 0);
}
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
/* Wait for messages to be delivered */
while (run && rd_kafka_outq_len(rk) > 0)
rd_kafka_poll(rk, 100);
rd_kafka_topic_destroy(rkt);
}
/* transmitter worker */
void *kaf_worker(void *thread_id) {
char buf[MAX_BUF], *b;
int rc=-1, nr, len, l, count=0,kr;
/* kafka connection setup */
char errstr[512];
rd_kafka_t *k;
rd_kafka_topic_t *t;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
int partition = RD_KAFKA_PARTITION_UA;
char *key = NULL;
int keylen = key ? strlen(key) : 0;
/* set up global options */
conf = rd_kafka_conf_new();
rd_kafka_conf_set_error_cb(conf, err_cb);
//rd_kafka_conf_set_throttle_cb(conf, throttle_cb);
rd_kafka_conf_set_stats_cb(conf, stats_cb);
kr = rd_kafka_conf_set(conf, "statistics.interval.ms", "60000", errstr, sizeof(errstr));
if (kr != RD_KAFKA_CONF_OK) {
fprintf(stderr,"error: rd_kafka_conf_set: statistics.interval.ms 60000 => %s\n", errstr);
goto done;
}
char **opt=NULL;
while( (opt=(char **)utarray_next(CF.rdkafka_options,opt))) {
char *eq = strchr(*opt,'=');
if (eq == NULL) {
fprintf(stderr,"error: specify rdkafka params as key=value\n");
goto done;
}
char *k = strdup(*opt), *v;
k[eq-*opt] = '\0';
v = &k[eq-*opt + 1];
if (CF.verbose) fprintf(stderr,"setting %s %s\n", k, v);
kr = rd_kafka_conf_set(conf, k, v, errstr, sizeof(errstr));
if (kr != RD_KAFKA_CONF_OK) {
fprintf(stderr,"error: rd_kafka_conf_set: %s %s => %s\n", k, v, errstr);
goto done;
}
free(k);
}
/* set up topic options */
topic_conf = rd_kafka_topic_conf_new();
opt=NULL;
while( (opt=(char **)utarray_next(CF.rdkafka_topic_options,opt))) {
char *eq = strchr(*opt,'=');
if (eq == NULL) {
fprintf(stderr,"error: specify rdkafka topic params as key=value\n");
goto done;
}
char *k = strdup(*opt), *v;
k[eq-*opt] = '\0';
v = &k[eq-*opt + 1];
if (CF.verbose) fprintf(stderr,"setting %s %s\n", k, v);
kr = rd_kafka_topic_conf_set(topic_conf, k, v, errstr, sizeof(errstr));
if (kr != RD_KAFKA_CONF_OK) {
fprintf(stderr,"error: rd_kafka_conf_set: %s %s => %s\n", k, v, errstr);
goto done;
}
free(k);
}
k = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr));
if (k == NULL) {
fprintf(stderr, "rd_kafka_new: %s\n", errstr);
goto done;
}
if (rd_kafka_brokers_add(k, CF.broker) < 1) {
fprintf(stderr, "invalid broker\n");
goto done;
}
t = rd_kafka_topic_new(k, CF.topic, topic_conf);
while (CF.shutdown == 0) {
len = nn_recv(CF.egress_socket_pull, buf, MAX_BUF, 0);
if (len < 0) {
fprintf(stderr,"nn_recv: %s\n", nn_strerror(errno));
goto done;
}
rc = rd_kafka_produce(t, partition, RD_KAFKA_MSG_F_COPY, buf, len,
key, keylen, NULL);
if (rc == -1) {
fprintf(stderr,"rd_kafka_produce: %s %s\n",
rd_kafka_err2str( rd_kafka_errno2err(errno)),
((errno == ENOBUFS) ? "(backpressure)" : ""));
goto done;
}
// cause rdkafka to invoke optional callbacks (msg delivery reports or error)
if ((++count % 1000) == 0) rd_kafka_poll(k, 0);
if (thread_id == 0) {
/* only emit these stats from the first worker thread (not thread safe) */
ts_add(CF.kaf_bytes_ts, CF.now, &len);
ts_add(CF.kaf_msgs_ts, CF.now, NULL);
}
}
rc = 0;
done:
CF.shutdown = 1;
return NULL;
}
int main_0006_symbols (int argc, char **argv) {
if (argc < 0 /* always false */) {
rd_kafka_version();
rd_kafka_version_str();
rd_kafka_get_debug_contexts();
rd_kafka_get_err_descs(NULL, NULL);
rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR);
rd_kafka_err2name(RD_KAFKA_RESP_ERR_NO_ERROR);
rd_kafka_errno2err(EINVAL);
rd_kafka_errno();
rd_kafka_last_error();
rd_kafka_conf_new();
rd_kafka_conf_destroy(NULL);
rd_kafka_conf_dup(NULL);
rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_conf_set_dr_cb(NULL, NULL);
rd_kafka_conf_set_dr_msg_cb(NULL, NULL);
rd_kafka_conf_set_error_cb(NULL, NULL);
rd_kafka_conf_set_stats_cb(NULL, NULL);
rd_kafka_conf_set_log_cb(NULL, NULL);
rd_kafka_conf_set_socket_cb(NULL, NULL);
rd_kafka_conf_set_rebalance_cb(NULL, NULL);
rd_kafka_conf_set_offset_commit_cb(NULL, NULL);
rd_kafka_conf_set_throttle_cb(NULL, NULL);
rd_kafka_conf_set_default_topic_conf(NULL, NULL);
rd_kafka_conf_get(NULL, NULL, NULL, NULL);
#ifndef _MSC_VER
rd_kafka_conf_set_open_cb(NULL, NULL);
#endif
rd_kafka_conf_set_opaque(NULL, NULL);
rd_kafka_opaque(NULL);
rd_kafka_conf_dump(NULL, NULL);
rd_kafka_topic_conf_dump(NULL, NULL);
rd_kafka_conf_dump_free(NULL, 0);
rd_kafka_conf_properties_show(NULL);
rd_kafka_topic_conf_new();
rd_kafka_topic_conf_dup(NULL);
rd_kafka_topic_conf_destroy(NULL);
rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0);
rd_kafka_topic_conf_set_opaque(NULL, NULL);
rd_kafka_topic_conf_get(NULL, NULL, NULL, NULL);
rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL);
rd_kafka_topic_partition_available(NULL, 0);
rd_kafka_topic_opaque(NULL);
rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_msg_partitioner_consistent(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_msg_partitioner_consistent_random(NULL, NULL, 0, 0, NULL, NULL);
rd_kafka_new(0, NULL, NULL, 0);
rd_kafka_destroy(NULL);
rd_kafka_name(NULL);
rd_kafka_memberid(NULL);
rd_kafka_topic_new(NULL, NULL, NULL);
rd_kafka_topic_destroy(NULL);
rd_kafka_topic_name(NULL);
rd_kafka_message_destroy(NULL);
rd_kafka_message_errstr(NULL);
rd_kafka_message_timestamp(NULL, NULL);
rd_kafka_consume_start(NULL, 0, 0);
rd_kafka_consume_stop(NULL, 0);
rd_kafka_consume(NULL, 0, 0);
rd_kafka_consume_batch(NULL, 0, 0, NULL, 0);
rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL);
rd_kafka_offset_store(NULL, 0, 0);
rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL);
rd_kafka_produce_batch(NULL, 0, 0, NULL, 0);
rd_kafka_poll(NULL, 0);
rd_kafka_brokers_add(NULL, NULL);
/* DEPRECATED: rd_kafka_set_logger(NULL, NULL); */
rd_kafka_set_log_level(NULL, 0);
rd_kafka_log_print(NULL, 0, NULL, NULL);
#ifndef _MSC_VER
rd_kafka_log_syslog(NULL, 0, NULL, NULL);
#endif
rd_kafka_outq_len(NULL);
rd_kafka_dump(NULL, NULL);
rd_kafka_thread_cnt();
rd_kafka_wait_destroyed(0);
rd_kafka_metadata(NULL, 0, NULL, NULL, 0);
rd_kafka_metadata_destroy(NULL);
rd_kafka_queue_destroy(NULL);
rd_kafka_consume_start_queue(NULL, 0, 0, NULL);
rd_kafka_consume_queue(NULL, 0);
rd_kafka_consume_batch_queue(NULL, 0, NULL, 0);
rd_kafka_consume_callback_queue(NULL, 0, NULL, NULL);
rd_kafka_seek(NULL, 0, 0, 0);
rd_kafka_yield(NULL);
rd_kafka_mem_free(NULL, NULL);
rd_kafka_list_groups(NULL, NULL, NULL, 0);
rd_kafka_group_list_destroy(NULL);
/* KafkaConsumer API */
rd_kafka_subscribe(NULL, NULL);
rd_kafka_unsubscribe(NULL);
rd_kafka_subscription(NULL, NULL);
rd_kafka_consumer_poll(NULL, 0);
rd_kafka_consumer_close(NULL);
rd_kafka_assign(NULL, NULL);
rd_kafka_assignment(NULL, NULL);
rd_kafka_commit(NULL, NULL, 0);
rd_kafka_commit_message(NULL, NULL, 0);
rd_kafka_committed(NULL, NULL, 0);
rd_kafka_position(NULL, NULL);
/* TopicPartition */
rd_kafka_topic_partition_list_new(0);
rd_kafka_topic_partition_list_destroy(NULL);
rd_kafka_topic_partition_list_add(NULL, NULL, 0);
rd_kafka_topic_partition_list_add_range(NULL, NULL, 0, 0);
rd_kafka_topic_partition_list_del(NULL, NULL, 0);
rd_kafka_topic_partition_list_del_by_idx(NULL, 0);
rd_kafka_topic_partition_list_copy(NULL);
rd_kafka_topic_partition_list_set_offset(NULL, NULL, 0, 0);
rd_kafka_topic_partition_list_find(NULL, NULL, 0);
rd_kafka_query_watermark_offsets(NULL, NULL, 0, NULL, NULL, 0);
rd_kafka_get_watermark_offsets(NULL, NULL, 0, NULL, NULL);
}
return 0;
}
rd_kafka_t *kafka_get_connection(kafka_connection_params params, const char *brokers)
{
rd_kafka_t *r = NULL;
char errstr[512];
rd_kafka_conf_t *conf = rd_kafka_conf_new();
//set error callback
rd_kafka_conf_set_error_cb(conf, kafka_err_cb);
if (params.type == RD_KAFKA_CONSUMER)
{
if (params.queue_buffer)
rd_kafka_conf_set(conf, "queued.min.messages", params.queue_buffer, NULL, 0);
r = rd_kafka_new(params.type, conf, errstr, sizeof errstr);
if (!r)
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "Failed to connect to kafka: %s", errstr);
}
//destroy config, no connection to use it...
rd_kafka_conf_destroy(conf);
return NULL;
}
if (!rd_kafka_brokers_add(r, brokers))
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "Failed to connect to brokers %s", brokers);
}
rd_kafka_destroy(r);
return NULL;
}
return r;
}
if (params.compression)
{
rd_kafka_conf_res_t result = rd_kafka_conf_set(
conf, "compression.codec",params.compression, errstr, sizeof errstr
);
if (result != RD_KAFKA_CONF_OK)
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ALERT, "Failed to set compression %s: %s", params.compression, errstr);
}
rd_kafka_conf_destroy(conf);
return NULL;
}
}
if (params.retry_count)
{
rd_kafka_conf_res_t result = rd_kafka_conf_set(
conf, "message.send.max.retries",params.retry_count, errstr, sizeof errstr
);
if (result != RD_KAFKA_CONF_OK)
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ALERT, "Failed to set compression %s: %s", params.compression, errstr);
}
rd_kafka_conf_destroy(conf);
return NULL;
}
}
if (params.retry_interval)
{
rd_kafka_conf_res_t result = rd_kafka_conf_set(
conf, "retry.backoff.ms",params.retry_interval, errstr, sizeof errstr
);
if (result != RD_KAFKA_CONF_OK)
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ALERT, "Failed to set compression %s: %s", params.compression, errstr);
}
rd_kafka_conf_destroy(conf);
return NULL;
}
}
if (params.reporting == 1)
rd_kafka_conf_set_dr_cb(conf, kafka_produce_cb_simple);
else if (params.reporting == 2)
rd_kafka_conf_set_dr_msg_cb(conf, kafka_produce_detailed_cb);
r = rd_kafka_new(params.type, conf, errstr, sizeof errstr);
if (!r)
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "Failed to connect to kafka: %s", errstr);
}
//destroy config, no connection to use it...
rd_kafka_conf_destroy(conf);
return NULL;
}
if (!rd_kafka_brokers_add(r, brokers))
{
if (params.log_level)
{
openlog("phpkafka", 0, LOG_USER);
syslog(LOG_ERR, "Failed to connect to brokers %s", brokers);
}
rd_kafka_destroy(r);
return NULL;
}
return r;
}
int main (int argc, char **argv) {
char mode = 'C';
char *brokers = "localhost:9092";
int opt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char errstr[512];
const char *debug = NULL;
int do_conf_dump = 0;
char tmp[16];
rd_kafka_resp_err_t err;
char *group = NULL;
rd_kafka_topic_partition_list_t *topics;
int i;
quiet = !isatty(STDIN_FILENO);
/* Kafka configuration */
conf = rd_kafka_conf_new();
/* Set logger */
rd_kafka_conf_set_log_cb(conf, logger);
/* Quick termination */
snprintf(tmp, sizeof(tmp), "%i", SIGIO);
rd_kafka_conf_set(conf, "internal.termination.signal", tmp, NULL, 0);
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
while ((opt = getopt(argc, argv, "g:b:qd:eX:As:DO")) != -1) {
switch (opt) {
case 'b':
brokers = optarg;
break;
case 'g':
group = optarg;
break;
case 'e':
exit_eof = 1;
break;
case 'd':
debug = optarg;
break;
case 'q':
quiet = 1;
break;
case 'A':
output = OUTPUT_RAW;
break;
case 'X':
{
char *name, *val;
rd_kafka_conf_res_t res;
if (!strcmp(optarg, "list") ||
!strcmp(optarg, "help")) {
rd_kafka_conf_properties_show(stdout);
exit(0);
}
if (!strcmp(optarg, "dump")) {
do_conf_dump = 1;
continue;
}
name = optarg;
if (!(val = strchr(name, '='))) {
fprintf(stderr, "%% Expected "
"-X property=value, not %s\n", name);
exit(1);
}
*val = '\0';
val++;
res = RD_KAFKA_CONF_UNKNOWN;
/* Try "topic." prefixed properties on topic
* conf first, and then fall through to global if
* it didnt match a topic configuration property. */
if (!strncmp(name, "topic.", strlen("topic.")))
res = rd_kafka_topic_conf_set(topic_conf,
name+
strlen("topic."),
val,
errstr,
sizeof(errstr));
if (res == RD_KAFKA_CONF_UNKNOWN)
res = rd_kafka_conf_set(conf, name, val,
errstr, sizeof(errstr));
if (res != RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
}
break;
case 'D':
case 'O':
mode = opt;
break;
default:
goto usage;
}
}
if (do_conf_dump) {
const char **arr;
size_t cnt;
int pass;
for (pass = 0 ; pass < 2 ; pass++) {
if (pass == 0) {
arr = rd_kafka_conf_dump(conf, &cnt);
printf("# Global config\n");
} else {
printf("# Topic config\n");
arr = rd_kafka_topic_conf_dump(topic_conf,
&cnt);
}
for (i = 0 ; i < (int)cnt ; i += 2)
printf("%s = %s\n",
arr[i], arr[i+1]);
printf("\n");
rd_kafka_conf_dump_free(arr, cnt);
}
exit(0);
}
if (strchr("OC", mode) && optind == argc) {
usage:
fprintf(stderr,
"Usage: %s [options] <topic[:part]> <topic[:part]>..\n"
"\n"
"librdkafka version %s (0x%08x)\n"
"\n"
" Options:\n"
" -g <group> Consumer group (%s)\n"
" -b <brokers> Broker address (%s)\n"
" -e Exit consumer when last message\n"
" in partition has been received.\n"
" -D Describe group.\n"
" -O Get commmitted offset(s)\n"
" -d [facs..] Enable debugging contexts:\n"
" %s\n"
" -q Be quiet\n"
" -A Raw payload output (consumer)\n"
" -X <prop=name> Set arbitrary librdkafka "
"configuration property\n"
" Properties prefixed with \"topic.\" "
"will be set on topic object.\n"
" Use '-X list' to see the full list\n"
" of supported properties.\n"
"\n"
"\n",
argv[0],
rd_kafka_version_str(), rd_kafka_version(),
group, brokers,
RD_KAFKA_DEBUG_CONTEXTS);
exit(1);
}
signal(SIGINT, stop);
signal(SIGUSR1, sig_usr1);
if (debug &&
rd_kafka_conf_set(conf, "debug", debug, errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% Debug configuration failed: %s: %s\n",
errstr, debug);
exit(1);
}
/*
* Client/Consumer group
*/
if (strchr("CO", mode)) {
/* Consumer groups require a group id */
if (!group)
group = "rdkafka_consumer_example";
if (rd_kafka_conf_set(conf, "group.id", group,
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
/* Consumer groups always use broker based offset storage */
if (rd_kafka_topic_conf_set(topic_conf, "offset.store.method",
"broker",
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
/* Set default topic config for pattern-matched topics. */
rd_kafka_conf_set_default_topic_conf(conf, topic_conf);
/* Callback called on partition assignment changes */
rd_kafka_conf_set_rebalance_cb(conf, rebalance_cb);
}
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new consumer: %s\n",
errstr);
exit(1);
}
rd_kafka_set_log_level(rk, LOG_DEBUG);
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
if (mode == 'D') {
int r;
/* Describe groups */
r = describe_groups(rk, group);
rd_kafka_destroy(rk);
exit(r == -1 ? 1 : 0);
}
/* Redirect rd_kafka_poll() to consumer_poll() */
rd_kafka_poll_set_consumer(rk);
topics = rd_kafka_topic_partition_list_new(argc - optind);
for (i = optind ; i < argc ; i++) {
/* Parse "topic[:part] */
char *topic = argv[i];
char *t;
int32_t partition = -1;
if ((t = strstr(topic, ":"))) {
*t = '\0';
partition = atoi(t+1);
}
rd_kafka_topic_partition_list_add(topics, topic, partition);
}
if (mode == 'O') {
/* Offset query */
err = rd_kafka_position(rk, topics, 5000);
if (err) {
fprintf(stderr, "%% Failed to fetch offsets: %s\n",
rd_kafka_err2str(err));
exit(1);
}
for (i = 0 ; i < topics->cnt ; i++) {
rd_kafka_topic_partition_t *p = &topics->elems[i];
printf("Topic \"%s\" partition %"PRId32,
p->topic, p->partition);
if (p->err)
printf(" error %s",
rd_kafka_err2str(p->err));
else {
printf(" offset %"PRId64"",
p->offset);
if (p->metadata_size)
printf(" (%d bytes of metadata)",
(int)p->metadata_size);
}
printf("\n");
}
goto done;
}
if ((err = rd_kafka_subscribe(rk, topics))) {
fprintf(stderr, "%% Failed to start consuming topics: %s\n",
rd_kafka_err2str(err));
exit(1);
}
while (run) {
rd_kafka_message_t *rkmessage;
rkmessage = rd_kafka_consumer_poll(rk, 1000);
if (rkmessage) {
msg_consume(rkmessage, NULL);
rd_kafka_message_destroy(rkmessage);
}
}
done:
err = rd_kafka_consumer_close(rk);
if (err)
fprintf(stderr, "%% Failed to close consumer: %s\n",
rd_kafka_err2str(err));
else
fprintf(stderr, "%% Consumer closed\n");
rd_kafka_topic_partition_list_destroy(topics);
/* Destroy handle */
rd_kafka_destroy(rk);
/* Let background threads clean up and terminate cleanly. */
run = 5;
while (run-- > 0 && rd_kafka_wait_destroyed(1000) == -1)
printf("Waiting for librdkafka to decommission\n");
if (run <= 0)
rd_kafka_dump(stdout, rk);
return 0;
}
VCL_STRING
vmod_send_msg(const struct vrt_ctx *ctx, VCL_STRING broker, VCL_STRING topic_name, VCL_STRING name)
{
char *p;
unsigned u, v;
char errstr[512];
char *brokers = (char*)broker;
char *topic = (char*)topic_name;
rd_kafka_topic_t *rkt;
int partition = RD_KAFKA_PARTITION_UA;
u = WS_Reserve(ctx->ws, 0); /* Reserve some work space */
p = ctx->ws->f; /* Front of workspace area */
v = snprintf(p, u, "%s", name);
v++;
if (v > u) {
/* No space, reset and leave */
WS_Release(ctx->ws, 0);
return (NULL);
}
/* Update work space with what we've used */
WS_Release(ctx->ws, v);
/*
* Producer
*/
//char buf[2048];
//char *buf= "essai de test du Test msg sur topic fred";
char *buf= name;
int sendcnt = 0;
/* Set up a message delivery report callback.
* It will be called once for each message, either on successful
* delivery to broker, or upon failure to deliver to broker. */
// rd_kafka_conf_set_dr_cb(conf, msg_delivered);
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new producer: %s\n",
errstr);
exit(1);
}
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Create topic */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
size_t len = strlen(buf);
if (buf[len-1] == '\n')
buf[--len] = '\0';
/* Send/Produce message. */
if (rd_kafka_produce(rkt, partition,
RD_KAFKA_MSG_F_COPY,
/* Payload and length */
buf, len,
/* Optional key and its length */
NULL, 0,
/* Message opaque, provided in
* delivery report callback as
* msg_opaque. */
NULL) == -1) {
fprintf(stderr,
"%% Failed to produce to topic %s "
"partition %i: %s\n",
rd_kafka_topic_name(rkt), partition,
rd_kafka_err2str(
rd_kafka_errno2err(errno)));
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
exit(2);
}
if (!quiet)
fprintf(stderr, "%% Sent %zd bytes to topic "
"%s partition %i\n",
len, rd_kafka_topic_name(rkt), partition);
sendcnt++;
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
/* Wait for messages to be delivered */
while ( rd_kafka_outq_len(rk) > 0)
rd_kafka_poll(rk, 100);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy the handle */
rd_kafka_destroy(rk);
return (p);
}
int main (int argc, char **argv) {
rd_kafka_topic_t *rkt;
char *brokers = 0;
char *topic = NULL;
int partition = RD_KAFKA_PARTITION_UA;
int opt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char errstr[512];
char tmp[16];
char* msg = 0;
char* key = 0;
/* Kafka configuration */
conf = rd_kafka_conf_new();
/* Quick termination */
snprintf(tmp, sizeof(tmp), "%i", SIGIO);
rd_kafka_conf_set(conf, "internal.termination.signal", tmp, NULL, 0);
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
while ((opt = getopt(argc, argv, "t:p:b:z:m:k:")) != -1) {
switch (opt) {
case 't':
topic = optarg;
break;
case 'p':
partition = atoi(optarg);
break;
case 'b':
brokers = optarg;
break;
case 'z':
if (rd_kafka_conf_set(conf, "compression.codec",
optarg,
errstr, sizeof(errstr)) !=
RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% %s\n", errstr);
exit(1);
}
break;
case 'm':
msg = optarg;
break;
case 'k':
key = optarg;
break;
default:
goto usage;
}
}
if (optind != argc || msg == 0 || brokers == 0) {
usage:
fprintf(stderr,
"Usage: %s -t <topic> -m <message>"
"[-p <partition>] [-b <host1:port1,host2:port2,..>] [-k <key>]\n"
"\n"
"librdkafka version %s (0x%08x)\n"
"\n"
" Options:\n"
" -t <topic> Topic to fetch / produce\n"
" -p <num> Partition (random partitioner)\n"
" -b <brokers> Broker address (localhost:9092)\n"
" -z <codec> Enable compression:\n"
" none|gzip|snappy\n"
" -m <msg> Message to send\n"
" -k <key> Key of message\n"
"\n"
"\n",
argv[0],
rd_kafka_version_str(), rd_kafka_version());
exit(1);
}
{
/*
* Producer
*/
/* Set up a message delivery report callback.
* It will be called once for each message, either on successful
* delivery to broker, or upon failure to deliver to broker. */
/* If offset reporting (-o report) is enabled, use the
* richer dr_msg_cb instead. */
rd_kafka_conf_set_dr_cb(conf, msg_delivered);
/* Create Kafka handle */
if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
errstr, sizeof(errstr)))) {
fprintf(stderr,
"%% Failed to create new producer: %s\n",
errstr);
exit(1);
}
/* Set logger */
rd_kafka_set_logger(rk, logger);
rd_kafka_set_log_level(rk, LOG_DEBUG);
/* Add brokers */
if (rd_kafka_brokers_add(rk, brokers) == 0) {
fprintf(stderr, "%% No valid brokers specified\n");
exit(1);
}
/* Create topic */
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
/* Send/Produce message. */
if (rd_kafka_produce(rkt, partition,
RD_KAFKA_MSG_F_COPY,
/* Payload and length */
msg, strlen(msg),
/* Optional key and its length */
key, key?strlen(key): 0,
/* Message opaque, provided in
* delivery report callback as
* msg_opaque. */
NULL) == -1) {
fprintf(stderr,
"%% Failed to produce to topic %s "
"partition %i: %s\n",
rd_kafka_topic_name(rkt), partition,
rd_kafka_err2str(
rd_kafka_errno2err(errno)));
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
exit(1);
}
/* Poll to handle delivery reports */
// rd_kafka_poll(rk, 0);
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
/* Wait for messages to be delivered */
while (rd_kafka_outq_len(rk) > 0)
rd_kafka_poll(rk, 100);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy the handle */
rd_kafka_destroy(rk);
}
/* Let background threads clean up and terminate cleanly. */
rd_kafka_wait_destroyed(2000);
return 0;
}
ECL_KAFKA_API __int32 ECL_KAFKA_CALL getTopicPartitionCount(ICodeContext* ctx, const char* brokers, const char* topic)
{
// We have to use librdkafka's C API for this right now, as the C++ API
// does not expose a topic's metadata. In addition, there is no easy
// link between the exposed C++ objects and the structs used by the
// C API, so we are basically creating a brand-new connection from
// scratch.
__int32 pCount = 0;
char errstr[512];
rd_kafka_conf_t* conf = rd_kafka_conf_new();
rd_kafka_t* rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr));
if (rk)
{
if (rd_kafka_brokers_add(rk, brokers) != 0)
{
rd_kafka_topic_conf_t* topic_conf = rd_kafka_topic_conf_new();
rd_kafka_topic_t* rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (rkt)
{
const struct rd_kafka_metadata* metadata = NULL;
rd_kafka_resp_err_t err = rd_kafka_metadata(rk, 0, rkt, &metadata, 5000);
if (err == RD_KAFKA_RESP_ERR_NO_ERROR)
{
pCount = metadata->topics[0].partition_cnt;
rd_kafka_metadata_destroy(metadata);
}
else
{
if (ctx->queryContextLogger().queryTraceLevel() > 4)
{
DBGLOG("Kafka: Error retrieving metadata from topic: %s @ %s: '%s'", topic, brokers, rd_kafka_err2str(err));
}
}
rd_kafka_topic_destroy(rkt);
}
else
{
if (ctx->queryContextLogger().queryTraceLevel() > 4)
{
DBGLOG("Kafka: Could not create topic object: %s @ %s", topic, brokers);
}
}
}
else
{
if (ctx->queryContextLogger().queryTraceLevel() > 4)
{
DBGLOG("Kafka: Could not add brokers: %s @ %s", topic, brokers);
}
}
rd_kafka_destroy(rk);
}
if (pCount == 0)
{
DBGLOG("Kafka: Unable to retrieve partition count from topic: %s @ %s", topic, brokers);
}
return pCount;
}
