int kafka_consumer_::PullData(std::string& data) { rd_kafka_message_t *rkmessage; int consume_result = PULL_DATA_SUCCESS; printf("217consumer_info->rk:%p, rkt:%p, partition=%d\n", rk_, rkt_, partition_); printf("217 begin poll\n"); rd_kafka_poll(rk_, 0); printf("219begin consume\n"); rkmessage = rd_kafka_consume(rkt_, partition_, 1000); if (!rkmessage) return PULL_DATA_TIMEOUT; printf("rkmessage->err:%d\n", (int)rkmessage->err); if (rkmessage->err) { if (rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION || rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC) return CONSUMER_CONFIG_ERROR; return PULL_DATA_FAILED; } data = std::string((const char*)rkmessage->payload, rkmessage->len); rd_kafka_message_destroy(rkmessage); return consume_result; }
static void consume_messages (uint64_t testid, const char *topic, int32_t partition, int msg_base, int batch_cnt, int msgcnt) { rd_kafka_t *rk; rd_kafka_topic_t *rkt; rd_kafka_conf_t *conf; rd_kafka_topic_conf_t *topic_conf; int i; test_conf_init(&conf, &topic_conf, 20); /* Create kafka instance */ rk = test_create_handle(RD_KAFKA_CONSUMER, conf); rkt = rd_kafka_topic_new(rk, topic, topic_conf); if (!rkt) TEST_FAIL("Failed to create topic: %s\n", rd_kafka_err2str(rd_kafka_last_error())); TEST_SAY("Consuming %i messages from partition %i\n", batch_cnt, partition); /* Consume messages */ if (rd_kafka_consume_start(rkt, partition, RD_KAFKA_OFFSET_TAIL(batch_cnt)) == -1) TEST_FAIL("consume_start(%i, -%i) failed: %s", (int)partition, batch_cnt, rd_kafka_err2str(rd_kafka_last_error())); for (i = 0 ; i < batch_cnt ; i++) { rd_kafka_message_t *rkmessage; rkmessage = rd_kafka_consume(rkt, partition, tmout_multip(5000)); if (!rkmessage) TEST_FAIL("Failed to consume message %i/%i from " "partition %i: %s", i, batch_cnt, (int)partition, rd_kafka_err2str(rd_kafka_last_error())); if (rkmessage->err) TEST_FAIL("Consume message %i/%i from partition %i " "has error: %s", i, batch_cnt, (int)partition, rd_kafka_err2str(rkmessage->err)); verify_consumed_msg(testid, partition, msg_base+i, rkmessage); rd_kafka_message_destroy(rkmessage); } rd_kafka_consume_stop(rkt, partition); /* Destroy topic */ rd_kafka_topic_destroy(rkt); /* Destroy rdkafka instance */ TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk)); rd_kafka_destroy(rk); }
int main (int argc, char **argv) { if (argc < 0 /* always false */) { rd_kafka_version(); rd_kafka_version_str(); rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR); rd_kafka_errno2err(EINVAL); rd_kafka_conf_new(); rd_kafka_conf_destroy(NULL); rd_kafka_conf_dup(NULL); rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0); rd_kafka_conf_set_dr_cb(NULL, NULL); rd_kafka_conf_set_error_cb(NULL, NULL); rd_kafka_conf_set_stats_cb(NULL, NULL); rd_kafka_conf_set_opaque(NULL, NULL); rd_kafka_conf_dump(NULL, NULL); rd_kafka_topic_conf_dump(NULL, NULL); rd_kafka_conf_dump_free(NULL, 0); rd_kafka_conf_properties_show(NULL); rd_kafka_topic_conf_new(); rd_kafka_topic_conf_dup(NULL); rd_kafka_topic_conf_destroy(NULL); rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0); rd_kafka_topic_conf_set_opaque(NULL, NULL); rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL); rd_kafka_topic_partition_available(NULL, 0); rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL); rd_kafka_new(0, NULL, NULL, 0); rd_kafka_destroy(NULL); rd_kafka_name(NULL); rd_kafka_topic_new(NULL, NULL, NULL); rd_kafka_topic_destroy(NULL); rd_kafka_topic_name(NULL); rd_kafka_message_destroy(NULL); rd_kafka_message_errstr(NULL); rd_kafka_consume_start(NULL, 0, 0); rd_kafka_consume_stop(NULL, 0); rd_kafka_consume(NULL, 0, 0); rd_kafka_consume_batch(NULL, 0, 0, NULL, 0); rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL); rd_kafka_offset_store(NULL, 0, 0); rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL); rd_kafka_poll(NULL, 0); rd_kafka_brokers_add(NULL, NULL); rd_kafka_set_logger(NULL, NULL); rd_kafka_set_log_level(NULL, 0); rd_kafka_log_print(NULL, 0, NULL, NULL); rd_kafka_log_syslog(NULL, 0, NULL, NULL); rd_kafka_outq_len(NULL); rd_kafka_dump(NULL, NULL); rd_kafka_thread_cnt(); rd_kafka_wait_destroyed(0); } return 0; }
RdKafka::Message *RdKafka::ConsumerImpl::consume (Topic *topic, int32_t partition, int timeout_ms) { RdKafka::TopicImpl *topicimpl = dynamic_cast<RdKafka::TopicImpl *>(topic); rd_kafka_message_t *rkmessage; rkmessage = rd_kafka_consume(topicimpl->rkt_, partition, timeout_ms); if (!rkmessage) return new RdKafka::MessageImpl(topic, static_cast<RdKafka::ErrorCode> (rd_kafka_last_error())); return new RdKafka::MessageImpl(topic, rkmessage); }
static void *consumer_recv_msg(void *ptr) { rd_kafka_message_t *ret; HermannInstanceConfig *consumerConfig = (HermannInstanceConfig *) ptr; ret = rd_kafka_consume(consumerConfig->rkt, consumerConfig->partition, CONSUMER_RECVMSG_TIMEOUT_MS); if ( ret == NULL ) { if ( errno != ETIMEDOUT ) fprintf(stderr, "%% Error: %s\n", rd_kafka_err2str( rd_kafka_errno2err(errno))); } return (void *) ret; }
int p_kafka_consume_poller(struct p_kafka_host *kafka_host, void **data, int timeout) { rd_kafka_message_t *kafka_msg; int ret = SUCCESS; if (kafka_host && data && timeout) { kafka_msg = rd_kafka_consume(kafka_host->topic, kafka_host->partition, timeout); if (!kafka_msg) ret = FALSE; /* timeout */ else ret = TRUE; /* got data */ (*data) = kafka_msg; } else return ERR; return ret; }
int consumer_pull_data(const wrapper_Info* consumer_info) { rd_kafka_message_t *rkmessage; int consume_result; printf("217consumer_info->rk:%p, rkt:%p, partition=%d\n", consumer_info->rk, consumer_info->rkt, consumer_info->partition); printf("217 begin poll\n"); /* Poll for errors, etc. */ rd_kafka_poll(consumer_info->rk, 0); printf("219begin consume\n"); /* Consume single message. * See rdkafka_performance.c for high speed * consuming of messages. */ rkmessage = rd_kafka_consume(consumer_info->rkt, consumer_info->partition, 1000); if (!rkmessage) /* timeout */ return PULL_DATA_FAILED; if (rkmessage->err) { if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) { // fprintf(stderr, // "%% Consumer reached end of %s [%"PRId32"] " // "message queue at offset %"PRId64"\n", // rd_kafka_topic_name(rkmessage->rkt), // rkmessage->partition, rkmessage->offset); consume_result = PULL_DATA_SUCCESS; } // fprintf(stderr, "%% Consume error for topic \"%s\" [%"PRId32"] " // "offset %"PRId64": %s\n", // rd_kafka_topic_name(rkmessage->rkt), // rkmessage->partition, // rkmessage->offset, // rd_kafka_message_errstr(rkmessage)); if (rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION || rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC) return rkmessage->err; consume_result = PULL_DATA_FAILED; } consumer_info->func_consume_data((const char*)rkmessage->payload, (const int)rkmessage->len); /* Return message to rdkafka */ rd_kafka_message_destroy(rkmessage); return consume_result; }
int Http::kafka_consumer_::PullData(std::string& data) { rd_kafka_message_t *rkmessage; rd_kafka_poll(rk_, 0); rkmessage = rd_kafka_consume(rkt_, partition_, 10000); if (!rkmessage) return -2; if (rkmessage->err) { if (rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION || rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC) { return -1; } return -3; } data = std::string((const char*)rkmessage->payload, rkmessage->len); rd_kafka_message_destroy(rkmessage); return 1; }
int kafka_consume(rd_kafka_t *r, zval* return_value, char* topic, char* offset, int item_count, int partition) { int64_t start_offset = 0; int read_counter = 0, run = 1; //nothing to consume? if (item_count == 0) return 0; if (strlen(offset) != 0) { if (!strcmp(offset, "end")) start_offset = RD_KAFKA_OFFSET_END; else if (!strcmp(offset, "beginning")) start_offset = RD_KAFKA_OFFSET_BEGINNING; else if (!strcmp(offset, "stored")) start_offset = RD_KAFKA_OFFSET_STORED; else { start_offset = strtoll(offset, NULL, 10); if (start_offset < 1) return -1; } } rd_kafka_topic_t *rkt; if (r == NULL) { if (log_level) { openlog("phpkafka", 0, LOG_USER); syslog(LOG_ERR, "phpkafka - no connection to consume from topic: %s", topic); } return -2; } rd_kafka_topic_conf_t *topic_conf; /* Topic configuration */ topic_conf = rd_kafka_topic_conf_new(); /* Create topic */ rkt = rd_kafka_topic_new(r, topic, topic_conf); if (rkt == NULL) { if (log_level) { openlog("phpkafka", 0, LOG_USER); syslog( LOG_ERR, "Failed to consume from topic %s: %s", topic, rd_kafka_err2str( rd_kafka_errno2err(errno) ) ); } return -3; } if (log_level) { openlog("phpkafka", 0, LOG_USER); syslog(LOG_INFO, "phpkafka - start_offset: %"PRId64" and offset passed: %s", start_offset, offset); } /* Start consuming */ if (rd_kafka_consume_start(rkt, partition, start_offset) == -1) { if (log_level) { openlog("phpkafka", 0, LOG_USER); syslog(LOG_INFO, "phpkafka - %% Failed to start consuming: %s", rd_kafka_err2str(rd_kafka_errno2err(errno))); } return -4; } /** * Keep reading until run == 0, or read_counter == item_count */ for (read_counter=0;read_counter!=item_count;++read_counter) { if (run == 0) break; if (log_level) { openlog("phpkafka", 0, LOG_USER); syslog(LOG_INFO, "Consuming, count at %d (of %d - run: %d)", read_counter, item_count, run ); } rd_kafka_message_t *rkmessage = NULL, *rkmessage_return = NULL; /* Consume single message. * See rdkafka_performance.c for high speed * consuming of messages. */ rkmessage = rd_kafka_consume(rkt, partition, 1000); //timeout ONLY if error didn't cause run to be 0 if (!rkmessage) { //break on timeout, makes second call redundant if (errno == ETIMEDOUT) { if (log_level) { openlog("phpkafka", 0, LOG_USER); syslog(LOG_INFO, "Consumer timed out, count at %d (of %d) stop consuming after %d messages", read_counter, item_count, read_counter +1 ); } break; } continue; } rkmessage_return = msg_consume(rkmessage, &run); if (rkmessage_return != NULL) { if ((int) rkmessage_return->len > 0) { //ensure there is a payload char payload[(int) rkmessage_return->len]; sprintf(payload, "%.*s", (int) rkmessage_return->len, (char *) rkmessage_return->payload); add_index_string(return_value, (int) rkmessage_return->offset, payload, 1); } else { //add empty value char payload[1] = "";//empty string add_index_string(return_value, (int) rkmessage_return->offset, payload, 1); } } /* Return message to rdkafka */ rd_kafka_message_destroy(rkmessage); } /* Stop consuming */ rd_kafka_consume_stop(rkt, partition); rd_kafka_topic_destroy(rkt); return 0; }
int main (int argc, char **argv) { rd_kafka_t *rk; char *broker = NULL; char mode = 'C'; char *topic = NULL; int partition = 0; int opt; int msgsize = 1024; int msgcnt = -1; int sendflags = 0; int dispintvl = 1000; struct { rd_ts_t t_start; rd_ts_t t_end; rd_ts_t t_end_send; uint64_t msgs; uint64_t bytes; rd_ts_t t_latency; rd_ts_t t_last; rd_ts_t t_total; } cnt = {}; rd_ts_t now; char *dirstr = ""; while ((opt = getopt(argc, argv, "PCt:p:b:s:c:fi:D")) != -1) { switch (opt) { case 'P': case 'C': mode = opt; break; case 't': topic = optarg; break; case 'p': partition = atoi(optarg); break; case 'b': broker = optarg; break; case 's': msgsize = atoi(optarg); break; case 'c': msgcnt = atoi(optarg); break; case 'D': sendflags |= RD_KAFKA_OP_F_FREE; break; case 'i': dispintvl = atoi(optarg); break; default: goto usage; } } if (!topic || optind != argc) { usage: fprintf(stderr, "Usage: %s [-C|-P] -t <topic> " "[-p <partition>] [-b <broker>] [options..]\n" "\n" " Options:\n" " -C | -P Consumer or Producer mode\n" " -t <topic> Topic to fetch / produce\n" " -p <num> Partition (defaults to 0)\n" " -b <broker> Broker address (localhost:9092)\n" " -s <size> Message size (producer)\n" " -c <cnt> Messages to transmit/receive\n" " -D Copy/Duplicate data buffer (producer)\n" " -i <ms> Display interval\n" "\n" " In Consumer mode:\n" " consumes messages and prints thruput\n" " In Producer mode:\n" " writes messages of size -s <..> and prints thruput\n" "\n", argv[0]); exit(1); } dispintvl *= 1000; /* us */ signal(SIGINT, stop); /* Socket hangups are gracefully handled in librdkafka on socket error * without the use of signals, so SIGPIPE should be ignored by the * calling program. */ signal(SIGPIPE, SIG_IGN); if (mode == 'P') { /* * Producer */ char *sbuf = malloc(msgsize); int endwait; int outq; int i; memset(sbuf, 'R', msgsize); if (msgcnt == -1) printf("%% Sending messages of size %i bytes\n", msgsize); else printf("%% Sending %i messages of size %i bytes\n", msgcnt ,msgsize); /* Create Kafka handle */ if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, broker, NULL))) { perror("kafka_new producer"); exit(1); } cnt.t_start = rd_clock(); while (run && (msgcnt == -1 || cnt.msgs < msgcnt)) { char *pbuf = sbuf; /* Send/Produce message. */ if (sendflags & RD_KAFKA_OP_F_FREE) { /* Duplicate memory */ pbuf = malloc(msgsize); memcpy(pbuf, sbuf, msgsize); } rd_kafka_produce(rk, topic, partition, sendflags, pbuf, msgsize); cnt.msgs++; cnt.bytes += msgsize; now = rd_clock(); if (cnt.t_last + dispintvl <= now) { printf("%% %"PRIu64" messages and %"PRIu64 "bytes: %"PRIu64" msgs/s and " "%.2f Mb/s\n", cnt.msgs, cnt.bytes, (cnt.msgs / (now - cnt.t_start)) * 1000000, (float)(cnt.bytes / (now - cnt.t_start))); cnt.t_last = now; } } /* Wait for messaging to finish. */ i = 0; while (run && rd_kafka_outq_len(rk) > 0) { if (!(i++ % (dispintvl/1000))) printf("%% Waiting for %i messages in outq " "to be sent. Abort with Ctrl-c\n", rd_kafka_outq_len(rk)); usleep(1000); } cnt.t_end_send = rd_clock(); outq = rd_kafka_outq_len(rk); cnt.msgs -= outq; cnt.bytes -= msgsize * outq; cnt.t_end = rd_clock(); /* Since there is no ack for produce messages in 0.7 * we wait some more for any packets in the socket buffers * to be sent. * This is fixed in protocol version 0.8 */ endwait = cnt.msgs * 10; printf("%% Test timers stopped, but waiting %ims more " "for the %"PRIu64 " messages to be transmitted from " "socket buffers.\n" "%% End with Ctrl-c\n", endwait / 1000, cnt.msgs); run = 1; while (run && endwait > 0) { usleep(10000); endwait -= 10000; } /* Destroy the handle */ rd_kafka_destroy(rk); dirstr = "sent"; } else if (mode == 'C') { /* * Consumer */ rd_kafka_op_t *rko; /* Base our configuration on the default config. */ rd_kafka_conf_t conf = rd_kafka_defaultconf; /* The offset storage file is optional but its presence * avoids starting all over from offset 0 again when * the program restarts. * ZooKeeper functionality will be implemented in future * versions and then the offset will be stored there instead. */ conf.consumer.offset_file = "."; /* current directory */ /* Indicate to rdkafka that the application is responsible * for storing the offset. This allows the application to * succesfully handle a message before storing the offset. * If this flag is not set rdkafka will store the offset * just prior to returning the message from rd_kafka_consume(). */ conf.flags |= RD_KAFKA_CONF_F_APP_OFFSET_STORE; /* Tell rdkafka to (try to) maintain 10000 messages * in its internal receive buffers. This is to avoid * application -> rdkafka -> broker per-message ping-pong * latency. */ conf.consumer.replyq_low_thres = 100000; /* Use the consumer convenience function * to create a Kafka handle. */ if (!(rk = rd_kafka_new_consumer(broker, topic, (uint32_t)partition, 0, &conf))) { perror("kafka_new_consumer"); exit(1); } cnt.t_start = rd_clock(); while (run && (msgcnt == -1 || msgcnt > cnt.msgs)) { /* Fetch an "op" which is one of: * - a kafka message (if rko_len>0 && rko_err==0) * - an error (if rko_err) */ uint64_t latency; latency = rd_clock(); if (!(rko = rd_kafka_consume(rk, 1000/*timeout ms*/))) continue; cnt.t_latency += rd_clock() - latency; if (rko->rko_err) fprintf(stderr, "%% Error: %.*s\n", rko->rko_len, rko->rko_payload); else if (rko->rko_len) { cnt.msgs++; cnt.bytes += rko->rko_len; } /* rko_offset contains the offset of the _next_ * message. We store it when we're done processing * the current message. */ if (rko->rko_offset) rd_kafka_offset_store(rk, rko->rko_offset); /* Destroy the op */ rd_kafka_op_destroy(rk, rko); now = rd_clock(); if (cnt.t_last + dispintvl <= now && cnt.t_start + 1000000 < now) { printf("%% %"PRIu64" messages and %"PRIu64 " bytes: %"PRIu64" msgs/s and " "%.2f Mb/s\n", cnt.msgs, cnt.bytes, (cnt.msgs / ((now - cnt.t_start)/1000)) * 1000, (float)(cnt.bytes / ((now - cnt.t_start) / 1000))); cnt.t_last = now; } } cnt.t_end = rd_clock(); /* Destroy the handle */ rd_kafka_destroy(rk); dirstr = "received"; } if (cnt.t_end_send) cnt.t_total = cnt.t_end_send - cnt.t_start; else cnt.t_total = cnt.t_end - cnt.t_start; printf("%% %"PRIu64" messages and %"PRIu64" bytes " "%s in %"PRIu64"ms: %"PRIu64" msgs/s and %.02f Mb/s\n", cnt.msgs, cnt.bytes, dirstr, cnt.t_total / 1000, (cnt.msgs / (cnt.t_total / 1000)) * 1000, (float)(cnt.bytes / (cnt.t_total / 1000))); if (cnt.t_latency) printf("%% Average application fetch latency: %"PRIu64"us\n", cnt.t_latency / cnt.msgs); return 0; }
static void consume_messages (uint64_t testid, const char *topic, int32_t partition, int msg_base, int batch_cnt, int msgcnt) { rd_kafka_t *rk; rd_kafka_topic_t *rkt; rd_kafka_conf_t *conf; rd_kafka_topic_conf_t *topic_conf; char errstr[512]; int i; test_conf_init(&conf, &topic_conf, 20); /* Create kafka instance */ rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr)); if (!rk) TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr); TEST_SAY("Created kafka instance %s\n", rd_kafka_name(rk)); rkt = rd_kafka_topic_new(rk, topic, topic_conf); if (!rkt) TEST_FAIL("Failed to create topic: %s\n", rd_strerror(errno)); TEST_SAY("Consuming %i messages from partition %i\n", batch_cnt, partition); /* Consume messages */ if (rd_kafka_consume_start(rkt, partition, RD_KAFKA_OFFSET_TAIL(batch_cnt)) == -1) TEST_FAIL("consume_start(%i, -%i) failed: %s", (int)partition, batch_cnt, rd_kafka_err2str(rd_kafka_errno2err(errno))); for (i = 0 ; i < batch_cnt ; ) { rd_kafka_message_t *rkmessage; rkmessage = rd_kafka_consume(rkt, partition, 5000); if (!rkmessage) TEST_FAIL("Failed to consume message %i/%i from " "partition %i: %s", i, batch_cnt, (int)partition, rd_kafka_err2str(rd_kafka_errno2err(errno))); if (rkmessage->err) { if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF){ rd_kafka_message_destroy(rkmessage); continue; } TEST_FAIL("Consume message %i/%i from partition %i " "has error: %s", i, batch_cnt, (int)partition, rd_kafka_err2str(rkmessage->err)); } verify_consumed_msg(testid, partition, msg_base+i, rkmessage); rd_kafka_message_destroy(rkmessage); i++; } rd_kafka_consume_stop(rkt, partition); /* Destroy topic */ rd_kafka_topic_destroy(rkt); /* Destroy rdkafka instance */ TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk)); rd_kafka_destroy(rk); }
int main_0006_symbols (int argc, char **argv) { if (argc < 0 /* always false */) { rd_kafka_version(); rd_kafka_version_str(); rd_kafka_get_debug_contexts(); rd_kafka_get_err_descs(NULL, NULL); rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR); rd_kafka_err2name(RD_KAFKA_RESP_ERR_NO_ERROR); rd_kafka_errno2err(EINVAL); rd_kafka_errno(); rd_kafka_last_error(); rd_kafka_conf_new(); rd_kafka_conf_destroy(NULL); rd_kafka_conf_dup(NULL); rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0); rd_kafka_conf_set_dr_cb(NULL, NULL); rd_kafka_conf_set_dr_msg_cb(NULL, NULL); rd_kafka_conf_set_error_cb(NULL, NULL); rd_kafka_conf_set_stats_cb(NULL, NULL); rd_kafka_conf_set_log_cb(NULL, NULL); rd_kafka_conf_set_socket_cb(NULL, NULL); rd_kafka_conf_set_rebalance_cb(NULL, NULL); rd_kafka_conf_set_offset_commit_cb(NULL, NULL); rd_kafka_conf_set_throttle_cb(NULL, NULL); rd_kafka_conf_set_default_topic_conf(NULL, NULL); rd_kafka_conf_get(NULL, NULL, NULL, NULL); #ifndef _MSC_VER rd_kafka_conf_set_open_cb(NULL, NULL); #endif rd_kafka_conf_set_opaque(NULL, NULL); rd_kafka_opaque(NULL); rd_kafka_conf_dump(NULL, NULL); rd_kafka_topic_conf_dump(NULL, NULL); rd_kafka_conf_dump_free(NULL, 0); rd_kafka_conf_properties_show(NULL); rd_kafka_topic_conf_new(); rd_kafka_topic_conf_dup(NULL); rd_kafka_topic_conf_destroy(NULL); rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0); rd_kafka_topic_conf_set_opaque(NULL, NULL); rd_kafka_topic_conf_get(NULL, NULL, NULL, NULL); rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL); rd_kafka_topic_partition_available(NULL, 0); rd_kafka_topic_opaque(NULL); rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL); rd_kafka_msg_partitioner_consistent(NULL, NULL, 0, 0, NULL, NULL); rd_kafka_msg_partitioner_consistent_random(NULL, NULL, 0, 0, NULL, NULL); rd_kafka_new(0, NULL, NULL, 0); rd_kafka_destroy(NULL); rd_kafka_name(NULL); rd_kafka_memberid(NULL); rd_kafka_topic_new(NULL, NULL, NULL); rd_kafka_topic_destroy(NULL); rd_kafka_topic_name(NULL); rd_kafka_message_destroy(NULL); rd_kafka_message_errstr(NULL); rd_kafka_message_timestamp(NULL, NULL); rd_kafka_consume_start(NULL, 0, 0); rd_kafka_consume_stop(NULL, 0); rd_kafka_consume(NULL, 0, 0); rd_kafka_consume_batch(NULL, 0, 0, NULL, 0); rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL); rd_kafka_offset_store(NULL, 0, 0); rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL); rd_kafka_produce_batch(NULL, 0, 0, NULL, 0); rd_kafka_poll(NULL, 0); rd_kafka_brokers_add(NULL, NULL); /* DEPRECATED: rd_kafka_set_logger(NULL, NULL); */ rd_kafka_set_log_level(NULL, 0); rd_kafka_log_print(NULL, 0, NULL, NULL); #ifndef _MSC_VER rd_kafka_log_syslog(NULL, 0, NULL, NULL); #endif rd_kafka_outq_len(NULL); rd_kafka_dump(NULL, NULL); rd_kafka_thread_cnt(); rd_kafka_wait_destroyed(0); rd_kafka_metadata(NULL, 0, NULL, NULL, 0); rd_kafka_metadata_destroy(NULL); rd_kafka_queue_destroy(NULL); rd_kafka_consume_start_queue(NULL, 0, 0, NULL); rd_kafka_consume_queue(NULL, 0); rd_kafka_consume_batch_queue(NULL, 0, NULL, 0); rd_kafka_consume_callback_queue(NULL, 0, NULL, NULL); rd_kafka_seek(NULL, 0, 0, 0); rd_kafka_yield(NULL); rd_kafka_mem_free(NULL, NULL); rd_kafka_list_groups(NULL, NULL, NULL, 0); rd_kafka_group_list_destroy(NULL); /* KafkaConsumer API */ rd_kafka_subscribe(NULL, NULL); rd_kafka_unsubscribe(NULL); rd_kafka_subscription(NULL, NULL); rd_kafka_consumer_poll(NULL, 0); rd_kafka_consumer_close(NULL); rd_kafka_assign(NULL, NULL); rd_kafka_assignment(NULL, NULL); rd_kafka_commit(NULL, NULL, 0); rd_kafka_commit_message(NULL, NULL, 0); rd_kafka_committed(NULL, NULL, 0); rd_kafka_position(NULL, NULL); /* TopicPartition */ rd_kafka_topic_partition_list_new(0); rd_kafka_topic_partition_list_destroy(NULL); rd_kafka_topic_partition_list_add(NULL, NULL, 0); rd_kafka_topic_partition_list_add_range(NULL, NULL, 0, 0); rd_kafka_topic_partition_list_del(NULL, NULL, 0); rd_kafka_topic_partition_list_del_by_idx(NULL, 0); rd_kafka_topic_partition_list_copy(NULL); rd_kafka_topic_partition_list_set_offset(NULL, NULL, 0, 0); rd_kafka_topic_partition_list_find(NULL, NULL, 0); rd_kafka_query_watermark_offsets(NULL, NULL, 0, NULL, NULL, 0); rd_kafka_get_watermark_offsets(NULL, NULL, 0, NULL, NULL); } return 0; }
int main (int argc, char **argv) { rd_kafka_t *rk; char *broker = NULL; char mode = 'C'; char *topic = NULL; int partition = 0; int opt; while ((opt = getopt(argc, argv, "PCt:p:b:")) != -1) { switch (opt) { case 'P': case 'C': mode = opt; break; case 't': topic = optarg; break; case 'p': partition = atoi(optarg); break; case 'b': broker = optarg; break; default: goto usage; } } if (!topic || optind != argc) { usage: fprintf(stderr, "Usage: %s [-C|-P] -t <topic> " "[-p <partition>] [-b <broker>]\n" "\n" " Options:\n" " -C | -P Consumer or Producer mode\n" " -t <topic> Topic to fetch / produce\n" " -p <num> Partition (defaults to 0)\n" " -b <broker> Broker address (localhost:9092)\n" "\n" " In Consumer mode:\n" " writes fetched messages to stdout\n" " In Producer mode:\n" " reads messages from stdin and sends to broker\n" "\n", argv[0]); exit(1); } signal(SIGINT, stop); if (mode == 'P') { /* * Producer */ char buf[2048]; int sendcnt = 0; /* Create Kafka handle */ if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, broker, NULL))) { perror("kafka_new producer"); exit(1); } fprintf(stderr, "%% Type stuff and hit enter to send\n"); while (run && (fgets(buf, sizeof(buf), stdin))) { int len = strlen(buf); char *opbuf = malloc(len + 1); strncpy(opbuf, buf, len + 1); /* Send/Produce message. */ rd_kafka_produce(rk, topic, partition, RD_KAFKA_OP_F_FREE, opbuf, len); fprintf(stderr, "%% Sent %i bytes to topic " "%s partition %i\n", len, topic, partition); sendcnt++; } /* Wait for messaging to finish. */ while (rd_kafka_outq_len(rk) > 0) usleep(50000); /* Since there is no ack for produce messages in 0.7 * we wait some more for any packets to be sent. * This is fixed in protocol version 0.8 */ if (sendcnt > 0) usleep(500000); /* Destroy the handle */ rd_kafka_destroy(rk); } else if (mode == 'C') { /* * Consumer */ rd_kafka_op_t *rko; /* Base our configuration on the default config. */ rd_kafka_conf_t conf = rd_kafka_defaultconf; /* The offset storage file is optional but its presence * avoids starting all over from offset 0 again when * the program restarts. * ZooKeeper functionality will be implemented in future * versions and then the offset will be stored there instead. */ conf.consumer.offset_file = "."; /* current directory */ /* Indicate to rdkafka that the application is responsible * for storing the offset. This allows the application to * succesfully handle a message before storing the offset. * If this flag is not set rdkafka will store the offset * just prior to returning the message from rd_kafka_consume(). */ conf.flags |= RD_KAFKA_CONF_F_APP_OFFSET_STORE; /* Use the consumer convenience function * to create a Kafka handle. */ if (!(rk = rd_kafka_new_consumer(broker, topic, (uint32_t)partition, 0, &conf))) { perror("kafka_new_consumer"); exit(1); } while (run) { /* Fetch an "op" which is one of: * - a kafka message (if rko_len>0 && rko_err==0) * - an error (if rko_err) */ if (!(rko = rd_kafka_consume(rk, 1000/*timeout ms*/))) continue; if (rko->rko_err) fprintf(stderr, "%% Error: %.*s\n", rko->rko_len, rko->rko_payload); else if (rko->rko_len) { fprintf(stderr, "%% Message with " "next-offset %"PRIu64" is %i bytes\n", rko->rko_offset, rko->rko_len); hexdump(stdout, "Message", rko->rko_payload, rko->rko_len); } /* rko_offset contains the offset of the _next_ * message. We store it when we're done processing * the current message. */ if (rko->rko_offset) rd_kafka_offset_store(rk, rko->rko_offset); /* Destroy the op */ rd_kafka_op_destroy(rk, rko); } /* Destroy the handle */ rd_kafka_destroy(rk); } return 0; }
void kafka_consume(zval* return_value, char* topic, char* offset, int item_count) { int read_counter = 0; if (strlen(offset) != 0) { if (!strcmp(offset, "end")) start_offset = RD_KAFKA_OFFSET_END; else if (!strcmp(offset, "beginning")) start_offset = RD_KAFKA_OFFSET_BEGINNING; else if (!strcmp(offset, "stored")) start_offset = RD_KAFKA_OFFSET_STORED; else start_offset = strtoll(offset, NULL, 10); } rd_kafka_topic_t *rkt; char errstr[512]; rd_kafka_conf_t *conf; /* Kafka configuration */ conf = rd_kafka_conf_new(); /* Create Kafka handle */ if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr)))) { openlog("phpkafka", 0, LOG_USER); syslog(LOG_INFO, "phpkafka - failed to create new consumer: %s", errstr); exit(1); } /* Add brokers */ if (rd_kafka_brokers_add(rk, brokers) == 0) { openlog("phpkafka", 0, LOG_USER); syslog(LOG_INFO, "php kafka - No valid brokers specified"); exit(1); } rd_kafka_topic_conf_t *topic_conf; /* Topic configuration */ topic_conf = rd_kafka_topic_conf_new(); /* Create topic */ rkt = rd_kafka_topic_new(rk, topic, topic_conf); openlog("phpkafka", 0, LOG_USER); syslog(LOG_INFO, "phpkafka - start_offset: %d and offset passed: %d", start_offset, offset); /* Start consuming */ if (rd_kafka_consume_start(rkt, partition, start_offset) == -1) { openlog("phpkafka", 0, LOG_USER); syslog(LOG_INFO, "phpkafka - %% Failed to start consuming: %s", rd_kafka_err2str(rd_kafka_errno2err(errno))); exit(1); } if (item_count != 0) { read_counter = item_count; } while (run) { if (item_count != 0 && read_counter >= 0) { read_counter--; openlog("phpkafka", 0, LOG_USER); syslog(LOG_INFO, "phpkafka - read_counter: %d", read_counter); if (read_counter == -1) { run = 0; continue; } } rd_kafka_message_t *rkmessage; /* Consume single message. * See rdkafka_performance.c for high speed * consuming of messages. */ rkmessage = rd_kafka_consume(rkt, partition, 1000); if (!rkmessage) /* timeout */ continue; rd_kafka_message_t *rkmessage_return; rkmessage_return = msg_consume(rkmessage, NULL); char payload[(int)rkmessage_return->len]; sprintf(payload, "%.*s", (int)rkmessage_return->len, (char *)rkmessage_return->payload); add_index_string(return_value, (int)rkmessage_return->offset, payload, 1); /* Return message to rdkafka */ rd_kafka_message_destroy(rkmessage); } /* Stop consuming */ rd_kafka_consume_stop(rkt, partition); rd_kafka_topic_destroy(rkt); rd_kafka_destroy(rk); }
/** * Returns offset of the last message consumed */ int64_t test_consume_msgs (const char *what, rd_kafka_topic_t *rkt, uint64_t testid, int32_t partition, int64_t offset, int exp_msg_base, int exp_cnt, int parse_fmt) { int cnt = 0; int msg_next = exp_msg_base; int fails = 0; int64_t offset_last = -1; test_timing_t t_first, t_all; TEST_SAY("%s: consume_msgs: %s [%"PRId32"]: expect msg #%d..%d " "at offset %"PRId64"\n", what, rd_kafka_topic_name(rkt), partition, exp_msg_base, exp_cnt, offset); if (offset != TEST_NO_SEEK) { rd_kafka_resp_err_t err; test_timing_t t_seek; TIMING_START(&t_seek, "SEEK"); if ((err = rd_kafka_seek(rkt, partition, offset, 5000))) TEST_FAIL("%s: consume_msgs: %s [%"PRId32"]: " "seek to %"PRId64" failed: %s\n", what, rd_kafka_topic_name(rkt), partition, offset, rd_kafka_err2str(err)); TIMING_STOP(&t_seek); TEST_SAY("%s: seeked to offset %"PRId64"\n", what, offset); } TIMING_START(&t_first, "FIRST MSG"); TIMING_START(&t_all, "ALL MSGS"); while (cnt < exp_cnt) { rd_kafka_message_t *rkmessage; int msg_id; rkmessage = rd_kafka_consume(rkt, partition, 5000); if (!rkmessage) TEST_FAIL("%s: consume_msgs: %s [%"PRId32"]: " "expected msg #%d (%d/%d): timed out\n", what, rd_kafka_topic_name(rkt), partition, msg_next, cnt, exp_cnt); if (rkmessage->err) TEST_FAIL("%s: consume_msgs: %s [%"PRId32"]: " "expected msg #%d (%d/%d): got error: %s\n", what, rd_kafka_topic_name(rkt), partition, msg_next, cnt, exp_cnt, rd_kafka_err2str(rkmessage->err)); if (cnt == 0) TIMING_STOP(&t_first); if (parse_fmt) test_msg_parse(testid, rkmessage->key, rkmessage->key_len, partition, &msg_id); else msg_id = 0; if (test_level >= 3) TEST_SAY("%s: consume_msgs: %s [%"PRId32"]: " "got msg #%d at offset %"PRId64 " (expect #%d at offset %"PRId64")\n", what, rd_kafka_topic_name(rkt), partition, msg_id, rkmessage->offset, msg_next, offset >= 0 ? offset + cnt : -1); if (parse_fmt && msg_id != msg_next) { TEST_SAY("%s: consume_msgs: %s [%"PRId32"]: " "expected msg #%d (%d/%d): got msg #%d\n", what, rd_kafka_topic_name(rkt), partition, msg_next, cnt, exp_cnt, msg_id); fails++; } cnt++; msg_next++; offset_last = rkmessage->offset; rd_kafka_message_destroy(rkmessage); } TIMING_STOP(&t_all); if (fails) TEST_FAIL("%s: consume_msgs: %s [%"PRId32"]: %d failures\n", what, rd_kafka_topic_name(rkt), partition, fails); TEST_SAY("%s: consume_msgs: %s [%"PRId32"]: " "%d/%d messages consumed succesfully\n", what, rd_kafka_topic_name(rkt), partition, cnt, exp_cnt); return offset_last; }
int main (int argc, char **argv) { rd_kafka_topic_t *rkt; char *brokers = "localhost:9092"; char mode = 'C'; char *topic = NULL; int partition = RD_KAFKA_PARTITION_UA; int opt; rd_kafka_conf_t *conf; rd_kafka_topic_conf_t *topic_conf; char errstr[512]; const char *debug = NULL; int64_t start_offset = 0; int report_offsets = 0; int do_conf_dump = 0; quiet = !isatty(STDIN_FILENO); /* Kafka configuration */ conf = rd_kafka_conf_new(); /* Topic configuration */ topic_conf = rd_kafka_topic_conf_new(); while ((opt = getopt(argc, argv, "PCLt:p:b:z:qd:o:eX:A")) != -1) { switch (opt) { case 'P': case 'C': case 'L': mode = opt; break; case 't': topic = optarg; break; case 'p': partition = atoi(optarg); break; case 'b': brokers = optarg; break; case 'z': if (rd_kafka_conf_set(conf, "compression.codec", optarg, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) { fprintf(stderr, "%% %s\n", errstr); exit(1); } break; case 'o': if (!strcmp(optarg, "end")) start_offset = RD_KAFKA_OFFSET_END; else if (!strcmp(optarg, "beginning")) start_offset = RD_KAFKA_OFFSET_BEGINNING; else if (!strcmp(optarg, "stored")) start_offset = RD_KAFKA_OFFSET_STORED; else if (!strcmp(optarg, "report")) report_offsets = 1; else start_offset = strtoll(optarg, NULL, 10); break; case 'e': exit_eof = 1; break; case 'd': debug = optarg; break; case 'q': quiet = 1; break; case 'A': output = OUTPUT_RAW; break; case 'X': { char *name, *val; rd_kafka_conf_res_t res; if (!strcmp(optarg, "list") || !strcmp(optarg, "help")) { rd_kafka_conf_properties_show(stdout); exit(0); } if (!strcmp(optarg, "dump")) { do_conf_dump = 1; continue; } name = optarg; if (!(val = strchr(name, '='))) { fprintf(stderr, "%% Expected " "-X property=value, not %s\n", name); exit(1); } *val = '\0'; val++; res = RD_KAFKA_CONF_UNKNOWN; /* Try "topic." prefixed properties on topic * conf first, and then fall through to global if * it didnt match a topic configuration property. */ if (!strncmp(name, "topic.", strlen("topic."))) res = rd_kafka_topic_conf_set(topic_conf, name+ strlen("topic."), val, errstr, sizeof(errstr)); if (res == RD_KAFKA_CONF_UNKNOWN) res = rd_kafka_conf_set(conf, name, val, errstr, sizeof(errstr)); if (res != RD_KAFKA_CONF_OK) { fprintf(stderr, "%% %s\n", errstr); exit(1); } } break; default: goto usage; } } if (do_conf_dump) { const char **arr; size_t cnt; int pass; for (pass = 0 ; pass < 2 ; pass++) { int i; if (pass == 0) { arr = rd_kafka_conf_dump(conf, &cnt); printf("# Global config\n"); } else { printf("# Topic config\n"); arr = rd_kafka_topic_conf_dump(topic_conf, &cnt); } for (i = 0 ; i < cnt ; i += 2) printf("%s = %s\n", arr[i], arr[i+1]); printf("\n"); rd_kafka_conf_dump_free(arr, cnt); } exit(0); } if (optind != argc || (mode != 'L' && !topic)) { usage: fprintf(stderr, "Usage: %s -C|-P|-L -t <topic> " "[-p <partition>] [-b <host1:port1,host2:port2,..>]\n" "\n" "librdkafka version %s (0x%08x)\n" "\n" " Options:\n" " -C | -P Consumer or Producer mode\n" " -L Metadata list mode\n" " -t <topic> Topic to fetch / produce\n" " -p <num> Partition (random partitioner)\n" " -b <brokers> Broker address (localhost:9092)\n" " -z <codec> Enable compression:\n" " none|gzip|snappy\n" " -o <offset> Start offset (consumer)\n" " -o report Report message offsets (producer)\n" " -e Exit consumer when last message\n" " in partition has been received.\n" " -d [facs..] Enable debugging contexts:\n" " -q Be quiet\n" " -A Raw payload output (consumer)\n" " %s\n" " -X <prop=name> Set arbitrary librdkafka " "configuration property\n" " Properties prefixed with \"topic.\" " "will be set on topic object.\n" " Use '-X list' to see the full list\n" " of supported properties.\n" "\n" " In Consumer mode:\n" " writes fetched messages to stdout\n" " In Producer mode:\n" " reads messages from stdin and sends to broker\n" " In List mode:\n" " queries broker for metadata information, " "topic is optional.\n" "\n" "\n" "\n", argv[0], rd_kafka_version_str(), rd_kafka_version(), RD_KAFKA_DEBUG_CONTEXTS); exit(1); } signal(SIGINT, stop); signal(SIGUSR1, sig_usr1); if (debug && rd_kafka_conf_set(conf, "debug", debug, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) { fprintf(stderr, "%% Debug configuration failed: %s: %s\n", errstr, debug); exit(1); } if (mode == 'P') { /* * Producer */ char buf[2048]; int sendcnt = 0; /* Set up a message delivery report callback. * It will be called once for each message, either on successful * delivery to broker, or upon failure to deliver to broker. */ /* If offset reporting (-o report) is enabled, use the * richer dr_msg_cb instead. */ if (report_offsets) { rd_kafka_topic_conf_set(topic_conf, "produce.offset.report", "true", errstr, sizeof(errstr)); rd_kafka_conf_set_dr_msg_cb(conf, msg_delivered2); } else rd_kafka_conf_set_dr_cb(conf, msg_delivered); /* Create Kafka handle */ if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)))) { fprintf(stderr, "%% Failed to create new producer: %s\n", errstr); exit(1); } /* Set logger */ rd_kafka_set_logger(rk, logger); rd_kafka_set_log_level(rk, LOG_DEBUG); /* Add brokers */ if (rd_kafka_brokers_add(rk, brokers) == 0) { fprintf(stderr, "%% No valid brokers specified\n"); exit(1); } /* Create topic */ rkt = rd_kafka_topic_new(rk, topic, topic_conf); if (!quiet) fprintf(stderr, "%% Type stuff and hit enter to send\n"); while (run && fgets(buf, sizeof(buf), stdin)) { size_t len = strlen(buf); if (buf[len-1] == '\n') buf[--len] = '\0'; /* Send/Produce message. */ if (rd_kafka_produce(rkt, partition, RD_KAFKA_MSG_F_COPY, /* Payload and length */ buf, len, /* Optional key and its length */ NULL, 0, /* Message opaque, provided in * delivery report callback as * msg_opaque. */ NULL) == -1) { fprintf(stderr, "%% Failed to produce to topic %s " "partition %i: %s\n", rd_kafka_topic_name(rkt), partition, rd_kafka_err2str( rd_kafka_errno2err(errno))); /* Poll to handle delivery reports */ rd_kafka_poll(rk, 0); continue; } if (!quiet) fprintf(stderr, "%% Sent %zd bytes to topic " "%s partition %i\n", len, rd_kafka_topic_name(rkt), partition); sendcnt++; /* Poll to handle delivery reports */ rd_kafka_poll(rk, 0); } /* Poll to handle delivery reports */ rd_kafka_poll(rk, 0); /* Wait for messages to be delivered */ while (run && rd_kafka_outq_len(rk) > 0) rd_kafka_poll(rk, 100); /* Destroy the handle */ rd_kafka_destroy(rk); } else if (mode == 'C') { /* * Consumer */ /* Create Kafka handle */ if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr)))) { fprintf(stderr, "%% Failed to create new consumer: %s\n", errstr); exit(1); } /* Set logger */ rd_kafka_set_logger(rk, logger); rd_kafka_set_log_level(rk, LOG_DEBUG); /* Add brokers */ if (rd_kafka_brokers_add(rk, brokers) == 0) { fprintf(stderr, "%% No valid brokers specified\n"); exit(1); } /* Create topic */ rkt = rd_kafka_topic_new(rk, topic, topic_conf); /* Start consuming */ if (rd_kafka_consume_start(rkt, partition, start_offset) == -1){ fprintf(stderr, "%% Failed to start consuming: %s\n", rd_kafka_err2str(rd_kafka_errno2err(errno))); exit(1); } while (run) { rd_kafka_message_t *rkmessage; /* Consume single message. * See rdkafka_performance.c for high speed * consuming of messages. */ rkmessage = rd_kafka_consume(rkt, partition, 1000); if (!rkmessage) /* timeout */ continue; msg_consume(rkmessage, NULL); /* Return message to rdkafka */ rd_kafka_message_destroy(rkmessage); } /* Stop consuming */ rd_kafka_consume_stop(rkt, partition); rd_kafka_topic_destroy(rkt); rd_kafka_destroy(rk); } else if (mode == 'L') { rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR; /* Create Kafka handle */ if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)))) { fprintf(stderr, "%% Failed to create new producer: %s\n", errstr); exit(1); } /* Set logger */ rd_kafka_set_logger(rk, logger); rd_kafka_set_log_level(rk, LOG_DEBUG); /* Add brokers */ if (rd_kafka_brokers_add(rk, brokers) == 0) { fprintf(stderr, "%% No valid brokers specified\n"); exit(1); } /* Create topic */ if (topic) rkt = rd_kafka_topic_new(rk, topic, topic_conf); else rkt = NULL; while (run) { const struct rd_kafka_metadata *metadata; /* Fetch metadata */ err = rd_kafka_metadata(rk, rkt ? 0 : 1, rkt, &metadata, 5000); if (err != RD_KAFKA_RESP_ERR_NO_ERROR) { fprintf(stderr, "%% Failed to acquire metadata: %s\n", rd_kafka_err2str(err)); run = 0; break; } metadata_print(topic, metadata); rd_kafka_metadata_destroy(metadata); run = 0; } /* Destroy the handle */ rd_kafka_destroy(rk); /* Exit right away, dont wait for background cleanup, we haven't * done anything important anyway. */ exit(err ? 2 : 0); } /* Let background threads clean up and terminate cleanly. */ rd_kafka_wait_destroyed(2000); return 0; }
int main (int argc, char **argv) { rd_kafka_t *rk; char *broker = NULL; int mode_p = 0; int mode_c = 0; char *topic; int partition; /* Command line argument option definition. */ rd_opt_t opts[] = { { RD_OPT_BOOL|RD_OPT_MUT1|RD_OPT_REQ, 'P', "produce", 0, &mode_p, "Run as producer" }, { RD_OPT_BOOL|RD_OPT_MUT1|RD_OPT_REQ, 'C', "consume", 0, &mode_c, "Run as consumer" }, { RD_OPT_STR|RD_OPT_REQ, 't', "topic", 1, &topic, "Topic" }, { RD_OPT_INT|RD_OPT_REQ, 'p', "partition", 1, &partition, "Partition" }, { RD_OPT_STR, 'b', "broker", 1, &broker, "Broker host:port" }, { RD_OPT_END }, }; /* Initialize librd */ rd_init(); /* Parse command line arguments. */ if (!rd_opt_get(opts, argc, argv, NULL, NULL)) exit(1); signal(SIGINT, stop); if (mode_p) { /* * Producer */ char buf[1024]; /* Create Kafka handle */ if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, broker, NULL))) { perror("kafka_new producer"); exit(1); } fprintf(stderr, "%% Type stuff and hit enter to send\n"); while (run && (fgets(buf, sizeof(buf), stdin))) { int len = strlen(buf); /* Send/Produce message. */ rd_kafka_produce(rk, topic, partition, 0, buf, len); fprintf(stderr, "%% Sent %i bytes to topic " "%s partition %i\n", len, topic, partition); } /* Destroy the handle */ rd_kafka_destroy(rk); } else { /* * Consumer */ rd_kafka_op_t *rko; /* Base our configuration on the default config. */ rd_kafka_conf_t conf = rd_kafka_defaultconf; /* The offset storage file is optional but its presence * avoids starting all over from offset 0 again when * the program restarts. * ZooKeeper functionality will be implemented in future * versions and then the offset will be stored there instead. */ conf.consumer.offset_file = "."; /* current directory */ /* Indicate to rdkafka that the application is responsible * for storing the offset. This allows the application to * succesfully handle a message before storing the offset. * If this flag is not set rdkafka will store the offset * just prior to returning the message from rd_kafka_consume(). */ conf.flags |= RD_KAFKA_CONF_F_APP_OFFSET_STORE; /* Use the consumer convenience function * to create a Kafka handle. */ if (!(rk = rd_kafka_new_consumer(broker, topic, (uint32_t)partition, 0, &conf))) { perror("kafka_new_consumer"); exit(1); } while (run) { /* Fetch an "op" which is one of: * - a kafka message (if rko_len>0 && rko_err==0) * - an error (if rko_err) */ if (!(rko = rd_kafka_consume(rk, 1000/*timeout ms*/))) continue; if (rko->rko_err) fprintf(stderr, "%% Error: %.*s\n", rko->rko_len, rko->rko_payload); else if (rko->rko_len) { fprintf(stderr, "%% Message with " "next-offset %"PRIu64" is %i bytes\n", rko->rko_offset, rko->rko_len); rd_hexdump(stdout, "Message", rko->rko_payload, rko->rko_len); } /* rko_offset contains the offset of the _next_ * message. We store it when we're done processing * the current message. */ if (rko->rko_offset) rd_kafka_offset_store(rk, rko->rko_offset); /* Destroy the op */ rd_kafka_op_destroy(rk, rko); } /* Destroy the handle */ rd_kafka_destroy(rk); } return 0; }