int main_0095_all_brokers_down (int argc, char **argv) { RdKafka::Conf *conf; std::string errstr; Test::conf_init(&conf, NULL, 20); /* Two broker addresses that will quickly reject the connection */ Test::conf_set(conf, "bootstrap.servers", "127.0.0.1:1,127.0.0.1:2"); /* * First test producer */ errorEventCb pEvent = errorEventCb(); if (conf->set("event_cb", &pEvent, errstr) != RdKafka::Conf::CONF_OK) Test::Fail(errstr); Test::Say("Test Producer\n"); RdKafka::Producer *p = RdKafka::Producer::create(conf, errstr); if (!p) Test::Fail("Failed to create Producer: " + errstr); /* Wait for all brokers down */ while (!pEvent.error_seen) p->poll(1000); delete p; /* * Test high-level consumer that has a logical broker (group coord), * which has caused AllBrokersDown generation problems (#2259) */ errorEventCb cEvent = errorEventCb(); Test::conf_set(conf, "group.id", "test"); if (conf->set("event_cb", &cEvent, errstr) != RdKafka::Conf::CONF_OK) Test::Fail(errstr); Test::Say("Test KafkaConsumer\n"); RdKafka::KafkaConsumer *c = RdKafka::KafkaConsumer::create(conf, errstr); if (!c) Test::Fail("Failed to create KafkaConsumer: " + errstr); /* Wait for all brokers down */ while (!cEvent.error_seen) { RdKafka::Message *m = c->consume(1000); if (m) delete m; } c->close(); delete c; return 0; }
static void do_test_null_empty (bool api_version_request) { std::string topic = Test::mk_topic_name("0070_null_empty", 1); const int partition = 0; Test::Say(tostr() << "Testing with api.version.request=" << api_version_request << " on topic " << topic << " partition " << partition << "\n"); RdKafka::Conf *conf; Test::conf_init(&conf, NULL, 0); Test::conf_set(conf, "api.version.request", api_version_request ? "true" : "false"); Test::conf_set(conf, "acks", "all"); std::string errstr; RdKafka::Producer *p = RdKafka::Producer::create(conf, errstr); if (!p) Test::Fail("Failed to create Producer: " + errstr); delete conf; const int msgcnt = 8; static const char *msgs[msgcnt*2] = { NULL, NULL, "key2", NULL, "key3", "val3", NULL, "val4", "", NULL, NULL, "", "", "" }; RdKafka::ErrorCode err; for (int i = 0 ; i < msgcnt * 2 ; i += 2) { Test::Say(3, tostr() << "Produce message #" << (i/2) << ": key=\"" << (msgs[i] ? msgs[i] : "Null") << "\", value=\"" << (msgs[i+1] ? msgs[i+1] : "Null") << "\"\n"); err = p->produce(topic, partition, RdKafka::Producer::RK_MSG_COPY, /* Value */ (void *)msgs[i+1], msgs[i+1] ? strlen(msgs[i+1]) : 0, /* Key */ (void *)msgs[i], msgs[i] ? strlen(msgs[i]) : 0, 0, NULL); if (err != RdKafka::ERR_NO_ERROR) Test::Fail("Produce failed: " + RdKafka::err2str(err)); } if (p->flush(tmout_multip(3*5000)) != 0) Test::Fail("Not all messages flushed"); Test::Say(tostr() << "Produced " << msgcnt << " messages to " << topic << "\n"); delete p; /* * Now consume messages from the beginning, making sure they match * what was produced. */ /* Create consumer */ Test::conf_init(&conf, NULL, 10); Test::conf_set(conf, "group.id", topic); Test::conf_set(conf, "api.version.request", api_version_request ? "true" : "false"); Test::conf_set(conf, "enable.auto.commit", "false"); RdKafka::KafkaConsumer *c = RdKafka::KafkaConsumer::create(conf, errstr); if (!c) Test::Fail("Failed to create KafkaConsumer: " + errstr); delete conf; /* Assign the partition */ std::vector<RdKafka::TopicPartition*> parts; parts.push_back(RdKafka::TopicPartition::create(topic, partition, RdKafka::Topic::OFFSET_BEGINNING)); err = c->assign(parts); if (err != RdKafka::ERR_NO_ERROR) Test::Fail("assign() failed: " + RdKafka::err2str(err)); RdKafka::TopicPartition::destroy(parts); /* Start consuming */ int failures = 0; for (int i = 0 ; i < msgcnt * 2 ; i += 2) { RdKafka::Message *msg = c->consume(tmout_multip(5000)); if (msg->err()) Test::Fail(tostr() << "consume() failed at message " << (i/2) << ": " << msg->errstr()); /* verify key */ failures += check_equal(msgs[i], msg->key() ? msg->key()->c_str() : NULL, msg->key_len(), tostr() << "message #" << (i/2) << " (offset " << msg->offset() << ") key"); /* verify key_pointer() API as too */ failures += check_equal(msgs[i], (const char *)msg->key_pointer(), msg->key_len(), tostr() << "message #" << (i/2) << " (offset " << msg->offset() << ") key"); /* verify value */ failures += check_equal(msgs[i+1], (const char *)msg->payload(), msg->len(), tostr() << "message #" << (i/2) << " (offset " << msg->offset() << ") value"); delete msg; } Test::Say(tostr() << "Done consuming, closing. " << failures << " test failures\n"); if (failures) Test::Fail(tostr() << "See " << failures << " previous test failure(s)"); c->close(); delete c; }
static void do_test_empty_topic_consumer () { std::string errstr; std::string topic = Test::mk_topic_name("0067_empty_topic", 1); const int32_t partition = 0; RdKafka::Conf *conf; Test::conf_init(&conf, NULL, 0); /* Create simple consumer */ RdKafka::Consumer *consumer = RdKafka::Consumer::create(conf, errstr); if (!consumer) Test::Fail("Failed to create Consumer: " + errstr); RdKafka::Topic *rkt = RdKafka::Topic::create(consumer, topic, NULL, errstr); if (!rkt) Test::Fail("Simple Topic failed: " + errstr); /* Create the topic through a metadata request. */ Test::Say("Creating empty topic " + topic + "\n"); RdKafka::Metadata *md; RdKafka::ErrorCode err = consumer->metadata(false, rkt, &md, tmout_multip(10*1000)); if (err) Test::Fail("Failed to create topic " + topic + ": " + RdKafka::err2str(err)); delete md; /* Start consumer */ err = consumer->start(rkt, partition, RdKafka::Topic::OFFSET_BEGINNING); if (err) Test::Fail("Consume start() failed: " + RdKafka::err2str(err)); /* Consume using legacy consumer, should give an EOF and nothing else. */ Test::Say("Simple Consumer: consuming\n"); RdKafka::Message *msg = consumer->consume(rkt, partition, tmout_multip(10 * 1000)); if (msg->err() != RdKafka::ERR__PARTITION_EOF) Test::Fail("Simple consume() expected EOF, got " + RdKafka::err2str(msg->err())); delete msg; /* Nothing else should come now, just a consume() timeout */ msg = consumer->consume(rkt, partition, 1 * 1000); if (msg->err() != RdKafka::ERR__TIMED_OUT) Test::Fail("Simple consume() expected timeout, got " + RdKafka::err2str(msg->err())); delete msg; consumer->stop(rkt, partition); delete rkt; delete consumer; /* * Now do the same thing using the high-level KafkaConsumer. */ Test::conf_set(conf, "group.id", topic); RdKafka::KafkaConsumer *kconsumer = RdKafka::KafkaConsumer::create(conf, errstr); if (!kconsumer) Test::Fail("Failed to create KafkaConsumer: " + errstr); std::vector<RdKafka::TopicPartition*> part; part.push_back(RdKafka::TopicPartition::create(topic, partition)); err = kconsumer->assign(part); if (err) Test::Fail("assign() failed: " + RdKafka::err2str(err)); RdKafka::TopicPartition::destroy(part); Test::Say("KafkaConsumer: consuming\n"); msg = kconsumer->consume(tmout_multip(5 * 1000)); if (msg->err() != RdKafka::ERR__PARTITION_EOF) Test::Fail("KafkaConsumer consume() expected EOF, got " + RdKafka::err2str(msg->err())); delete msg; /* Nothing else should come now, just a consume() timeout */ msg = kconsumer->consume(1 * 1000); if (msg->err() != RdKafka::ERR__TIMED_OUT) Test::Fail("KafkaConsumer consume() expected timeout, got " + RdKafka::err2str(msg->err())); delete msg; kconsumer->close(); delete kconsumer; delete conf; }
static void do_test_fetch_max_bytes (void) { const int partcnt = 3; int msgcnt = 10 * partcnt; const int msgsize = 900*1024; /* Less than 1 Meg to account * for batch overhead */ std::string errstr; RdKafka::ErrorCode err; std::string topic = Test::mk_topic_name("0081-fetch_max_bytes", 1); /* Produce messages to partitions */ for (int32_t p = 0 ; p < (int32_t)partcnt ; p++) test_produce_msgs_easy_size(topic.c_str(), 0, p, msgcnt, msgsize); /* Create consumer */ RdKafka::Conf *conf; Test::conf_init(&conf, NULL, 10); Test::conf_set(conf, "group.id", topic); Test::conf_set(conf, "auto.offset.reset", "earliest"); /* We try to fetch 20 Megs per partition, but only allow 1 Meg as total * response size, this ends up serving the first batch from the * first partition. * receive.message.max.bytes is set low to trigger the original bug, * but this value is now adjusted upwards automatically by rd_kafka_new() * to hold both fetch.max.bytes and the protocol / batching overhead. * Prior to the introduction of fetch.max.bytes the fetcher code * would use receive.message.max.bytes to limit the total Fetch response, * but due to batching overhead it would result in situations where * the consumer asked for 1000000 bytes and got 1000096 bytes batch, which * was higher than the 1000000 limit. * See https://github.com/edenhill/librdkafka/issues/1616 * * With the added configuration strictness checks, a user-supplied * value is no longer over-written: * receive.message.max.bytes must be configured to be at least 512 bytes * larger than fetch.max.bytes. */ Test::conf_set(conf, "max.partition.fetch.bytes", "20000000"); /* ~20MB */ Test::conf_set(conf, "fetch.max.bytes", "1000000"); /* ~1MB */ Test::conf_set(conf, "receive.message.max.bytes", "1000512"); /* ~1MB+512 */ RdKafka::KafkaConsumer *c = RdKafka::KafkaConsumer::create(conf, errstr); if (!c) Test::Fail("Failed to create KafkaConsumer: " + errstr); delete conf; /* Subscribe */ std::vector<std::string> topics; topics.push_back(topic); if ((err = c->subscribe(topics))) Test::Fail("subscribe failed: " + RdKafka::err2str(err)); /* Start consuming */ Test::Say("Consuming topic " + topic + "\n"); int cnt = 0; while (cnt < msgcnt) { RdKafka::Message *msg = c->consume(tmout_multip(1000)); switch (msg->err()) { case RdKafka::ERR__TIMED_OUT: break; case RdKafka::ERR_NO_ERROR: cnt++; break; default: Test::Fail("Consume error: " + msg->errstr()); break; } delete msg; } Test::Say("Done\n"); c->close(); delete c; }
int main (int argc, char **argv) { std::string brokers = "localhost"; std::string errstr; std::string topic_str; std::string mode; std::string debug; std::vector<std::string> topics; bool do_conf_dump = false; int opt; int use_ccb = 0; /* * Create configuration objects */ RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL); RdKafka::Conf *tconf = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC); ExampleRebalanceCb ex_rebalance_cb; conf->set("rebalance_cb", &ex_rebalance_cb, errstr); while ((opt = getopt(argc, argv, "g:b:z:qd:eX:AM:f:qv")) != -1) { switch (opt) { case 'g': if (conf->set("group.id", optarg, errstr) != RdKafka::Conf::CONF_OK) { std::cerr << errstr << std::endl; exit(1); } break; case 'b': brokers = optarg; break; case 'z': if (conf->set("compression.codec", optarg, errstr) != RdKafka::Conf::CONF_OK) { std::cerr << errstr << std::endl; exit(1); } break; case 'e': exit_eof = true; break; case 'd': debug = optarg; break; case 'M': if (conf->set("statistics.interval.ms", optarg, errstr) != RdKafka::Conf::CONF_OK) { std::cerr << errstr << std::endl; exit(1); } break; case 'X': { char *name, *val; if (!strcmp(optarg, "dump")) { do_conf_dump = true; continue; } name = optarg; if (!(val = strchr(name, '='))) { std::cerr << "%% Expected -X property=value, not " << name << std::endl; exit(1); } *val = '\0'; val++; /* Try "topic." prefixed properties on topic * conf first, and then fall through to global if * it didnt match a topic configuration property. */ RdKafka::Conf::ConfResult res = RdKafka::Conf::CONF_UNKNOWN; if (!strncmp(name, "topic.", strlen("topic."))) res = tconf->set(name+strlen("topic."), val, errstr); if (res == RdKafka::Conf::CONF_UNKNOWN) res = conf->set(name, val, errstr); if (res != RdKafka::Conf::CONF_OK) { std::cerr << errstr << std::endl; exit(1); } } break; case 'f': if (!strcmp(optarg, "ccb")) use_ccb = 1; else { std::cerr << "Unknown option: " << optarg << std::endl; exit(1); } break; case 'q': verbosity--; break; case 'v': verbosity++; break; default: goto usage; } } for (; optind < argc ; optind++) topics.push_back(std::string(argv[optind])); if (topics.empty() || optind != argc) { usage: fprintf(stderr, "Usage: %s -g <group-id> [options] topic1 topic2..\n" "\n" "librdkafka version %s (0x%08x)\n" "\n" " Options:\n" " -g <group-id> Consumer group id\n" " -b <brokers> Broker address (localhost:9092)\n" " -z <codec> Enable compression:\n" " none|gzip|snappy\n" " -e Exit consumer when last message\n" " in partition has been received.\n" " -d [facs..] Enable debugging contexts:\n" " %s\n" " -M <intervalms> Enable statistics\n" " -X <prop=name> Set arbitrary librdkafka " "configuration property\n" " Properties prefixed with \"topic.\" " "will be set on topic object.\n" " Use '-X list' to see the full list\n" " of supported properties.\n" " -f <flag> Set option:\n" " ccb - use consume_callback\n" " -q Quiet / Decrease verbosity\n" " -v Increase verbosity\n" "\n" "\n", argv[0], RdKafka::version_str().c_str(), RdKafka::version(), RdKafka::get_debug_contexts().c_str()); exit(1); } /* * Set configuration properties */ conf->set("metadata.broker.list", brokers, errstr); if (!debug.empty()) { if (conf->set("debug", debug, errstr) != RdKafka::Conf::CONF_OK) { std::cerr << errstr << std::endl; exit(1); } } ExampleEventCb ex_event_cb; conf->set("event_cb", &ex_event_cb, errstr); conf->set("default_topic_conf", tconf, errstr); delete tconf; if (do_conf_dump) { int pass; for (pass = 0 ; pass < 2 ; pass++) { std::list<std::string> *dump; if (pass == 0) { dump = conf->dump(); std::cout << "# Global config" << std::endl; } else { dump = tconf->dump(); std::cout << "# Topic config" << std::endl; } for (std::list<std::string>::iterator it = dump->begin(); it != dump->end(); ) { std::cout << *it << " = "; it++; std::cout << *it << std::endl; it++; } std::cout << std::endl; } exit(0); } signal(SIGINT, sigterm); signal(SIGTERM, sigterm); /* * Consumer mode */ /* * Create consumer using accumulated global configuration. */ RdKafka::KafkaConsumer *consumer = RdKafka::KafkaConsumer::create(conf, errstr); if (!consumer) { std::cerr << "Failed to create consumer: " << errstr << std::endl; exit(1); } delete conf; std::cout << "% Created consumer " << consumer->name() << std::endl; /* * Subscribe to topics */ RdKafka::ErrorCode err = consumer->subscribe(topics); if (err) { std::cerr << "Failed to subscribe to " << topics.size() << " topics: " << RdKafka::err2str(err) << std::endl; exit(1); } /* * Consume messages */ while (run) { if (use_ccb) { std::cerr << "Use callback: Not implemented" << std::endl; break; } RdKafka::Message *msg = consumer->consume(1000); msg_consume(msg, NULL); delete msg; } alarm(10); /* * Stop consumer */ consumer->close(); delete consumer; std::cerr << "% Consumed " << msg_cnt << " messages (" << msg_bytes << " bytes)" << std::endl; /* * Wait for RdKafka to decommission. * This is not strictly needed (with check outq_len() above), but * allows RdKafka to clean up all its resources before the application * exits so that memory profilers such as valgrind wont complain about * memory leaks. */ RdKafka::wait_destroyed(5000); return 0; }
static void do_test_consumer_lag (void) { const int msgcnt = 10; std::string errstr; RdKafka::ErrorCode err; topic = Test::mk_topic_name("0061-consumer_lag", 1); test_produce_msgs_easy(topic.c_str(), 0, 0, msgcnt); /* * Create consumer */ /* Create consumer */ RdKafka::Conf *conf; Test::conf_init(&conf, NULL, 10); StatsCb stats; if (conf->set("event_cb", &stats, errstr) != RdKafka::Conf::CONF_OK) Test::Fail("set event_cb failed: " + errstr); Test::conf_set(conf, "group.id", topic); Test::conf_set(conf, "enable.auto.commit", "false"); Test::conf_set(conf, "enable.partition.eof", "false"); Test::conf_set(conf, "auto.offset.reset", "earliest"); Test::conf_set(conf, "statistics.interval.ms", "100"); RdKafka::KafkaConsumer *c = RdKafka::KafkaConsumer::create(conf, errstr); if (!c) Test::Fail("Failed to create KafkaConsumer: " + errstr); delete conf; /* Assign partitions */ /* Subscribe */ std::vector<RdKafka::TopicPartition*> parts; parts.push_back(RdKafka::TopicPartition::create(topic, 0)); if ((err = c->assign(parts))) Test::Fail("assign failed: " + RdKafka::err2str(err)); RdKafka::TopicPartition::destroy(parts); /* Start consuming */ Test::Say("Consuming topic " + topic + "\n"); int cnt = 0; while (cnt < msgcnt) { RdKafka::Message *msg = c->consume(tmout_multip(1000)); switch (msg->err()) { case RdKafka::ERR__TIMED_OUT: break; case RdKafka::ERR__PARTITION_EOF: Test::Fail(tostr() << "Consume error after " << cnt << "/" << msgcnt << " messages: " << msg->errstr()); break; case RdKafka::ERR_NO_ERROR: /* Proper message. Update calculated lag for later * checking in stats callback */ stats.calc_lag = msgcnt - (msg->offset()+1); cnt++; Test::Say(2, tostr() << "Received message #" << cnt << "/" << msgcnt << " at offset " << msg->offset() << " (calc lag " << stats.calc_lag << ")\n"); /* Slow down message "processing" to make sure we get * at least one stats callback per message. */ if (cnt < msgcnt) rd_sleep(1); break; default: Test::Fail("Consume error: " + msg->errstr()); break; } delete msg; } Test::Say(tostr() << "Done, lag was valid " << stats.lag_valid << " times\n"); if (stats.lag_valid == 0) Test::Fail("No valid consumer_lag in statistics seen"); c->close(); delete c; }