int main_0035_api_version (int argc, char **argv) {
rd_kafka_t *rk;
rd_kafka_conf_t *conf;
const struct rd_kafka_metadata *metadata;
rd_kafka_resp_err_t err;
test_timing_t t_meta;
test_conf_init(&conf, NULL, 30);
test_conf_set(conf, "socket.timeout.ms", "12000");
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
TEST_SAY("Querying for metadata\n");
TIMING_START(&t_meta, "metadata()");
err = rd_kafka_metadata(rk, 0, NULL, &metadata, 10*1000);
TIMING_STOP(&t_meta);
if (err)
TEST_FAIL("metadata() failed: %s",
rd_kafka_err2str(err));
if (TIMING_DURATION(&t_meta) / 1000 > 11*1000)
TEST_FAIL("metadata() took too long: %.3fms",
(float)TIMING_DURATION(&t_meta) / 1000.0f);
rd_kafka_metadata_destroy(metadata);
TEST_SAY("Metadata succeeded\n");
rd_kafka_destroy(rk);
return 0;
}
static void test_producer_no_connection (void) {
rd_kafka_t *rk;
rd_kafka_conf_t *conf;
rd_kafka_topic_t *rkt;
int i;
const int partition_cnt = 2;
int msgcnt = 0;
test_timing_t t_destroy;
test_conf_init(&conf, NULL, 20);
test_conf_set(conf, "bootstrap.servers", NULL);
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
rkt = test_create_topic_object(rk, __FUNCTION__,
"message.timeout.ms", "5000", NULL);
test_produce_msgs_nowait(rk, rkt, 0, RD_KAFKA_PARTITION_UA, 0, 100,
NULL, 100, 0, &msgcnt);
for (i = 0 ; i < partition_cnt ; i++)
test_produce_msgs_nowait(rk, rkt, 0, i,
0, 100, NULL, 100, 0, &msgcnt);
rd_kafka_poll(rk, 1000);
TEST_SAY("%d messages in queue\n", rd_kafka_outq_len(rk));
rd_kafka_topic_destroy(rkt);
TIMING_START(&t_destroy, "rd_kafka_destroy()");
rd_kafka_destroy(rk);
TIMING_STOP(&t_destroy);
}
int test_consumer_poll (const char *what, rd_kafka_t *rk, uint64_t testid,
int exp_eof_cnt, int exp_msg_base, int exp_cnt) {
int eof_cnt = 0;
int cnt = 0;
test_timing_t t_cons;
TEST_SAY("%s: consume %d messages\n", what, exp_cnt);
TIMING_START(&t_cons, "CONSUME");
while ((exp_eof_cnt == -1 || eof_cnt < exp_eof_cnt) &&
(cnt < exp_cnt)) {
rd_kafka_message_t *rkmessage;
rkmessage = rd_kafka_consumer_poll(rk, 10*1000);
if (!rkmessage) /* Shouldn't take this long to get a msg */
TEST_FAIL("%s: consumer_poll() timeout\n", what);
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
TEST_SAY("%s [%"PRId32"] reached EOF at "
"offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset);
eof_cnt++;
} else if (rkmessage->err) {
TEST_SAY("%s [%"PRId32"] error (offset %"PRId64"): %s",
rkmessage->rkt ?
rd_kafka_topic_name(rkmessage->rkt) :
"(no-topic)",
rkmessage->partition,
rkmessage->offset,
rd_kafka_message_errstr(rkmessage));
} else {
if (test_level > 2)
TEST_SAY("%s [%"PRId32"] "
"message at offset %"PRId64"\n",
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset);
test_verify_rkmessage(rkmessage, testid, -1, -1);
cnt++;
}
rd_kafka_message_destroy(rkmessage);
}
TIMING_STOP(&t_cons);
TEST_SAY("%s: consumed %d/%d messages (%d/%d EOFs)\n",
what, cnt, exp_cnt, eof_cnt, exp_eof_cnt);
return cnt;
}
static int run_test0 (struct run_args *run_args) {
struct test *test = run_args->test;
test_timing_t t_run;
int r;
char stats_file[256];
rd_snprintf(stats_file, sizeof(stats_file), "stats_%s_%"PRIu64".json",
test->name, test_id_generate());
if (!(test->stats_fp = fopen(stats_file, "w+")))
TEST_SAY("=== Failed to create stats file %s: %s ===\n",
stats_file, strerror(errno));
test_curr = test;
TEST_SAY("================= Running test %s =================\n",
test->name);
if (test->stats_fp)
TEST_SAY("==== Stats written to file %s ====\n", stats_file);
TIMING_START(&t_run, test->name);
test->start = t_run.ts_start;
r = test->mainfunc(run_args->argc, run_args->argv);
TIMING_STOP(&t_run);
TEST_LOCK();
test->duration = TIMING_DURATION(&t_run);
if (r) {
test->state = TEST_FAILED;
TEST_SAY("\033[31m"
"================= Test %s FAILED ================="
"\033[0m\n",
run_args->test->name);
} else {
test->state = TEST_PASSED;
TEST_SAY("\033[32m"
"================= Test %s PASSED ================="
"\033[0m\n",
run_args->test->name);
}
TEST_UNLOCK();
if (test->stats_fp) {
long pos = ftell(test->stats_fp);
fclose(test->stats_fp);
test->stats_fp = NULL;
/* Delete file if nothing was written */
if (pos == 0) {
#ifndef _MSC_VER
unlink(stats_file);
#else
_unlink(stats_file);
#endif
}
}
return r;
}
/**
* Waits for the messages tracked by counter \p msgcounterp to be delivered.
*/
void test_wait_delivery (rd_kafka_t *rk, int *msgcounterp) {
test_timing_t t_all;
TIMING_START(&t_all, "PRODUCE.DELIVERY.WAIT");
/* Wait for messages to be delivered */
while (*msgcounterp > 0 && rd_kafka_outq_len(rk) > 0)
rd_kafka_poll(rk, 10);
TIMING_STOP(&t_all);
}
void test_consumer_unassign (const char *what, rd_kafka_t *rk) {
rd_kafka_resp_err_t err;
test_timing_t timing;
TIMING_START(&timing, "UNASSIGN.PARTITIONS");
err = rd_kafka_assign(rk, NULL);
TIMING_STOP(&timing);
if (err)
TEST_FAIL("%s: failed to unassign current partitions: %s\n",
what, rd_kafka_err2str(err));
else
TEST_SAY("%s: unassigned current partitions\n", what);
}
void test_consumer_close (rd_kafka_t *rk) {
rd_kafka_resp_err_t err;
test_timing_t timing;
TEST_SAY("Closing consumer\n");
TIMING_START(&timing, "CONSUMER.CLOSE");
err = rd_kafka_consumer_close(rk);
TIMING_STOP(&timing);
if (err)
TEST_FAIL("Failed to close consumer: %s\n",
rd_kafka_err2str(err));
}
void test_produce_msgs (rd_kafka_t *rk, rd_kafka_topic_t *rkt,
uint64_t testid, int32_t partition,
int msg_base, int cnt,
const char *payload, size_t size) {
int msg_id;
test_timing_t t_all;
int remains = 0;
TEST_SAY("Produce to %s [%"PRId32"]: messages #%d..%d\n",
rd_kafka_topic_name(rkt), partition, msg_base, msg_base+cnt);
TIMING_START(&t_all, "PRODUCE");
for (msg_id = msg_base ; msg_id < msg_base + cnt ; msg_id++) {
char key[128];
char buf[128];
const char *use_payload;
size_t use_size;
if (payload) {
use_payload = payload;
use_size = size;
} else {
test_msg_fmt(key, sizeof(key), testid, partition,
msg_id);
rd_snprintf(buf, sizeof(buf), "data: %s", key);
use_payload = buf;
use_size = strlen(buf);
}
remains++;
if (rd_kafka_produce(rkt, partition,
RD_KAFKA_MSG_F_COPY,
(void *)use_payload, use_size,
key, strlen(key),
&remains) == -1)
TEST_FAIL("Failed to produce message %i "
"to partition %i: %s",
msg_id, (int)partition,
rd_kafka_err2str(rd_kafka_errno2err(errno)));
}
/* Wait for messages to be delivered */
while (remains > 0 && rd_kafka_outq_len(rk) > 0)
rd_kafka_poll(rk, 10);
TIMING_STOP(&t_all);
}
int main_0001_multiobj (int argc, char **argv) {
int partition = RD_KAFKA_PARTITION_UA; /* random */
int i;
const int NUM_ITER = 10;
const char *topic = NULL;
TEST_SAY("Creating and destroying %i kafka instances\n", NUM_ITER);
/* Create, use and destroy NUM_ITER kafka instances. */
for (i = 0 ; i < NUM_ITER ; i++) {
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char msg[128];
test_timing_t t_destroy;
test_conf_init(&conf, &topic_conf, 30);
if (!topic)
topic = test_mk_topic_name("0001", 0);
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (!rkt)
TEST_FAIL("Failed to create topic for "
"rdkafka instance #%i: %s\n",
i, rd_kafka_err2str(rd_kafka_errno2err(errno)));
rd_snprintf(msg, sizeof(msg), "%s test message for iteration #%i",
argv[0], i);
/* Produce a message */
rd_kafka_produce(rkt, partition, RD_KAFKA_MSG_F_COPY,
msg, strlen(msg), NULL, 0, NULL);
/* Wait for it to be sent (and possibly acked) */
rd_kafka_flush(rk, -1);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy rdkafka instance */
TIMING_START(&t_destroy, "rd_kafka_destroy()");
rd_kafka_destroy(rk);
TIMING_STOP(&t_destroy);
}
return 0;
}
void test_consumer_assign (const char *what, rd_kafka_t *rk,
rd_kafka_topic_partition_list_t *partitions) {
rd_kafka_resp_err_t err;
test_timing_t timing;
TIMING_START(&timing, "ASSIGN.PARTITIONS");
err = rd_kafka_assign(rk, partitions);
TIMING_STOP(&timing);
if (err)
TEST_FAIL("%s: failed to assign %d partition(s): %s\n",
what, partitions->cnt, rd_kafka_err2str(err));
else
TEST_SAY("%s: assigned %d partition(s)\n",
what, partitions->cnt);
}
static void legacy_consume_many (char **topics, int topic_cnt, uint64_t testid){
rd_kafka_t *rk;
test_timing_t t_rkt_create;
int i;
rd_kafka_topic_t **rkts;
int msg_base = 0;
TEST_SAY(_C_MAG "%s\n" _C_CLR, __FUNCTION__);
test_conf_init(NULL, NULL, 60);
rk = test_create_consumer(NULL, NULL, NULL, NULL);
TEST_SAY("Creating %d topic objects\n", topic_cnt);
rkts = malloc(sizeof(*rkts) * topic_cnt);
TIMING_START(&t_rkt_create, "Topic object create");
for (i = 0 ; i < topic_cnt ; i++)
rkts[i] = test_create_topic_object(rk, topics[i], NULL);
TIMING_STOP(&t_rkt_create);
TEST_SAY("Start consumer for %d topics\n", topic_cnt);
for (i = 0 ; i < topic_cnt ; i++)
test_consumer_start("legacy", rkts[i], 0,
RD_KAFKA_OFFSET_BEGINNING);
TEST_SAY("Consuming from %d messages from each %d topics\n",
msgs_per_topic, topic_cnt);
for (i = 0 ; i < topic_cnt ; i++) {
test_consume_msgs("legacy", rkts[i], testid, 0, TEST_NO_SEEK,
msg_base, msgs_per_topic, 1);
msg_base += msgs_per_topic;
}
TEST_SAY("Stopping consumers\n");
for (i = 0 ; i < topic_cnt ; i++)
test_consumer_stop("legacy", rkts[i], 0);
TEST_SAY("Destroying %d topic objects\n", topic_cnt);
for (i = 0 ; i < topic_cnt ; i++)
rd_kafka_topic_destroy(rkts[i]);
free(rkts);
rd_kafka_destroy(rk);
}
/**
* @brief Verify that an unclean rd_kafka_destroy() does not hang.
*/
static void do_test_unclean_destroy (rd_kafka_type_t cltype, int with_mainq) {
rd_kafka_t *rk;
char errstr[512];
rd_kafka_conf_t *conf;
rd_kafka_queue_t *q;
rd_kafka_event_t *rkev;
rd_kafka_DeleteTopic_t *topic;
test_timing_t t_destroy;
test_conf_init(&conf, NULL, 0);
/* Remove brokers, if any, since this is a local test and we
* rely on the controller not being found. */
test_conf_set(conf, "bootstrap.servers", "");
test_conf_set(conf, "socket.timeout.ms", "60000");
rk = rd_kafka_new(cltype, conf, errstr, sizeof(errstr));
TEST_ASSERT(rk, "kafka_new(%d): %s", cltype, errstr);
TEST_SAY(_C_MAG "[ Test unclean destroy for %s using %s]\n", rd_kafka_name(rk),
with_mainq ? "mainq" : "tempq");
if (with_mainq)
q = rd_kafka_queue_get_main(rk);
else
q = rd_kafka_queue_new(rk);
topic = rd_kafka_DeleteTopic_new("test");
rd_kafka_DeleteTopics(rk, &topic, 1, NULL, q);
rd_kafka_DeleteTopic_destroy(topic);
/* We're not expecting a result yet since DeleteTopics will attempt
* to look up the controller for socket.timeout.ms (1 minute). */
rkev = rd_kafka_queue_poll(q, 100);
TEST_ASSERT(!rkev, "Did not expect result: %s", rd_kafka_event_name(rkev));
rd_kafka_queue_destroy(q);
TEST_SAY("Giving rd_kafka_destroy() 5s to finish, "
"despite Admin API request being processed\n");
test_timeout_set(5);
TIMING_START(&t_destroy, "rd_kafka_destroy()");
rd_kafka_destroy(rk);
TIMING_STOP(&t_destroy);
/* Restore timeout */
test_timeout_set(60);
}
/**
* Produces \p cnt messages and returns immediately.
* Does not wait for delivery.
* \p msgcounterp is incremented for each produced messages and passed
* as \p msg_opaque which is later used in test_dr_cb to decrement
* the counter on delivery.
*/
void test_produce_msgs_nowait (rd_kafka_t *rk, rd_kafka_topic_t *rkt,
uint64_t testid, int32_t partition,
int msg_base, int cnt,
const char *payload, size_t size,
int *msgcounterp) {
int msg_id;
test_timing_t t_all;
TEST_SAY("Produce to %s [%"PRId32"]: messages #%d..%d\n",
rd_kafka_topic_name(rkt), partition, msg_base, msg_base+cnt);
TIMING_START(&t_all, "PRODUCE");
for (msg_id = msg_base ; msg_id < msg_base + cnt ; msg_id++) {
char key[128];
char buf[128];
const char *use_payload;
size_t use_size;
if (payload) {
use_payload = payload;
use_size = size;
} else {
test_msg_fmt(key, sizeof(key), testid, partition,
msg_id);
rd_snprintf(buf, sizeof(buf), "%s: data", key);
use_payload = buf;
use_size = strlen(buf);
}
if (rd_kafka_produce(rkt, partition,
RD_KAFKA_MSG_F_COPY,
(void *)use_payload, use_size,
key, strlen(key),
msgcounterp) == -1)
TEST_FAIL("Failed to produce message %i "
"to partition %i: %s",
msg_id, (int)partition,
rd_kafka_err2str(rd_kafka_errno2err(errno)));
(*msgcounterp)++;
}
TIMING_STOP(&t_all);
}
/* evict_block should evict exactly one block if it is successful */
static int wb_evict_block(BD_t * object, bool only_dirty)
{
struct cache_info * info = (struct cache_info *) object;
#ifdef __KERNEL__
revision_tail_process_landing_requests();
#endif
for(;;)
{
int r = FLUSH_EMPTY;
struct cache_slot * slot;
for(slot = info->blocks[0].lru; slot != &info->blocks[0]; slot = slot->prev)
{
int code = wb_flush_block(object, slot);
if(code == FLUSH_DONE || (!only_dirty && code == FLUSH_EMPTY))
{
wb_pop_block(info, slot->block->cache_number, (uint32_t) (slot - &info->blocks[0]));
return 0;
}
r |= code;
}
#ifdef __KERNEL__
/* For both FLUSH_NONE and FLUSH_SOME we must wait to make
* progress if there are any flights in progress. For FLUSH_NONE
* this is obvious; for FLUSH_SOME you must consider that the
* only way more blocks can be written is by waiting for the
* blocks that were just written to be completed, assuming that
* we do not have stacked caches. */
if(revision_tail_flights_exist())
{
KERNEL_INTERVAL(wait);
TIMING_START(wait);
revision_tail_wait_for_landing_requests();
revision_tail_process_landing_requests();
TIMING_STOP(wait, wait);
}
else
#endif
if(r == FLUSH_NONE)
return -EBUSY;
}
}
/**
* Create producer, produce \p msgcnt messages to \p topic \p partition,
* destroy consumer, and returns the used testid.
*/
uint64_t
test_produce_msgs_easy (const char *topic, uint64_t testid,
int32_t partition, int msgcnt) {
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
test_timing_t t_produce;
if (!testid)
testid = test_id_generate();
rk = test_create_producer();
rkt = test_create_producer_topic(rk, topic, NULL);
TIMING_START(&t_produce, "PRODUCE");
test_produce_msgs(rk, rkt, testid, partition, 0, msgcnt, NULL, 0);
TIMING_STOP(&t_produce);
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
return testid;
}
static void test_producer_partition_cnt_change (void) {
rd_kafka_t *rk;
rd_kafka_conf_t *conf;
rd_kafka_topic_t *rkt;
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
const int partition_cnt = 4;
int msgcnt = 100000;
test_timing_t t_destroy;
int produced = 0;
test_kafka_topics("--create --topic %s --replication-factor 1 "
"--partitions %d",
topic, partition_cnt/2);
test_conf_init(&conf, NULL, 20);
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
rkt = test_create_topic_object(rk, __FUNCTION__,
"message.timeout.ms",
tsprintf("%d", tmout_multip(5000)),
NULL);
test_produce_msgs_nowait(rk, rkt, 0, RD_KAFKA_PARTITION_UA, 0, msgcnt/2,
NULL, 100, &produced);
test_kafka_topics("--alter --topic %s --partitions %d",
topic, partition_cnt);
test_produce_msgs_nowait(rk, rkt, 0, RD_KAFKA_PARTITION_UA,
msgcnt/2, msgcnt/2,
NULL, 100, &produced);
test_wait_delivery(rk, &produced);
rd_kafka_topic_destroy(rkt);
TIMING_START(&t_destroy, "rd_kafka_destroy()");
rd_kafka_destroy(rk);
TIMING_STOP(&t_destroy);
}
/**
* @brief Verify that an unclean rd_kafka_destroy() does not hang.
*/
static void do_test_unclean_destroy (rd_kafka_type_t cltype, int with_mainq) {
rd_kafka_t *rk;
char errstr[512];
rd_kafka_conf_t *conf;
rd_kafka_queue_t *q;
rd_kafka_NewTopic_t *topic;
test_timing_t t_destroy;
test_conf_init(&conf, NULL, 0);
rk = rd_kafka_new(cltype, conf, errstr, sizeof(errstr));
TEST_ASSERT(rk, "kafka_new(%d): %s", cltype, errstr);
TEST_SAY(_C_MAG "[ Test unclean destroy for %s using %s]\n", rd_kafka_name(rk),
with_mainq ? "mainq" : "tempq");
if (with_mainq)
q = rd_kafka_queue_get_main(rk);
else
q = rd_kafka_queue_new(rk);
topic = rd_kafka_NewTopic_new(test_mk_topic_name(__FUNCTION__, 1),
3, 1, NULL, 0);
rd_kafka_CreateTopics(rk, &topic, 1, NULL, q);
rd_kafka_NewTopic_destroy(topic);
rd_kafka_queue_destroy(q);
TEST_SAY("Giving rd_kafka_destroy() 5s to finish, "
"despite Admin API request being processed\n");
test_timeout_set(5);
TIMING_START(&t_destroy, "rd_kafka_destroy()");
rd_kafka_destroy(rk);
TIMING_STOP(&t_destroy);
/* Restore timeout */
test_timeout_set(60);;
}
static void produce_many (char **topics, int topic_cnt, uint64_t testid) {
rd_kafka_t *rk;
test_timing_t t_rkt_create;
int i;
rd_kafka_topic_t **rkts;
TEST_SAY(_C_MAG "%s\n" _C_CLR, __FUNCTION__);
rk = test_create_producer();
TEST_SAY("Creating %d topic objects\n", topic_cnt);
rkts = malloc(sizeof(*rkts) * topic_cnt);
TIMING_START(&t_rkt_create, "Topic object create");
for (i = 0 ; i < topic_cnt ; i++) {
rkts[i] = test_create_topic_object(rk, topics[i], NULL);
}
TIMING_STOP(&t_rkt_create);
TEST_SAY("Producing %d messages to each %d topics\n",
msgs_per_topic, topic_cnt);
/* Produce messages to each topic (so they are created) */
for (i = 0 ; i < topic_cnt ; i++) {
test_produce_msgs(rk, rkts[i], testid, 0,
i * msgs_per_topic, msgs_per_topic,
NULL, 100);
}
TEST_SAY("Destroying %d topic objects\n", topic_cnt);
for (i = 0 ; i < topic_cnt ; i++) {
rd_kafka_topic_destroy(rkts[i]);
}
free(rkts);
test_flush(rk, 30000);
rd_kafka_destroy(rk);
}
int main(int argc, char **argv) {
const char *tests_to_run = NULL; /* all */
int test_flags = 0;
int i, r;
test_timing_t t_all;
mtx_init(&test_mtx, mtx_plain);
test_init();
#ifndef _MSC_VER
tests_to_run = getenv("TESTS");
#endif
for (i = 1 ; i < argc ; i++) {
if (!strncmp(argv[i], "-p", 2) && strlen(argv[i]) > 2)
test_concurrent_max = strtod(argv[i]+2, NULL);
else if (!strcmp(argv[i], "-l"))
test_flags |= TEST_F_LOCAL;
else if (!strcmp(argv[i], "-a"))
test_assert_on_fail = 1;
else if (*argv[i] != '-')
tests_to_run = argv[i];
else {
printf("Unknown option: %s\n"
"\n"
"Usage: %s [options] [<test-match-substr>]\n"
"Options:\n"
" -p<N> Run N tests in parallel\n"
" -l Only run local tests (no broker needed)\n"
" -a Assert on failures\n"
"\n",
argv[0], argv[i]);
exit(1);
}
}
test_curr = &tests[0];
test_curr->state = TEST_PASSED;
test_curr->start = test_clock();
TEST_SAY("Tests to run: %s\n", tests_to_run ? tests_to_run : "all");
TEST_SAY("Test filter: %s\n",
(test_flags & TEST_F_LOCAL) ?
"local tests only" : "no filter");
TEST_SAY("Action on test failure: %s\n",
test_assert_on_fail ? "assert crash" : "continue other tests");
test_timeout_set(20);
TIMING_START(&t_all, "ALL-TESTS");
run_tests(tests_to_run, test_flags, argc, argv);
TEST_LOCK();
while (tests_running_cnt > 0) {
struct test *test;
TEST_SAY("%d test(s) running:", tests_running_cnt);
for (test = tests ; test->name ; test++)
if (test->state == TEST_RUNNING)
TEST_SAY0(" %s", test->name);
TEST_SAY0("\n");
TEST_UNLOCK();
rd_sleep(1);
TEST_LOCK();
}
TIMING_STOP(&t_all);
test_curr = &tests[0];
test_curr->duration = test_clock() - test_curr->start;
TEST_UNLOCK();
/* Wait for everything to be cleaned up since broker destroys are
* handled in its own thread. */
test_wait_exit(10);
r = test_summary(1/*lock*/) ? 1 : 0;
/* If we havent failed at this point then
* there were no threads leaked */
if (r == 0)
TEST_SAY("\n============== ALL TESTS PASSED ==============\n");
return r;
}
/**
* @brief Test creation of partitions
*
*
*/
static void do_test_CreatePartitions (const char *what,
rd_kafka_t *rk, rd_kafka_queue_t *useq,
int op_timeout) {
rd_kafka_queue_t *q = useq ? useq : rd_kafka_queue_new(rk);
#define MY_CRP_TOPICS_CNT 9
char *topics[MY_CRP_TOPICS_CNT];
rd_kafka_NewTopic_t *new_topics[MY_CRP_TOPICS_CNT];
rd_kafka_NewPartitions_t *crp_topics[MY_CRP_TOPICS_CNT];
rd_kafka_AdminOptions_t *options = NULL;
/* Expected topics in metadata */
rd_kafka_metadata_topic_t exp_mdtopics[MY_CRP_TOPICS_CNT] = {{0}};
rd_kafka_metadata_partition_t exp_mdparts[2] = {{0}};
int exp_mdtopic_cnt = 0;
int i;
char errstr[512];
rd_kafka_resp_err_t err;
test_timing_t timing;
int metadata_tmout;
int num_replicas = (int)avail_broker_cnt;
TEST_SAY(_C_MAG "[ %s CreatePartitions with %s, op_timeout %d ]\n",
rd_kafka_name(rk), what, op_timeout);
/* Set up two expected partitions with different replication sets
* so they can be matched by the metadata checker later.
* Even partitions use exp_mdparts[0] while odd partitions
* use exp_mdparts[1]. */
/* Set valid replica assignments (even, and odd (reverse) ) */
exp_mdparts[0].replicas = rd_alloca(sizeof(*exp_mdparts[0].replicas) *
num_replicas);
exp_mdparts[1].replicas = rd_alloca(sizeof(*exp_mdparts[1].replicas) *
num_replicas);
exp_mdparts[0].replica_cnt = num_replicas;
exp_mdparts[1].replica_cnt = num_replicas;
for (i = 0 ; i < num_replicas ; i++) {
exp_mdparts[0].replicas[i] = avail_brokers[i];
exp_mdparts[1].replicas[i] = avail_brokers[num_replicas-i-1];
}
/**
* Construct CreatePartitions array
*/
for (i = 0 ; i < MY_CRP_TOPICS_CNT ; i++) {
char *topic = rd_strdup(test_mk_topic_name(__FUNCTION__, 1));
int initial_part_cnt = 1 + (i * 2);
int new_part_cnt = 1 + (i / 2);
int final_part_cnt = initial_part_cnt + new_part_cnt;
int set_replicas = !(i % 2);
int pi;
topics[i] = topic;
/* Topic to create with initial partition count */
new_topics[i] = rd_kafka_NewTopic_new(topic, initial_part_cnt,
set_replicas ?
-1 : num_replicas,
NULL, 0);
/* .. and later add more partitions to */
crp_topics[i] = rd_kafka_NewPartitions_new(topic,
final_part_cnt,
errstr,
sizeof(errstr));
if (set_replicas) {
exp_mdtopics[exp_mdtopic_cnt].partitions =
rd_alloca(final_part_cnt *
sizeof(*exp_mdtopics[exp_mdtopic_cnt].
partitions));
for (pi = 0 ; pi < final_part_cnt ; pi++) {
const rd_kafka_metadata_partition_t *exp_mdp =
&exp_mdparts[pi & 1];
exp_mdtopics[exp_mdtopic_cnt].
partitions[pi] = *exp_mdp; /* copy */
exp_mdtopics[exp_mdtopic_cnt].
partitions[pi].id = pi;
if (pi < initial_part_cnt) {
/* Set replica assignment
* for initial partitions */
err = rd_kafka_NewTopic_set_replica_assignment(
new_topics[i], pi,
exp_mdp->replicas,
(size_t)exp_mdp->replica_cnt,
errstr, sizeof(errstr));
TEST_ASSERT(!err, "NewTopic_set_replica_assignment: %s",
errstr);
} else {
/* Set replica assignment for new
* partitions */
err = rd_kafka_NewPartitions_set_replica_assignment(
crp_topics[i],
pi - initial_part_cnt,
exp_mdp->replicas,
(size_t)exp_mdp->replica_cnt,
errstr, sizeof(errstr));
TEST_ASSERT(!err, "NewPartitions_set_replica_assignment: %s",
errstr);
}
}
}
TEST_SAY(_C_YEL "Topic %s with %d initial partitions will grow "
"by %d to %d total partitions with%s replicas set\n",
topics[i],
initial_part_cnt, new_part_cnt, final_part_cnt,
set_replicas ? "" : "out");
exp_mdtopics[exp_mdtopic_cnt].topic = topic;
exp_mdtopics[exp_mdtopic_cnt].partition_cnt = final_part_cnt;
exp_mdtopic_cnt++;
}
if (op_timeout != -1) {
options = rd_kafka_AdminOptions_new(
rk, RD_KAFKA_ADMIN_OP_ANY);
err = rd_kafka_AdminOptions_set_operation_timeout(
options, op_timeout, errstr, sizeof(errstr));
TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
}
/*
* Create topics with initial partition count
*/
TIMING_START(&timing, "CreateTopics");
TEST_SAY("Creating topics with initial partition counts\n");
rd_kafka_CreateTopics(rk, new_topics, MY_CRP_TOPICS_CNT,
options, q);
TIMING_ASSERT_LATER(&timing, 0, 50);
err = test_wait_topic_admin_result(q,
RD_KAFKA_EVENT_CREATETOPICS_RESULT,
NULL, 15000);
TEST_ASSERT(!err, "CreateTopics failed: %s", rd_kafka_err2str(err));
rd_kafka_NewTopic_destroy_array(new_topics, MY_CRP_TOPICS_CNT);
/*
* Create new partitions
*/
TIMING_START(&timing, "CreatePartitions");
TEST_SAY("Creating partitions\n");
rd_kafka_CreatePartitions(rk, crp_topics, MY_CRP_TOPICS_CNT,
options, q);
TIMING_ASSERT_LATER(&timing, 0, 50);
err = test_wait_topic_admin_result(q,
RD_KAFKA_EVENT_CREATEPARTITIONS_RESULT,
NULL, 15000);
TEST_ASSERT(!err, "CreatePartitions failed: %s", rd_kafka_err2str(err));
rd_kafka_NewPartitions_destroy_array(crp_topics, MY_CRP_TOPICS_CNT);
/**
* Verify that the expected topics are deleted and the non-expected
* are not. Allow it some time to propagate.
*/
if (op_timeout > 0)
metadata_tmout = op_timeout + 1000;
else
metadata_tmout = 10 * 1000;
test_wait_metadata_update(rk,
exp_mdtopics,
exp_mdtopic_cnt,
NULL, 0,
metadata_tmout);
for (i = 0 ; i < MY_CRP_TOPICS_CNT ; i++)
rd_free(topics[i]);
if (options)
rd_kafka_AdminOptions_destroy(options);
if (!useq)
rd_kafka_queue_destroy(q);
#undef MY_CRP_TOPICS_CNT
}
static void do_test_CreateTopics (const char *what,
rd_kafka_t *rk, rd_kafka_queue_t *useq,
int op_timeout, rd_bool_t validate_only) {
rd_kafka_queue_t *q = useq ? useq : rd_kafka_queue_new(rk);
#define MY_NEW_TOPICS_CNT 6
char *topics[MY_NEW_TOPICS_CNT];
rd_kafka_NewTopic_t *new_topics[MY_NEW_TOPICS_CNT];
rd_kafka_AdminOptions_t *options = NULL;
rd_kafka_resp_err_t exp_topicerr[MY_NEW_TOPICS_CNT] = {0};
rd_kafka_resp_err_t exp_err = RD_KAFKA_RESP_ERR_NO_ERROR;
/* Expected topics in metadata */
rd_kafka_metadata_topic_t exp_mdtopics[MY_NEW_TOPICS_CNT] = {{0}};
int exp_mdtopic_cnt = 0;
/* Not expected topics in metadata */
rd_kafka_metadata_topic_t exp_not_mdtopics[MY_NEW_TOPICS_CNT] = {{0}};
int exp_not_mdtopic_cnt = 0;
int i;
char errstr[512];
const char *errstr2;
rd_kafka_resp_err_t err;
test_timing_t timing;
rd_kafka_event_t *rkev;
const rd_kafka_CreateTopics_result_t *res;
const rd_kafka_topic_result_t **restopics;
size_t restopic_cnt;
int metadata_tmout ;
int num_replicas = (int)avail_broker_cnt;
int32_t *replicas;
/* Set up replicas */
replicas = rd_alloca(sizeof(*replicas) * num_replicas);
for (i = 0 ; i < num_replicas ; i++)
replicas[i] = avail_brokers[i];
TEST_SAY(_C_MAG "[ %s CreateTopics with %s, "
"op_timeout %d, validate_only %d ]\n",
rd_kafka_name(rk), what, op_timeout, validate_only);
/**
* Construct NewTopic array with different properties for
* different partitions.
*/
for (i = 0 ; i < MY_NEW_TOPICS_CNT ; i++) {
char *topic = rd_strdup(test_mk_topic_name(__FUNCTION__, 1));
int num_parts = i * 7 + 1;
int set_config = (i & 1);
int add_invalid_config = (i == 1);
int set_replicas = !(i % 3);
rd_kafka_resp_err_t this_exp_err = RD_KAFKA_RESP_ERR_NO_ERROR;
topics[i] = topic;
new_topics[i] = rd_kafka_NewTopic_new(topic,
num_parts,
set_replicas ? -1 :
num_replicas,
NULL, 0);
if (set_config) {
/*
* Add various configuration properties
*/
err = rd_kafka_NewTopic_set_config(
new_topics[i], "compression.type", "lz4");
TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
err = rd_kafka_NewTopic_set_config(
new_topics[i], "delete.retention.ms", "900");
TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
}
if (add_invalid_config) {
/* Add invalid config property */
err = rd_kafka_NewTopic_set_config(
new_topics[i],
"dummy.doesntexist",
"broker is verifying this");
TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
this_exp_err = RD_KAFKA_RESP_ERR_INVALID_CONFIG;
}
TEST_SAY("Expected result for topic #%d: %s "
"(set_config=%d, add_invalid_config=%d, "
"set_replicas=%d)\n",
i, rd_kafka_err2name(this_exp_err),
set_config, add_invalid_config, set_replicas);
if (set_replicas) {
int32_t p;
/*
* Set valid replica assignments
*/
for (p = 0 ; p < num_parts ; p++) {
err = rd_kafka_NewTopic_set_replica_assignment(
new_topics[i], p,
replicas, num_replicas,
errstr, sizeof(errstr));
TEST_ASSERT(!err, "%s", errstr);
}
}
if (this_exp_err || validate_only) {
exp_topicerr[i] = this_exp_err;
exp_not_mdtopics[exp_not_mdtopic_cnt++].topic = topic;
} else {
exp_mdtopics[exp_mdtopic_cnt].topic = topic;
exp_mdtopics[exp_mdtopic_cnt].partition_cnt =
num_parts;
exp_mdtopic_cnt++;
}
}
if (op_timeout != -1 || validate_only) {
options = rd_kafka_AdminOptions_new(
rk, RD_KAFKA_ADMIN_OP_CREATETOPICS);
if (op_timeout != -1) {
err = rd_kafka_AdminOptions_set_operation_timeout(
options, op_timeout, errstr, sizeof(errstr));
TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
}
if (validate_only) {
err = rd_kafka_AdminOptions_set_validate_only(
options, validate_only, errstr, sizeof(errstr));
TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
}
}
TIMING_START(&timing, "CreateTopics");
TEST_SAY("Call CreateTopics\n");
rd_kafka_CreateTopics(rk, new_topics, MY_NEW_TOPICS_CNT,
options, q);
TIMING_ASSERT_LATER(&timing, 0, 50);
/* Poll result queue for CreateTopics result.
* Print but otherwise ignore other event types
* (typically generic Error events). */
TIMING_START(&timing, "CreateTopics.queue_poll");
do {
rkev = rd_kafka_queue_poll(q, tmout_multip(20*1000));
TEST_SAY("CreateTopics: got %s in %.3fms\n",
rd_kafka_event_name(rkev),
TIMING_DURATION(&timing) / 1000.0f);
if (rd_kafka_event_error(rkev))
TEST_SAY("%s: %s\n",
rd_kafka_event_name(rkev),
rd_kafka_event_error_string(rkev));
} while (rd_kafka_event_type(rkev) !=
RD_KAFKA_EVENT_CREATETOPICS_RESULT);
/* Convert event to proper result */
res = rd_kafka_event_CreateTopics_result(rkev);
TEST_ASSERT(res, "expected CreateTopics_result, not %s",
rd_kafka_event_name(rkev));
/* Expecting error */
err = rd_kafka_event_error(rkev);
errstr2 = rd_kafka_event_error_string(rkev);
TEST_ASSERT(err == exp_err,
"expected CreateTopics to return %s, not %s (%s)",
rd_kafka_err2str(exp_err),
rd_kafka_err2str(err),
err ? errstr2 : "n/a");
TEST_SAY("CreateTopics: returned %s (%s)\n",
rd_kafka_err2str(err), err ? errstr2 : "n/a");
/* Extract topics */
restopics = rd_kafka_CreateTopics_result_topics(res, &restopic_cnt);
/* Scan topics for proper fields and expected failures. */
for (i = 0 ; i < (int)restopic_cnt ; i++) {
const rd_kafka_topic_result_t *terr = restopics[i];
/* Verify that topic order matches our request. */
if (strcmp(rd_kafka_topic_result_name(terr), topics[i]))
TEST_FAIL_LATER("Topic result order mismatch at #%d: "
"expected %s, got %s",
i, topics[i],
rd_kafka_topic_result_name(terr));
TEST_SAY("CreateTopics result: #%d: %s: %s: %s\n",
i,
rd_kafka_topic_result_name(terr),
rd_kafka_err2name(rd_kafka_topic_result_error(terr)),
rd_kafka_topic_result_error_string(terr));
if (rd_kafka_topic_result_error(terr) != exp_topicerr[i])
TEST_FAIL_LATER(
"Expected %s, not %d: %s",
rd_kafka_err2name(exp_topicerr[i]),
rd_kafka_topic_result_error(terr),
rd_kafka_err2name(rd_kafka_topic_result_error(
terr)));
}
/**
* Verify that the expecteded topics are created and the non-expected
* are not. Allow it some time to propagate.
*/
if (validate_only) {
/* No topics should have been created, give it some time
* before checking. */
rd_sleep(2);
metadata_tmout = 5 * 1000;
} else {
if (op_timeout > 0)
metadata_tmout = op_timeout + 1000;
else
metadata_tmout = 10 * 1000;
}
test_wait_metadata_update(rk,
exp_mdtopics,
exp_mdtopic_cnt,
exp_not_mdtopics,
exp_not_mdtopic_cnt,
metadata_tmout);
rd_kafka_event_destroy(rkev);
for (i = 0 ; i < MY_NEW_TOPICS_CNT ; i++) {
rd_kafka_NewTopic_destroy(new_topics[i]);
rd_free(topics[i]);
}
if (options)
rd_kafka_AdminOptions_destroy(options);
if (!useq)
rd_kafka_queue_destroy(q);
#undef MY_NEW_TOPICS_CNT
}
/**
* @brief Test deletion of topics
*
*
*/
static void do_test_DeleteTopics (const char *what,
rd_kafka_t *rk, rd_kafka_queue_t *useq,
int op_timeout) {
rd_kafka_queue_t *q = useq ? useq : rd_kafka_queue_new(rk);
const int skip_topic_cnt = 2;
#define MY_DEL_TOPICS_CNT 9
char *topics[MY_DEL_TOPICS_CNT];
rd_kafka_DeleteTopic_t *del_topics[MY_DEL_TOPICS_CNT];
rd_kafka_AdminOptions_t *options = NULL;
rd_kafka_resp_err_t exp_topicerr[MY_DEL_TOPICS_CNT] = {0};
rd_kafka_resp_err_t exp_err = RD_KAFKA_RESP_ERR_NO_ERROR;
/* Expected topics in metadata */
rd_kafka_metadata_topic_t exp_mdtopics[MY_DEL_TOPICS_CNT] = {{0}};
int exp_mdtopic_cnt = 0;
/* Not expected topics in metadata */
rd_kafka_metadata_topic_t exp_not_mdtopics[MY_DEL_TOPICS_CNT] = {{0}};
int exp_not_mdtopic_cnt = 0;
int i;
char errstr[512];
const char *errstr2;
rd_kafka_resp_err_t err;
test_timing_t timing;
rd_kafka_event_t *rkev;
const rd_kafka_DeleteTopics_result_t *res;
const rd_kafka_topic_result_t **restopics;
size_t restopic_cnt;
int metadata_tmout;
TEST_SAY(_C_MAG "[ %s DeleteTopics with %s, op_timeout %d ]\n",
rd_kafka_name(rk), what, op_timeout);
/**
* Construct DeleteTopic array
*/
for (i = 0 ; i < MY_DEL_TOPICS_CNT ; i++) {
char *topic = rd_strdup(test_mk_topic_name(__FUNCTION__, 1));
int notexist_topic = i >= MY_DEL_TOPICS_CNT - skip_topic_cnt;
topics[i] = topic;
del_topics[i] = rd_kafka_DeleteTopic_new(topic);
if (notexist_topic)
exp_topicerr[i] =
RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART;
else {
exp_topicerr[i] =
RD_KAFKA_RESP_ERR_NO_ERROR;
exp_mdtopics[exp_mdtopic_cnt++].topic = topic;
}
exp_not_mdtopics[exp_not_mdtopic_cnt++].topic = topic;
}
if (op_timeout != -1) {
options = rd_kafka_AdminOptions_new(
rk, RD_KAFKA_ADMIN_OP_ANY);
err = rd_kafka_AdminOptions_set_operation_timeout(
options, op_timeout, errstr, sizeof(errstr));
TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
}
/* Create the topics first, minus the skip count. */
test_CreateTopics_simple(rk, NULL, topics,
MY_DEL_TOPICS_CNT-skip_topic_cnt,
2/*num_partitions*/,
NULL);
/* Verify that topics are reported by metadata */
test_wait_metadata_update(rk,
exp_mdtopics, exp_mdtopic_cnt,
NULL, 0,
15*1000);
TIMING_START(&timing, "DeleteTopics");
TEST_SAY("Call DeleteTopics\n");
rd_kafka_DeleteTopics(rk, del_topics, MY_DEL_TOPICS_CNT,
options, q);
TIMING_ASSERT_LATER(&timing, 0, 50);
/* Poll result queue for DeleteTopics result.
* Print but otherwise ignore other event types
* (typically generic Error events). */
TIMING_START(&timing, "DeleteTopics.queue_poll");
while (1) {
rkev = rd_kafka_queue_poll(q, tmout_multip(20*1000));
TEST_SAY("DeleteTopics: got %s in %.3fms\n",
rd_kafka_event_name(rkev),
TIMING_DURATION(&timing) / 1000.0f);
if (rd_kafka_event_error(rkev))
TEST_SAY("%s: %s\n",
rd_kafka_event_name(rkev),
rd_kafka_event_error_string(rkev));
if (rd_kafka_event_type(rkev) ==
RD_KAFKA_EVENT_DELETETOPICS_RESULT)
break;
rd_kafka_event_destroy(rkev);
}
/* Convert event to proper result */
res = rd_kafka_event_DeleteTopics_result(rkev);
TEST_ASSERT(res, "expected DeleteTopics_result, not %s",
rd_kafka_event_name(rkev));
/* Expecting error */
err = rd_kafka_event_error(rkev);
errstr2 = rd_kafka_event_error_string(rkev);
TEST_ASSERT(err == exp_err,
"expected DeleteTopics to return %s, not %s (%s)",
rd_kafka_err2str(exp_err),
rd_kafka_err2str(err),
err ? errstr2 : "n/a");
TEST_SAY("DeleteTopics: returned %s (%s)\n",
rd_kafka_err2str(err), err ? errstr2 : "n/a");
/* Extract topics */
restopics = rd_kafka_DeleteTopics_result_topics(res, &restopic_cnt);
/* Scan topics for proper fields and expected failures. */
for (i = 0 ; i < (int)restopic_cnt ; i++) {
const rd_kafka_topic_result_t *terr = restopics[i];
/* Verify that topic order matches our request. */
if (strcmp(rd_kafka_topic_result_name(terr), topics[i]))
TEST_FAIL_LATER("Topic result order mismatch at #%d: "
"expected %s, got %s",
i, topics[i],
rd_kafka_topic_result_name(terr));
TEST_SAY("DeleteTopics result: #%d: %s: %s: %s\n",
i,
rd_kafka_topic_result_name(terr),
rd_kafka_err2name(rd_kafka_topic_result_error(terr)),
rd_kafka_topic_result_error_string(terr));
if (rd_kafka_topic_result_error(terr) != exp_topicerr[i])
TEST_FAIL_LATER(
"Expected %s, not %d: %s",
rd_kafka_err2name(exp_topicerr[i]),
rd_kafka_topic_result_error(terr),
rd_kafka_err2name(rd_kafka_topic_result_error(
terr)));
}
/**
* Verify that the expected topics are deleted and the non-expected
* are not. Allow it some time to propagate.
*/
if (op_timeout > 0)
metadata_tmout = op_timeout + 1000;
else
metadata_tmout = 10 * 1000;
test_wait_metadata_update(rk,
NULL, 0,
exp_not_mdtopics,
exp_not_mdtopic_cnt,
metadata_tmout);
rd_kafka_event_destroy(rkev);
for (i = 0 ; i < MY_DEL_TOPICS_CNT ; i++) {
rd_kafka_DeleteTopic_destroy(del_topics[i]);
rd_free(topics[i]);
}
if (options)
rd_kafka_AdminOptions_destroy(options);
if (!useq)
rd_kafka_queue_destroy(q);
#undef MY_DEL_TOPICS_CNT
}
int main_0038_performance (int argc, char **argv) {
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
const int partition = 0;
const int msgsize = 100;
uint64_t testid;
rd_kafka_conf_t *conf;
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
test_timing_t t_create, t_produce, t_consume;
int totsize = 1024*1024*128;
int msgcnt;
if (!strcmp(test_mode, "valgrind") || !strcmp(test_mode, "helgrind") ||
!strcmp(test_mode, "drd"))
totsize = 1024*1024*8; /* 8 meg, valgrind is slow. */
msgcnt = totsize / msgsize;
TEST_SAY("Producing %d messages of size %d to %s [%d]\n",
msgcnt, (int)msgsize, topic, partition);
testid = test_id_generate();
test_conf_init(&conf, NULL, 120);
rd_kafka_conf_set_dr_cb(conf, test_dr_cb);
test_conf_set(conf, "queue.buffering.max.messages", "10000000");
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
rkt = test_create_producer_topic(rk, topic, "acks", "1", NULL);
/* First produce one message to create the topic, etc, this might take
* a while and we dont want this to affect the throughput timing. */
TIMING_START(&t_create, "CREATE TOPIC");
test_produce_msgs(rk, rkt, testid, partition, 0, 1, NULL, msgsize);
TIMING_STOP(&t_create);
TIMING_START(&t_produce, "PRODUCE");
test_produce_msgs(rk, rkt, testid, partition, 1, msgcnt-1, NULL, msgsize);
TIMING_STOP(&t_produce);
TEST_SAY("Destroying producer\n");
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
TEST_SAY("Creating consumer\n");
test_conf_init(&conf, NULL, 120);
rk = test_create_consumer(NULL, NULL, conf, NULL, NULL);
rkt = rd_kafka_topic_new(rk, topic, NULL);
test_consumer_start("CONSUME", rkt, partition,
RD_KAFKA_OFFSET_BEGINNING);
TIMING_START(&t_consume, "CONSUME");
test_consume_msgs("CONSUME", rkt, testid, partition, TEST_NO_SEEK,
0, msgcnt, 1);
TIMING_STOP(&t_consume);
test_consumer_stop("CONSUME", rkt, partition);
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
TEST_REPORT("{ \"producer\": "
" { \"mb_per_sec\": %.2f, \"records_per_sec\": %.2f },"
" \"consumer\": "
"{ \"mb_per_sec\": %.2f, \"records_per_sec\": %.2f } "
"}",
(double)
(totsize/((double)TIMING_DURATION(&t_produce)/1000000.0f)) /
1000000.0f,
(float)
(msgcnt/((double)TIMING_DURATION(&t_produce)/1000000.0f)),
(double)
(totsize/((double)TIMING_DURATION(&t_consume)/1000000.0f)) /
1000000.0f,
(float)
(msgcnt/((double)TIMING_DURATION(&t_consume)/1000000.0f)));
return 0;
}
static void do_offset_test (const char *what, int auto_commit, int auto_store,
int async) {
test_timing_t t_all;
char groupid[64];
rd_kafka_t *rk;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *tconf;
int cnt = 0;
const int extra_cnt = 5;
rd_kafka_resp_err_t err;
rd_kafka_topic_partition_list_t *parts;
rd_kafka_topic_partition_t *rktpar;
int64_t next_offset = -1;
test_conf_init(&conf, &tconf, 20);
test_conf_set(conf, "enable.auto.commit", auto_commit ? "true":"false");
test_conf_set(conf, "enable.auto.offset.store", auto_store ?"true":"false");
test_conf_set(conf, "auto.commit.interval.ms", "500");
rd_kafka_conf_set_offset_commit_cb(conf, offset_commit_cb);
test_topic_conf_set(tconf, "auto.offset.reset", "smallest");
test_str_id_generate(groupid, sizeof(groupid));
test_conf_set(conf, "group.id", groupid);
rd_kafka_conf_set_default_topic_conf(conf, tconf);
TEST_SAY(_C_MAG "[ do_offset_test: %s with group.id %s ]\n",
what, groupid);
TIMING_START(&t_all, what);
expected_offset = 0;
committed_offset = -1;
/* MO:
* - Create consumer.
* - Start consuming from beginning
* - Perform store & commits according to settings
* - Stop storing&committing when half of the messages are consumed,
* - but consume 5 more to check against.
* - Query position.
* - Destroy consumer.
* - Create new consumer with same group.id using stored offsets
* - Should consume the expected message.
*/
/* Create kafka instance */
rk = test_create_handle(RD_KAFKA_CONSUMER, rd_kafka_conf_dup(conf));
rd_kafka_poll_set_consumer(rk);
test_consumer_subscribe(rk, topic);
while (cnt - extra_cnt < msgcnt / 2) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(rk, 10*1000);
if (!rkm)
continue;
if (rkm->err == RD_KAFKA_RESP_ERR__TIMED_OUT)
TEST_FAIL("%s: Timed out waiting for message %d", what,cnt);
else if (rkm->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
rd_kafka_message_destroy(rkm);
continue;
} else if (rkm->err)
TEST_FAIL("%s: Consumer error: %s",
what, rd_kafka_message_errstr(rkm));
/* Offset of next message. */
next_offset = rkm->offset + 1;
if (cnt < msgcnt / 2) {
if (!auto_store) {
err = rd_kafka_offset_store(rkm->rkt,rkm->partition,
rkm->offset);
if (err)
TEST_FAIL("%s: offset_store failed: %s\n",
what, rd_kafka_err2str(err));
}
expected_offset = rkm->offset+1;
if (!auto_commit) {
test_timing_t t_commit;
TIMING_START(&t_commit,
async?"commit.async":"commit.sync");
err = rd_kafka_commit_message(rk, rkm, async);
TIMING_STOP(&t_commit);
if (err)
TEST_FAIL("%s: commit failed: %s\n",
what, rd_kafka_err2str(err));
}
} else if (auto_store && auto_commit)
expected_offset = rkm->offset+1;
rd_kafka_message_destroy(rkm);
cnt++;
}
TEST_SAY("%s: done consuming after %d messages, at offset %"PRId64"\n",
what, cnt, expected_offset);
if ((err = rd_kafka_assignment(rk, &parts)))
TEST_FAIL("%s: failed to get assignment(): %s\n",
what, rd_kafka_err2str(err));
/* Verify position */
if ((err = rd_kafka_position(rk, parts)))
TEST_FAIL("%s: failed to get position(): %s\n",
what, rd_kafka_err2str(err));
if (!(rktpar = rd_kafka_topic_partition_list_find(parts,
topic, partition)))
TEST_FAIL("%s: position(): topic lost\n", what);
if (rktpar->offset != next_offset)
TEST_FAIL("%s: Expected position() offset %"PRId64", got %"PRId64,
what, next_offset, rktpar->offset);
TEST_SAY("%s: Position is at %"PRId64", good!\n",
what, rktpar->offset);
/* Pause messages while waiting so we can serve callbacks
* without having more messages received. */
if ((err = rd_kafka_pause_partitions(rk, parts)))
TEST_FAIL("%s: failed to pause partitions: %s\n",
what, rd_kafka_err2str(err));
rd_kafka_topic_partition_list_destroy(parts);
/* Fire off any enqueued offset_commit_cb */
test_consumer_poll_no_msgs(what, rk, testid, 0);
TEST_SAY("%s: committed_offset %"PRId64", expected_offset %"PRId64"\n",
what, committed_offset, expected_offset);
if (!auto_commit && !async) {
/* Sync commits should be up to date at this point. */
if (committed_offset != expected_offset)
TEST_FAIL("%s: Sync commit: committed offset %"PRId64
" should be same as expected offset "
"%"PRId64,
what, committed_offset, expected_offset);
} else {
/* Wait for offset commits to catch up */
while (committed_offset < expected_offset) {
TEST_SAYL(3, "%s: Wait for committed offset %"PRId64
" to reach expected offset %"PRId64"\n",
what, committed_offset, expected_offset);
test_consumer_poll_no_msgs(what, rk, testid, 1000);
}
}
TEST_SAY("%s: phase 1 complete, %d messages consumed, "
"next expected offset is %"PRId64"\n",
what, cnt, expected_offset);
/* Issue #827: cause committed() to return prematurely by specifying
* low timeout. The bug (use after free) will only
* be catched by valgrind. */
do {
parts = rd_kafka_topic_partition_list_new(1);
rd_kafka_topic_partition_list_add(parts, topic, partition);
err = rd_kafka_committed(rk, parts, 1);
rd_kafka_topic_partition_list_destroy(parts);
TEST_SAY("Issue #827: committed() returned %s\n",
rd_kafka_err2str(err));
} while (err != RD_KAFKA_RESP_ERR__TIMED_OUT);
/* Query position */
parts = rd_kafka_topic_partition_list_new(1);
rd_kafka_topic_partition_list_add(parts, topic, partition);
err = rd_kafka_committed(rk, parts, tmout_multip(5*1000));
if (err)
TEST_FAIL("%s: committed() failed: %s", what, rd_kafka_err2str(err));
if (!(rktpar = rd_kafka_topic_partition_list_find(parts,
topic, partition)))
TEST_FAIL("%s: committed(): topic lost\n", what);
if (rktpar->offset != expected_offset)
TEST_FAIL("%s: Expected committed() offset %"PRId64", got %"PRId64,
what, expected_offset, rktpar->offset);
TEST_SAY("%s: Committed offset is at %"PRId64", good!\n",
what, rktpar->offset);
rd_kafka_topic_partition_list_destroy(parts);
test_consumer_close(rk);
rd_kafka_destroy(rk);
/* Fire up a new consumer and continue from where we left off. */
TEST_SAY("%s: phase 2: starting new consumer to resume consumption\n",what);
rk = test_create_handle(RD_KAFKA_CONSUMER, conf);
rd_kafka_poll_set_consumer(rk);
test_consumer_subscribe(rk, topic);
while (cnt < msgcnt) {
rd_kafka_message_t *rkm;
rkm = rd_kafka_consumer_poll(rk, 10*1000);
if (!rkm)
continue;
if (rkm->err == RD_KAFKA_RESP_ERR__TIMED_OUT)
TEST_FAIL("%s: Timed out waiting for message %d", what,cnt);
else if (rkm->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
rd_kafka_message_destroy(rkm);
continue;
} else if (rkm->err)
TEST_FAIL("%s: Consumer error: %s",
what, rd_kafka_message_errstr(rkm));
if (rkm->offset != expected_offset)
TEST_FAIL("%s: Received message offset %"PRId64
", expected %"PRId64" at msgcnt %d/%d\n",
what, rkm->offset, expected_offset,
cnt, msgcnt);
rd_kafka_message_destroy(rkm);
expected_offset++;
cnt++;
}
TEST_SAY("%s: phase 2: complete\n", what);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_all);
}
static void *
ui_updater_thr(void *args)
{
PSUIUPDATERPARAMS params = (PSUIUPDATERPARAMS)args;
uint8_t active = 1;
HPINLIST pin_list;
HPIN pin;
PSMETABUFER meta;
TIMING_MEASURE_AREA;
pin_list = pin_list_create(2);
pin_list_add_custom_fd(pin_list, params->params->infd, PIN_TYPE_CUSTOM);
printf("[updater] started\n");
while (active && pin_list_wait(pin_list, -1) != PIN_ERROR) {
while ( (pin = pin_list_get_next_event(pin_list, PIN_EVENT_READ)) != NULL ) {
pin_read_raw(pin, &meta, PTR_SIZE);
if (meta == MESSAGE_END) {
active = 0;
continue;
}
if ( meta != NULL ) {
TIMING_START();
/* calculate data. send to ui thread */
#if 1 /* samples debug print */
PRINT_LOCK(params->params->print_mutex);
print_header((PSSAMPLEHEADER)meta->left->buf,
meta->left->buf + HEADER_SIZE,
meta->left->size - HEADER_SIZE);
print_header((PSSAMPLEHEADER)meta->right->buf,
meta->right->buf + HEADER_SIZE,
meta->right->size - HEADER_SIZE);
print_header((PSSAMPLEHEADER)meta->left_fft->buf,
meta->left_fft->buf + HEADER_SIZE,
meta->left_fft->size - HEADER_SIZE);
print_header((PSSAMPLEHEADER)meta->right_fft->buf,
meta->right_fft->buf + HEADER_SIZE,
meta->right_fft->size - HEADER_SIZE);
print_header((PSSAMPLEHEADER)meta->sd_log->buf,
meta->sd_log->buf + HEADER_SIZE,
meta->sd_log->size - HEADER_SIZE);
print_header((PSSAMPLEHEADER)meta->sd_mod->buf,
meta->sd_mod->buf + HEADER_SIZE,
meta->sd_mod->size - HEADER_SIZE);
PRINT_UNLOCK(params->params->print_mutex);
#endif
#if 1 /* draw data */
glwin_draw_data(GRAPH_SAMPLES,
(float *)(meta->left->buf + HEADER_SIZE),
(float *)(meta->right->buf + HEADER_SIZE),
((PSSAMPLEHEADER)meta->right->buf)->samples);
glwin_draw_data_c(GRAPH_FFT,
(float *)(meta->left_fft->buf + HEADER_SIZE),
(float *)(meta->right_fft->buf + HEADER_SIZE),
((PSSAMPLEHEADER)meta->right_fft->buf)->samples);
glwin_draw_data(GRAPH_SDENS,
(float *)(meta->sd_log->buf + HEADER_SIZE),
(float *)(meta->sd_mod->buf + HEADER_SIZE),
((PSSAMPLEHEADER)meta->sd_mod->buf)->samples);
glwin_render_data();
#endif
metabuf_free(meta);
TIMING_END(" updater");
}
//buf_free(sample);
}
}
return NULL;
}
/**
* Returns offset of the last message consumed
*/
int64_t test_consume_msgs (const char *what, rd_kafka_topic_t *rkt,
uint64_t testid, int32_t partition, int64_t offset,
int exp_msg_base, int exp_cnt, int parse_fmt) {
int cnt = 0;
int msg_next = exp_msg_base;
int fails = 0;
int64_t offset_last = -1;
test_timing_t t_first, t_all;
TEST_SAY("%s: consume_msgs: %s [%"PRId32"]: expect msg #%d..%d "
"at offset %"PRId64"\n",
what, rd_kafka_topic_name(rkt), partition,
exp_msg_base, exp_cnt, offset);
if (offset != TEST_NO_SEEK) {
rd_kafka_resp_err_t err;
test_timing_t t_seek;
TIMING_START(&t_seek, "SEEK");
if ((err = rd_kafka_seek(rkt, partition, offset, 5000)))
TEST_FAIL("%s: consume_msgs: %s [%"PRId32"]: "
"seek to %"PRId64" failed: %s\n",
what, rd_kafka_topic_name(rkt), partition,
offset, rd_kafka_err2str(err));
TIMING_STOP(&t_seek);
TEST_SAY("%s: seeked to offset %"PRId64"\n", what, offset);
}
TIMING_START(&t_first, "FIRST MSG");
TIMING_START(&t_all, "ALL MSGS");
while (cnt < exp_cnt) {
rd_kafka_message_t *rkmessage;
int msg_id;
rkmessage = rd_kafka_consume(rkt, partition, 5000);
if (!rkmessage)
TEST_FAIL("%s: consume_msgs: %s [%"PRId32"]: "
"expected msg #%d (%d/%d): timed out\n",
what, rd_kafka_topic_name(rkt), partition,
msg_next, cnt, exp_cnt);
if (rkmessage->err)
TEST_FAIL("%s: consume_msgs: %s [%"PRId32"]: "
"expected msg #%d (%d/%d): got error: %s\n",
what, rd_kafka_topic_name(rkt), partition,
msg_next, cnt, exp_cnt,
rd_kafka_err2str(rkmessage->err));
if (cnt == 0)
TIMING_STOP(&t_first);
if (parse_fmt)
test_msg_parse(testid, rkmessage->key,
rkmessage->key_len, partition, &msg_id);
else
msg_id = 0;
if (test_level >= 3)
TEST_SAY("%s: consume_msgs: %s [%"PRId32"]: "
"got msg #%d at offset %"PRId64
" (expect #%d at offset %"PRId64")\n",
what, rd_kafka_topic_name(rkt), partition,
msg_id, rkmessage->offset,
msg_next,
offset >= 0 ? offset + cnt : -1);
if (parse_fmt && msg_id != msg_next) {
TEST_SAY("%s: consume_msgs: %s [%"PRId32"]: "
"expected msg #%d (%d/%d): got msg #%d\n",
what, rd_kafka_topic_name(rkt), partition,
msg_next, cnt, exp_cnt, msg_id);
fails++;
}
cnt++;
msg_next++;
offset_last = rkmessage->offset;
rd_kafka_message_destroy(rkmessage);
}
TIMING_STOP(&t_all);
if (fails)
TEST_FAIL("%s: consume_msgs: %s [%"PRId32"]: %d failures\n",
what, rd_kafka_topic_name(rkt), partition, fails);
TEST_SAY("%s: consume_msgs: %s [%"PRId32"]: "
"%d/%d messages consumed succesfully\n",
what, rd_kafka_topic_name(rkt), partition,
cnt, exp_cnt);
return offset_last;
}
int
main(int argc, char *argv[])
{
uint8_t active = 1;
int error;
int pipefd[2];
/* buffer length in samples. would be multiplied by channels and sample size */
int buffer_length = 1024;
int listen_port = 5002;
uint32_t counter = 0;
pthread_t audio_thr;
/* pthread_attr_t audio_thr_attr; */
/* record parameters */
pa_sample_spec ss = {
/* for fft need PA_SAMPLE_FLOAT32LE or PA_SAMPLE_FLOAT32BE */
.format = PA_SAMPLE_FLOAT32LE, // PA_SAMPLE_S16LE,
.rate = 44100,
.channels = 2
};
SAUDIORECTHRPARAMS audio_thr_params = {0};
HPINLIST connection = NULL;
HPIN pin, pipe_pin;
HBUF sample, dummy_sample;
TIMING_MEASURE_AREA;
if (argc < 2) {
printf("usage: source <listen_port>\n");
}
sscanf(argv[1], "%i", &listen_port);
printf("Will listen %i port\n", listen_port);
connection = pin_list_create(MAX_EVENTS);
pin_listen(connection, listen_port, BACKLOG, NULL);
dummy_sample = buf_alloc(dummy_size_callback);
if (pipe(pipefd) == -1) {
handle_error("pipe()");
}
setnonblocking(pipefd[0]);
setnonblocking(pipefd[1]);
pipe_pin = pin_list_add_custom_fd(connection, pipefd[0], PIN_TYPE_CUSTOM);
/* thread creation */
audio_thr_params.pipefd = pipefd[1];
audio_thr_params.buffer_size = buffer_length;
audio_thr_params.sample_spec = &ss;
audio_thr_params.argc = argc;
audio_thr_params.argv = argv;
audio_thr_params.active = 1;
#if 0
if ( (error = pthread_attr_init(&audio_thr_attr)) != 0 )
handle_error_en(error, "pthread_attr_init()");
if ( (error = pthread_attr_setstacksize(&audio_thr_attr, 5000)) != 0)
handle_error_en(error, "pthread_attr_setstacksize()");
#endif
if ( (error = pthread_create(&audio_thr, NULL, audio_capture_thr, &audio_thr_params)) != 0 )
handle_error_en(error, "pthread_create()");
/**
* :TODO: It's need to improve latency while sending buffers
*/
while (active && pin_list_wait(connection, -1) != PIN_ERROR) {
pin_list_deliver(connection);
while ( (pin = pin_list_get_next_event(connection, PIN_EVENT_READ)) != NULL ) {
if (pin == pipe_pin) {
TIMING_START();
counter = 0;
while ( pin_read_raw(pin, &sample, PTR_SIZE) != 0 ) {
/* if pin = pipe_pin, read pointer to buffer,
write buffer into socket and free it */
/* printf("[source] read %p\n", sample); */
print_header((PSSAMPLEHEADER)sample->buf,
sample->buf + HEADER_SIZE,
sample->size - HEADER_SIZE);
pin_list_write_sample(connection, sample, 0);
buf_free(sample);
counter++;
}
/* printf("[source] %u samples\n", counter); */
TIMING_END("source");
continue;
}
switch (pin_read_sample(pin, dummy_sample)) {
case PIN_STATUS_READY:
{
break;
}
case PIN_STATUS_CLOSED:
{
if (pin->type == PIN_TYPE_INPUT) {
printf("one of inputs closed. exit.\n");
active = 0;
continue;
} else {
printf("connection closed\n");
}
pin_disconnect(pin);
/* close data and skip iteration */
continue;
}
case PIN_STATUS_PARTIAL:
{
printf(" partial data. %u / %u\n", dummy_sample->size, dummy_sample->full_size);
/* do nothing since no data ready */
break;
}
case PIN_STATUS_NO_DATA:
{
printf(" no data. %u / %u\n", dummy_sample->size, dummy_sample->full_size);
/* do nothing since no data ready */
break;
}
default:
{
break;
}
}
dummy_sample->size = 0;
}
}
//finish:
/*
loop until pipe_pin is null.
and free recieved buffers.
*/
audio_thr_params.active = 0;
pthread_join(audio_thr, NULL);
pin_list_destroy(connection);
exit(EXIT_SUCCESS);
}
int main(int argc, char **argv) {
int r = 0;
const char *tests_to_run = NULL; /* all */
int i;
test_timing_t t_all;
#ifndef _MSC_VER
tests_to_run = getenv("TESTS");
#endif
for (i = 1 ; i < argc ; i++) {
if (!strcmp(argv[i], "-p"))
tests_run_in_parallel = 1;
else if (i == 1)
tests_to_run = argv[i];
else {
printf("Unknown option: %s\n"
"\n"
"Usage: %s [options] [<test-match-substr>]\n"
"Options:\n"
" -p Run tests in parallel\n"
"\n",
argv[0], argv[i]);
exit(1);
}
}
test_curr = "<MAIN>";
test_start = test_clock();
TEST_SAY("Tests to run: %s\n", tests_to_run ? tests_to_run : "all");
#define RUN_TEST(NAME) do { \
extern int main_ ## NAME (int, char **); \
if (!tests_to_run || strstr(# NAME, tests_to_run)) { \
r |= run_test(# NAME, main_ ## NAME, argc, argv); \
} else { \
TEST_SAY("================= Skipping test %s " \
"================\n", # NAME ); \
} \
} while (0)
TIMING_START(&t_all, "ALL-TESTS");
RUN_TEST(0001_multiobj);
RUN_TEST(0002_unkpart);
RUN_TEST(0003_msgmaxsize);
RUN_TEST(0004_conf);
RUN_TEST(0005_order);
RUN_TEST(0006_symbols);
RUN_TEST(0007_autotopic);
RUN_TEST(0008_reqacks);
RUN_TEST(0011_produce_batch);
RUN_TEST(0012_produce_consume);
RUN_TEST(0013_null_msgs);
RUN_TEST(0014_reconsume_191);
RUN_TEST(0015_offsets_seek);
RUN_TEST(0017_compression);
RUN_TEST(0018_cgrp_term);
if (tests_run_in_parallel) {
while (tests_running_cnt > 0)
rd_sleep(1);
}
TIMING_STOP(&t_all);
/* Wait for everything to be cleaned up since broker destroys are
* handled in its own thread. */
test_wait_exit(tests_run_in_parallel ? 10 : 5);
/* If we havent failed at this point then
* there were no threads leaked */
TEST_SAY("\n============== ALL TESTS PASSED ==============\n");
return r;
}
static void *
audio_capture_thr(void *args)
{
int error;
PSAUDIORECTHRPARAMS params = (PSAUDIORECTHRPARAMS)args;
pa_simple *pa_context = NULL;
uint8_t active = 1;
uint32_t buffer_size;
SSAMPLEHEADER sample_header = {0};
HBUF sample;
TIMING_MEASURE_AREA;
buffer_size = HEADER_SIZE + params->sample_spec->channels *
params->buffer_size * pa_sample_size(params->sample_spec);
sample_header.number = 0;
sample_header.buf_type = BUF_TYPE_INTERLEAVED;
sample_header.sample_size = pa_sample_size(params->sample_spec);
sample_header.samples = params->buffer_size;
sample_header.channels = params->sample_spec->channels;
sample_header.samplerate = params->sample_spec->rate;
if ( !(pa_context = pa_simple_new(NULL, params->argv[0],
PA_STREAM_RECORD, NULL, "record",
params->sample_spec, NULL, NULL, &error)) ) {
fprintf(stderr, __FILE__": pa_simple_new() failed: %s\n", pa_strerror(error));
goto audio_thr_finish;
}
printf("[source] audio thr\n");
while (active) {
TIMING_START();
sample = buf_alloc(NULL);
buf_resize(sample, buffer_size);
sample->full_size = sample->size = sample->alloced_size;
sample_zero_buffer(sample);
memcpy(sample->buf, &sample_header, sizeof(SSAMPLEHEADER));
if (pa_simple_read(pa_context, sample->buf + HEADER_SIZE,
sample->alloced_size - HEADER_SIZE, &error) < 0) {
fprintf(stderr, __FILE__": pa_simple_read() failed: %s\n", pa_strerror(error));
goto audio_thr_finish;
}
pa_gettimeofday(&(sample_header.timestamp));
sample_header.number += 1;
/* printf("[audio] read %p\n", sample); */
if ( (error = write(params->pipefd, &sample, sizeof(sample))) != sizeof(sample)) {
if (error == -1) {
handle_error("[audio] write()");
}
buf_free(sample);
perror("[audio] ");
printf("[audio] uverrun. free buffer\n");
}
active = params->active;
TIMING_END(" audio");
}
audio_thr_finish:
pa_simple_free(pa_context);
return NULL;
}
int main_0029_assign_offset (int argc, char **argv) {
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_topic_conf_t *tconf;
rd_kafka_topic_partition_list_t *parts;
uint64_t testid;
int i;
test_timing_t t_simple, t_hl;
test_msgver_t mv;
test_conf_init(NULL, &tconf, 20 + (test_session_timeout_ms * 3 / 1000));
test_topic_conf_set(tconf, "auto.offset.reset", "smallest");
/* Produce X messages to Y partitions so we get a
* nice seekable 0..X offset one each partition. */
/* Produce messages */
testid = test_id_generate();
rk = test_create_producer();
rkt = test_create_producer_topic(rk, topic, NULL);
parts = rd_kafka_topic_partition_list_new(partitions);
for (i = 0 ; i < partitions ; i++) {
test_produce_msgs(rk, rkt, testid, i, 0, msgcnt, NULL, 0);
rd_kafka_topic_partition_list_add(parts, topic, i)->offset =
msgcnt / 2;
}
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
/* Simple consumer */
TIMING_START(&t_simple, "SIMPLE.CONSUMER");
rk = test_create_consumer(topic, NULL, NULL, NULL);
test_msgver_init(&mv, testid);
test_consumer_assign("SIMPLE.ASSIGN", rk, parts);
test_consumer_poll("SIMPLE.CONSUME", rk, testid, -1, 0,
partitions * (msgcnt / 2), &mv);
for (i = 0 ; i < partitions ; i++)
test_msgver_verify_part("HL.MSGS", &mv, TEST_MSGVER_ALL_PART,
topic, i, msgcnt/2, msgcnt/2);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_simple);
rd_kafka_topic_partition_list_destroy(parts);
/* High-level consumer.
* Offsets are set in rebalance callback. */
TIMING_START(&t_hl, "HL.CONSUMER");
test_msgver_init(&mv, testid);
rk = test_create_consumer(topic, rebalance_cb, NULL, NULL);
test_consumer_subscribe(rk, topic);
test_consumer_poll("HL.CONSUME", rk, testid, -1, 0,
partitions * (msgcnt / 2), &mv);
for (i = 0 ; i < partitions ; i++)
test_msgver_verify_part("HL.MSGS", &mv, TEST_MSGVER_ALL_PART,
topic, i, msgcnt/2, msgcnt/2);
test_msgver_clear(&mv);
test_consumer_close(rk);
rd_kafka_destroy(rk);
TIMING_STOP(&t_hl);
rd_kafka_topic_conf_destroy(tconf);
return 0;
}
