/**
* Take action when the offset for a toppar becomes unusable.
*
* Locality: toppar handler thread
* Locks: toppar_lock() MUST be held
*/
void rd_kafka_offset_reset (rd_kafka_toppar_t *rktp, int64_t err_offset,
rd_kafka_resp_err_t err, const char *reason) {
int64_t offset = RD_KAFKA_OFFSET_INVALID;
rd_kafka_op_t *rko;
/* Enqueue op for toppar handler thread if we're on the wrong thread. */
if (!thrd_is_current(rktp->rktp_rkt->rkt_rk->rk_thread)) {
rd_kafka_op_t *rko = rd_kafka_op_new(RD_KAFKA_OP_CALLBACK);
rko->rko_op_cb = rd_kafka_offset_reset_op_cb;
rko->rko_rktp = rd_kafka_toppar_keep(rktp);
rko->rko_err = err;
rko->rko_offset = err_offset;
rko->rko_flags |= RD_KAFKA_OP_F_FREE;
rko->rko_payload = rd_strdup(reason);
rko->rko_len = strlen(reason);
rd_kafka_q_enq(&rktp->rktp_ops, rko);
return;
}
if (err_offset == RD_KAFKA_OFFSET_INVALID || err)
offset = rktp->rktp_rkt->rkt_conf.auto_offset_reset;
else
offset = err_offset;
if (offset == RD_KAFKA_OFFSET_INVALID) {
/* Error, auto.offset.reset tells us to error out. */
rko = rd_kafka_op_new(RD_KAFKA_OP_CONSUMER_ERR);
rko->rko_err = err;
rko->rko_rkmessage.offset = err_offset;
rko->rko_rkmessage.partition = rktp->rktp_partition;
rko->rko_payload = rd_strdup(reason);
rko->rko_len = strlen(rko->rko_payload);
rko->rko_flags |= RD_KAFKA_OP_F_FREE;
rko->rko_rktp = rd_kafka_toppar_keep(rktp);
rd_kafka_q_enq(&rktp->rktp_fetchq, rko);
rd_kafka_toppar_set_fetch_state(
rktp, RD_KAFKA_TOPPAR_FETCH_NONE);
} else {
/* Query logical offset */
rktp->rktp_query_offset = offset;
rd_kafka_toppar_set_fetch_state(
rktp, RD_KAFKA_TOPPAR_FETCH_OFFSET_QUERY);
}
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
"%s [%"PRId32"]: offset reset (at offset %s) "
"to %s: %s: %s",
rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition,
rd_kafka_offset2str(err_offset),
rd_kafka_offset2str(offset),
reason, rd_kafka_err2str(err));
if (rktp->rktp_fetch_state == RD_KAFKA_TOPPAR_FETCH_OFFSET_QUERY)
rd_kafka_toppar_offset_request(rktp, rktp->rktp_query_offset, 0);
}
int main_0042_many_topics (int argc, char **argv) {
char **topics;
const int topic_cnt = 20; /* up this as needed, topic creation
* takes time so unless hunting a bug
* we keep this low to keep the
* test suite run time down. */
uint64_t testid;
int i;
test_conf_init(NULL, NULL, 60);
testid = test_id_generate();
/* Generate unique topic names */
topics = malloc(sizeof(*topics) * topic_cnt);
for (i = 0 ; i < topic_cnt ; i++)
topics[i] = rd_strdup(test_mk_topic_name(__FUNCTION__, 1));
produce_many(topics, topic_cnt, testid);
legacy_consume_many(topics, topic_cnt, testid);
if (test_broker_version >= TEST_BRKVER(0,9,0,0)) {
subscribe_consume_many(topics, topic_cnt, testid);
assign_consume_many(topics, topic_cnt, testid);
}
for (i = 0 ; i < topic_cnt ; i++)
free(topics[i]);
free(topics);
return 0;
}
/**
* Enqueue ERR__THROTTLE op, if desired.
*/
void rd_kafka_op_throttle_time (rd_kafka_broker_t *rkb,
rd_kafka_q_t *rkq,
int throttle_time) {
rd_kafka_op_t *rko;
rd_avg_add(&rkb->rkb_avg_throttle, throttle_time);
if (!rkb->rkb_rk->rk_conf.quota_support)
return;
/* We send throttle events when:
* - throttle_time > 0
* - throttle_time == 0 and last throttle_time > 0
*/
if (!throttle_time && !rd_atomic32_get(&rkb->rkb_rk->rk_last_throttle))
return;
rd_atomic32_set(&rkb->rkb_rk->rk_last_throttle, throttle_time);
rko = rd_kafka_op_new(RD_KAFKA_OP_THROTTLE);
rko->rko_nodename = rd_strdup(rkb->rkb_nodename);
rko->rko_flags |= RD_KAFKA_OP_F_FREE; /* free nodename */
rko->rko_nodeid = rkb->rkb_nodeid;
rko->rko_throttle_time = throttle_time;
rd_kafka_q_enq(rkq, rko);
}
static void do_test_non_exist_and_partchange (void) {
char *topic_a = rd_strdup(test_mk_topic_name("topic_a", 1));
rd_kafka_t *rk;
rd_kafka_conf_t *conf;
rd_kafka_queue_t *queue;
/**
* Test #1:
* - Subscribe to non-existing topic.
* - Verify empty assignment
* - Create topic
* - Verify new assignment containing topic
*/
TEST_SAY("#1 & #2 testing\n");
test_conf_init(&conf, NULL, 60);
/* Decrease metadata interval to speed up topic change discovery. */
test_conf_set(conf, "metadata.max.age.ms", "5000");
rd_kafka_conf_set_events(conf, RD_KAFKA_EVENT_REBALANCE);
rk = test_create_consumer(test_str_id_generate_tmp(),
NULL, conf, NULL, NULL);
queue = rd_kafka_queue_get_consumer(rk);
TEST_SAY("#1: Subscribing to %s\n", topic_a);
test_consumer_subscribe(rk, topic_a);
/* Should not see a rebalance since no topics are matched. */
await_no_rebalance("#1: empty", rk, queue, 10000);
TEST_SAY("#1: creating topic %s\n", topic_a);
test_create_topic(topic_a, 2, 1);
await_assignment("#1: proper", rk, queue, 1,
topic_a, 2);
/**
* Test #2 (continue with #1 consumer)
* - Increase the partition count
* - Verify updated assignment
*/
test_kafka_topics("--alter --topic %s --partitions 4",
topic_a);
await_revoke("#2", rk, queue);
await_assignment("#2: more partitions", rk, queue, 1,
topic_a, 4);
test_consumer_close(rk);
rd_kafka_queue_destroy(queue);
rd_kafka_destroy(rk);
rd_free(topic_a);
}
/**
* @brief Latest thread-local dl error, normalized to suit our logging.
* @returns a newly allocated string that must be freed
*/
static char *rd_dl_error (void) {
#if WITH_LIBDL
char *errstr;
char *s;
errstr = dlerror();
if (!errstr)
return rd_strdup("No error returned from dlerror()");
errstr = rd_strdup(errstr);
/* Change newlines to separators. */
while ((s = strchr(errstr, '\n')))
*s = '.';
return errstr;
#elif defined(_MSC_VER)
char buf[1024];
rd_strerror_w32(GetLastError(), buf, sizeof(buf));
return rd_strdup(buf);
#endif
}
/**
* Enqueue op for app. Convenience function
*/
void rd_kafka_op_app_fmt (rd_kafka_q_t *rkq, rd_kafka_op_type_t type,
rd_kafka_toppar_t *rktp,
rd_kafka_resp_err_t err,
const char *fmt, ...) {
char buf[1024];
va_list ap;
va_start(ap, fmt);
rd_vsnprintf(buf, sizeof(buf), fmt, ap);
va_end(ap);
rd_kafka_op_app(rkq, type, RD_KAFKA_OP_F_FREE,
rktp, err, rd_strdup(buf), strlen(buf), NULL);
}
/**
* Propagate an error event to the application on a specific queue.
* \p optype should be RD_KAFKA_OP_ERR for generic errors and
* RD_KAFKA_OP_CONSUMER_ERR for consumer errors.
*/
void rd_kafka_q_op_err (rd_kafka_q_t *rkq, rd_kafka_op_type_t optype,
rd_kafka_resp_err_t err, int32_t version,
const char *fmt, ...) {
va_list ap;
char buf[2048];
va_start(ap, fmt);
rd_vsnprintf(buf, sizeof(buf), fmt, ap);
va_end(ap);
rd_kafka_op_app_reply(rkq, optype, err, version,
rd_strdup(buf), strlen(buf));
}
/**
* sprintf a message in rko->rko_payload (typically error string)
*/
void rd_kafka_op_sprintf (rd_kafka_op_t *rko, const char *fmt, ...) {
va_list ap;
char buf[2048];
va_start(ap, fmt);
rd_vsnprintf(buf, sizeof(buf), fmt, ap);
va_end(ap);
rd_kafka_assert(NULL, !rko->rko_payload);
rko->rko_payload = rd_strdup(buf);
rko->rko_len = strlen(buf);
rko->rko_flags |= RD_KAFKA_OP_F_FREE;
rko->rko_free_cb = rd_free;
}
/**
* Propogate an error event to the application.
* If no error_cb has been set by the application the error will
* be logged instead.
*/
void rd_kafka_op_err (rd_kafka_t *rk, rd_kafka_resp_err_t err,
const char *fmt, ...) {
va_list ap;
char buf[2048];
va_start(ap, fmt);
rd_vsnprintf(buf, sizeof(buf), fmt, ap);
va_end(ap);
if (rk->rk_conf.error_cb)
rd_kafka_op_app_reply(&rk->rk_rep, RD_KAFKA_OP_ERR, err, 0,
rd_strdup(buf), strlen(buf));
else
rd_kafka_log_buf(rk, LOG_ERR, "ERROR", buf);
}
/**
* Propagate an error event to the application on a specific queue.
* \p optype should be RD_KAFKA_OP_ERR for generic errors and
* RD_KAFKA_OP_CONSUMER_ERR for consumer errors.
*/
void rd_kafka_q_op_err (rd_kafka_q_t *rkq, rd_kafka_op_type_t optype,
rd_kafka_resp_err_t err, int32_t version,
rd_kafka_toppar_t *rktp, int64_t offset,
const char *fmt, ...) {
va_list ap;
char buf[2048];
rd_kafka_op_t *rko;
va_start(ap, fmt);
rd_vsnprintf(buf, sizeof(buf), fmt, ap);
va_end(ap);
rko = rd_kafka_op_new(optype);
rko->rko_version = version;
rko->rko_err = err;
rko->rko_u.err.offset = offset;
rko->rko_u.err.errstr = rd_strdup(buf);
if (rktp)
rko->rko_rktp = rd_kafka_toppar_keep(rktp);
rd_kafka_q_enq(rkq, rko);
}
/**
* Enqueue ERR__THROTTLE op, if desired.
*/
void rd_kafka_op_throttle_time (rd_kafka_broker_t *rkb,
rd_kafka_q_t *rkq,
int throttle_time) {
rd_kafka_op_t *rko;
rd_avg_add(&rkb->rkb_avg_throttle, throttle_time);
/* We send throttle events when:
* - throttle_time > 0
* - throttle_time == 0 and last throttle_time > 0
*/
if (!rkb->rkb_rk->rk_conf.throttle_cb ||
(!throttle_time && !rd_atomic32_get(&rkb->rkb_rk->rk_last_throttle)))
return;
rd_atomic32_set(&rkb->rkb_rk->rk_last_throttle, throttle_time);
rko = rd_kafka_op_new(RD_KAFKA_OP_THROTTLE);
rd_kafka_op_set_prio(rko, RD_KAFKA_PRIO_HIGH);
rko->rko_u.throttle.nodename = rd_strdup(rkb->rkb_nodename);
rko->rko_u.throttle.nodeid = rkb->rkb_nodeid;
rko->rko_u.throttle.throttle_time = throttle_time;
rd_kafka_q_enq(rkq, rko);
}
/**
* Send an op back to the application.
*
* Locality: Kafka thread
*/
void rd_kafka_op_app_reply (rd_kafka_q_t *rkq,
rd_kafka_op_type_t type,
rd_kafka_resp_err_t err,
int32_t version,
void *payload, size_t len) {
rd_kafka_op_t *rko;
rko = rd_kafka_op_new(type);
if (err && !payload) {
/* Provide human readable error string if not provided. */
payload = rd_strdup(rd_kafka_err2str(err));
len = strlen(payload);
}
rko->rko_flags |= RD_KAFKA_OP_F_FREE;
rko->rko_version = version;
rko->rko_payload = payload;
rko->rko_len = len;
rko->rko_err = err;
rd_kafka_q_enq(rkq, rko);
}
static void do_test_regex (void) {
char *base_topic = rd_strdup(test_mk_topic_name("topic", 1));
char *topic_b = rd_strdup(tsprintf("%s_b", base_topic));
char *topic_c = rd_strdup(tsprintf("%s_c", base_topic));
char *topic_d = rd_strdup(tsprintf("%s_d", base_topic));
char *topic_e = rd_strdup(tsprintf("%s_e", base_topic));
rd_kafka_t *rk;
rd_kafka_conf_t *conf;
rd_kafka_queue_t *queue;
/**
* Regex test:
* - Create topic b
* - Subscribe to b & d & e
* - Verify b assignment
* - Create topic c
* - Verify no rebalance
* - Create topic d
* - Verify b & d assignment
*/
TEST_SAY("Regex testing\n");
test_conf_init(&conf, NULL, 60);
/* Decrease metadata interval to speed up topic change discovery. */
test_conf_set(conf, "metadata.max.age.ms", "5000");
rd_kafka_conf_set_events(conf, RD_KAFKA_EVENT_REBALANCE);
rk = test_create_consumer(test_str_id_generate_tmp(),
NULL, conf, NULL, NULL);
queue = rd_kafka_queue_get_consumer(rk);
TEST_SAY("Regex: creating topic %s (subscribed)\n", topic_b);
test_create_topic(topic_b, 2, 1);
rd_sleep(1); // FIXME: do check&wait loop instead
TEST_SAY("Regex: Subscribing to %s & %s & %s\n",
topic_b, topic_d, topic_e);
test_consumer_subscribe(rk, tsprintf("^%s_[bde]$", base_topic));
await_assignment("Regex: just one topic exists", rk, queue, 1,
topic_b, 2);
TEST_SAY("Regex: creating topic %s (not subscribed)\n", topic_c);
test_create_topic(topic_c, 4, 1);
/* Should not see a rebalance since no topics are matched. */
await_no_rebalance("Regex: empty", rk, queue, 10000);
TEST_SAY("Regex: creating topic %s (subscribed)\n", topic_d);
test_create_topic(topic_d, 1, 1);
await_revoke("Regex: rebalance after topic creation", rk, queue);
await_assignment("Regex: two topics exist", rk, queue, 2,
topic_b, 2,
topic_d, 1);
test_consumer_close(rk);
rd_kafka_queue_destroy(queue);
rd_kafka_destroy(rk);
rd_free(base_topic);
rd_free(topic_b);
rd_free(topic_c);
rd_free(topic_d);
rd_free(topic_e);
}
/**
* Prepare a toppar for using an offset file.
*
* Locality: rdkafka main thread
* Locks: toppar_lock(rktp) must be held
*/
static void rd_kafka_offset_file_init (rd_kafka_toppar_t *rktp) {
char spath[4096];
const char *path = rktp->rktp_rkt->rkt_conf.offset_store_path;
int64_t offset = RD_KAFKA_OFFSET_INVALID;
if (rd_kafka_path_is_dir(path)) {
char tmpfile[1024];
char escfile[4096];
/* Include group.id in filename if configured. */
if (!RD_KAFKAP_STR_IS_NULL(rktp->rktp_rkt->rkt_rk->
rk_conf.group_id))
rd_snprintf(tmpfile, sizeof(tmpfile),
"%s-%"PRId32"-%.*s.offset",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_rk->
rk_conf.group_id));
else
rd_snprintf(tmpfile, sizeof(tmpfile),
"%s-%"PRId32".offset",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition);
/* Escape filename to make it safe. */
mk_esc_filename(tmpfile, escfile, sizeof(escfile));
rd_snprintf(spath, sizeof(spath), "%s%s%s",
path, path[strlen(path)-1] == '/' ? "" : "/", escfile);
path = spath;
}
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
"%s [%"PRId32"]: using offset file %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
path);
rktp->rktp_offset_path = rd_strdup(path);
/* Set up the offset file sync interval. */
if (rktp->rktp_rkt->rkt_conf.offset_store_sync_interval_ms > 0)
rd_kafka_timer_start(&rktp->rktp_rkt->rkt_rk->rk_timers,
&rktp->rktp_offset_sync_tmr,
rktp->rktp_rkt->rkt_conf.
offset_store_sync_interval_ms * 1000ll,
rd_kafka_offset_sync_tmr_cb, rktp);
if (rd_kafka_offset_file_open(rktp) != -1) {
/* Read offset from offset file. */
offset = rd_kafka_offset_file_read(rktp);
}
if (offset != RD_KAFKA_OFFSET_INVALID) {
/* Start fetching from offset */
rktp->rktp_stored_offset = offset;
rktp->rktp_committed_offset = offset;
rd_kafka_toppar_next_offset_handle(rktp, offset);
} else {
/* Offset was not usable: perform offset reset logic */
rktp->rktp_committed_offset = RD_KAFKA_OFFSET_INVALID;
rd_kafka_offset_reset(rktp, RD_KAFKA_OFFSET_INVALID,
RD_KAFKA_RESP_ERR__FS,
"non-readable offset file");
}
}
/**
* @brief Test creation of partitions
*
*
*/
static void do_test_CreatePartitions (const char *what,
rd_kafka_t *rk, rd_kafka_queue_t *useq,
int op_timeout) {
rd_kafka_queue_t *q = useq ? useq : rd_kafka_queue_new(rk);
#define MY_CRP_TOPICS_CNT 9
char *topics[MY_CRP_TOPICS_CNT];
rd_kafka_NewTopic_t *new_topics[MY_CRP_TOPICS_CNT];
rd_kafka_NewPartitions_t *crp_topics[MY_CRP_TOPICS_CNT];
rd_kafka_AdminOptions_t *options = NULL;
/* Expected topics in metadata */
rd_kafka_metadata_topic_t exp_mdtopics[MY_CRP_TOPICS_CNT] = {{0}};
rd_kafka_metadata_partition_t exp_mdparts[2] = {{0}};
int exp_mdtopic_cnt = 0;
int i;
char errstr[512];
rd_kafka_resp_err_t err;
test_timing_t timing;
int metadata_tmout;
int num_replicas = (int)avail_broker_cnt;
TEST_SAY(_C_MAG "[ %s CreatePartitions with %s, op_timeout %d ]\n",
rd_kafka_name(rk), what, op_timeout);
/* Set up two expected partitions with different replication sets
* so they can be matched by the metadata checker later.
* Even partitions use exp_mdparts[0] while odd partitions
* use exp_mdparts[1]. */
/* Set valid replica assignments (even, and odd (reverse) ) */
exp_mdparts[0].replicas = rd_alloca(sizeof(*exp_mdparts[0].replicas) *
num_replicas);
exp_mdparts[1].replicas = rd_alloca(sizeof(*exp_mdparts[1].replicas) *
num_replicas);
exp_mdparts[0].replica_cnt = num_replicas;
exp_mdparts[1].replica_cnt = num_replicas;
for (i = 0 ; i < num_replicas ; i++) {
exp_mdparts[0].replicas[i] = avail_brokers[i];
exp_mdparts[1].replicas[i] = avail_brokers[num_replicas-i-1];
}
/**
* Construct CreatePartitions array
*/
for (i = 0 ; i < MY_CRP_TOPICS_CNT ; i++) {
char *topic = rd_strdup(test_mk_topic_name(__FUNCTION__, 1));
int initial_part_cnt = 1 + (i * 2);
int new_part_cnt = 1 + (i / 2);
int final_part_cnt = initial_part_cnt + new_part_cnt;
int set_replicas = !(i % 2);
int pi;
topics[i] = topic;
/* Topic to create with initial partition count */
new_topics[i] = rd_kafka_NewTopic_new(topic, initial_part_cnt,
set_replicas ?
-1 : num_replicas,
NULL, 0);
/* .. and later add more partitions to */
crp_topics[i] = rd_kafka_NewPartitions_new(topic,
final_part_cnt,
errstr,
sizeof(errstr));
if (set_replicas) {
exp_mdtopics[exp_mdtopic_cnt].partitions =
rd_alloca(final_part_cnt *
sizeof(*exp_mdtopics[exp_mdtopic_cnt].
partitions));
for (pi = 0 ; pi < final_part_cnt ; pi++) {
const rd_kafka_metadata_partition_t *exp_mdp =
&exp_mdparts[pi & 1];
exp_mdtopics[exp_mdtopic_cnt].
partitions[pi] = *exp_mdp; /* copy */
exp_mdtopics[exp_mdtopic_cnt].
partitions[pi].id = pi;
if (pi < initial_part_cnt) {
/* Set replica assignment
* for initial partitions */
err = rd_kafka_NewTopic_set_replica_assignment(
new_topics[i], pi,
exp_mdp->replicas,
(size_t)exp_mdp->replica_cnt,
errstr, sizeof(errstr));
TEST_ASSERT(!err, "NewTopic_set_replica_assignment: %s",
errstr);
} else {
/* Set replica assignment for new
* partitions */
err = rd_kafka_NewPartitions_set_replica_assignment(
crp_topics[i],
pi - initial_part_cnt,
exp_mdp->replicas,
(size_t)exp_mdp->replica_cnt,
errstr, sizeof(errstr));
TEST_ASSERT(!err, "NewPartitions_set_replica_assignment: %s",
errstr);
}
}
}
TEST_SAY(_C_YEL "Topic %s with %d initial partitions will grow "
"by %d to %d total partitions with%s replicas set\n",
topics[i],
initial_part_cnt, new_part_cnt, final_part_cnt,
set_replicas ? "" : "out");
exp_mdtopics[exp_mdtopic_cnt].topic = topic;
exp_mdtopics[exp_mdtopic_cnt].partition_cnt = final_part_cnt;
exp_mdtopic_cnt++;
}
if (op_timeout != -1) {
options = rd_kafka_AdminOptions_new(
rk, RD_KAFKA_ADMIN_OP_ANY);
err = rd_kafka_AdminOptions_set_operation_timeout(
options, op_timeout, errstr, sizeof(errstr));
TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
}
/*
* Create topics with initial partition count
*/
TIMING_START(&timing, "CreateTopics");
TEST_SAY("Creating topics with initial partition counts\n");
rd_kafka_CreateTopics(rk, new_topics, MY_CRP_TOPICS_CNT,
options, q);
TIMING_ASSERT_LATER(&timing, 0, 50);
err = test_wait_topic_admin_result(q,
RD_KAFKA_EVENT_CREATETOPICS_RESULT,
NULL, 15000);
TEST_ASSERT(!err, "CreateTopics failed: %s", rd_kafka_err2str(err));
rd_kafka_NewTopic_destroy_array(new_topics, MY_CRP_TOPICS_CNT);
/*
* Create new partitions
*/
TIMING_START(&timing, "CreatePartitions");
TEST_SAY("Creating partitions\n");
rd_kafka_CreatePartitions(rk, crp_topics, MY_CRP_TOPICS_CNT,
options, q);
TIMING_ASSERT_LATER(&timing, 0, 50);
err = test_wait_topic_admin_result(q,
RD_KAFKA_EVENT_CREATEPARTITIONS_RESULT,
NULL, 15000);
TEST_ASSERT(!err, "CreatePartitions failed: %s", rd_kafka_err2str(err));
rd_kafka_NewPartitions_destroy_array(crp_topics, MY_CRP_TOPICS_CNT);
/**
* Verify that the expected topics are deleted and the non-expected
* are not. Allow it some time to propagate.
*/
if (op_timeout > 0)
metadata_tmout = op_timeout + 1000;
else
metadata_tmout = 10 * 1000;
test_wait_metadata_update(rk,
exp_mdtopics,
exp_mdtopic_cnt,
NULL, 0,
metadata_tmout);
for (i = 0 ; i < MY_CRP_TOPICS_CNT ; i++)
rd_free(topics[i]);
if (options)
rd_kafka_AdminOptions_destroy(options);
if (!useq)
rd_kafka_queue_destroy(q);
#undef MY_CRP_TOPICS_CNT
}
static void do_test_CreateTopics (const char *what,
rd_kafka_t *rk, rd_kafka_queue_t *useq,
int op_timeout, rd_bool_t validate_only) {
rd_kafka_queue_t *q = useq ? useq : rd_kafka_queue_new(rk);
#define MY_NEW_TOPICS_CNT 6
char *topics[MY_NEW_TOPICS_CNT];
rd_kafka_NewTopic_t *new_topics[MY_NEW_TOPICS_CNT];
rd_kafka_AdminOptions_t *options = NULL;
rd_kafka_resp_err_t exp_topicerr[MY_NEW_TOPICS_CNT] = {0};
rd_kafka_resp_err_t exp_err = RD_KAFKA_RESP_ERR_NO_ERROR;
/* Expected topics in metadata */
rd_kafka_metadata_topic_t exp_mdtopics[MY_NEW_TOPICS_CNT] = {{0}};
int exp_mdtopic_cnt = 0;
/* Not expected topics in metadata */
rd_kafka_metadata_topic_t exp_not_mdtopics[MY_NEW_TOPICS_CNT] = {{0}};
int exp_not_mdtopic_cnt = 0;
int i;
char errstr[512];
const char *errstr2;
rd_kafka_resp_err_t err;
test_timing_t timing;
rd_kafka_event_t *rkev;
const rd_kafka_CreateTopics_result_t *res;
const rd_kafka_topic_result_t **restopics;
size_t restopic_cnt;
int metadata_tmout ;
int num_replicas = (int)avail_broker_cnt;
int32_t *replicas;
/* Set up replicas */
replicas = rd_alloca(sizeof(*replicas) * num_replicas);
for (i = 0 ; i < num_replicas ; i++)
replicas[i] = avail_brokers[i];
TEST_SAY(_C_MAG "[ %s CreateTopics with %s, "
"op_timeout %d, validate_only %d ]\n",
rd_kafka_name(rk), what, op_timeout, validate_only);
/**
* Construct NewTopic array with different properties for
* different partitions.
*/
for (i = 0 ; i < MY_NEW_TOPICS_CNT ; i++) {
char *topic = rd_strdup(test_mk_topic_name(__FUNCTION__, 1));
int num_parts = i * 7 + 1;
int set_config = (i & 1);
int add_invalid_config = (i == 1);
int set_replicas = !(i % 3);
rd_kafka_resp_err_t this_exp_err = RD_KAFKA_RESP_ERR_NO_ERROR;
topics[i] = topic;
new_topics[i] = rd_kafka_NewTopic_new(topic,
num_parts,
set_replicas ? -1 :
num_replicas,
NULL, 0);
if (set_config) {
/*
* Add various configuration properties
*/
err = rd_kafka_NewTopic_set_config(
new_topics[i], "compression.type", "lz4");
TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
err = rd_kafka_NewTopic_set_config(
new_topics[i], "delete.retention.ms", "900");
TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
}
if (add_invalid_config) {
/* Add invalid config property */
err = rd_kafka_NewTopic_set_config(
new_topics[i],
"dummy.doesntexist",
"broker is verifying this");
TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
this_exp_err = RD_KAFKA_RESP_ERR_INVALID_CONFIG;
}
TEST_SAY("Expected result for topic #%d: %s "
"(set_config=%d, add_invalid_config=%d, "
"set_replicas=%d)\n",
i, rd_kafka_err2name(this_exp_err),
set_config, add_invalid_config, set_replicas);
if (set_replicas) {
int32_t p;
/*
* Set valid replica assignments
*/
for (p = 0 ; p < num_parts ; p++) {
err = rd_kafka_NewTopic_set_replica_assignment(
new_topics[i], p,
replicas, num_replicas,
errstr, sizeof(errstr));
TEST_ASSERT(!err, "%s", errstr);
}
}
if (this_exp_err || validate_only) {
exp_topicerr[i] = this_exp_err;
exp_not_mdtopics[exp_not_mdtopic_cnt++].topic = topic;
} else {
exp_mdtopics[exp_mdtopic_cnt].topic = topic;
exp_mdtopics[exp_mdtopic_cnt].partition_cnt =
num_parts;
exp_mdtopic_cnt++;
}
}
if (op_timeout != -1 || validate_only) {
options = rd_kafka_AdminOptions_new(
rk, RD_KAFKA_ADMIN_OP_CREATETOPICS);
if (op_timeout != -1) {
err = rd_kafka_AdminOptions_set_operation_timeout(
options, op_timeout, errstr, sizeof(errstr));
TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
}
if (validate_only) {
err = rd_kafka_AdminOptions_set_validate_only(
options, validate_only, errstr, sizeof(errstr));
TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
}
}
TIMING_START(&timing, "CreateTopics");
TEST_SAY("Call CreateTopics\n");
rd_kafka_CreateTopics(rk, new_topics, MY_NEW_TOPICS_CNT,
options, q);
TIMING_ASSERT_LATER(&timing, 0, 50);
/* Poll result queue for CreateTopics result.
* Print but otherwise ignore other event types
* (typically generic Error events). */
TIMING_START(&timing, "CreateTopics.queue_poll");
do {
rkev = rd_kafka_queue_poll(q, tmout_multip(20*1000));
TEST_SAY("CreateTopics: got %s in %.3fms\n",
rd_kafka_event_name(rkev),
TIMING_DURATION(&timing) / 1000.0f);
if (rd_kafka_event_error(rkev))
TEST_SAY("%s: %s\n",
rd_kafka_event_name(rkev),
rd_kafka_event_error_string(rkev));
} while (rd_kafka_event_type(rkev) !=
RD_KAFKA_EVENT_CREATETOPICS_RESULT);
/* Convert event to proper result */
res = rd_kafka_event_CreateTopics_result(rkev);
TEST_ASSERT(res, "expected CreateTopics_result, not %s",
rd_kafka_event_name(rkev));
/* Expecting error */
err = rd_kafka_event_error(rkev);
errstr2 = rd_kafka_event_error_string(rkev);
TEST_ASSERT(err == exp_err,
"expected CreateTopics to return %s, not %s (%s)",
rd_kafka_err2str(exp_err),
rd_kafka_err2str(err),
err ? errstr2 : "n/a");
TEST_SAY("CreateTopics: returned %s (%s)\n",
rd_kafka_err2str(err), err ? errstr2 : "n/a");
/* Extract topics */
restopics = rd_kafka_CreateTopics_result_topics(res, &restopic_cnt);
/* Scan topics for proper fields and expected failures. */
for (i = 0 ; i < (int)restopic_cnt ; i++) {
const rd_kafka_topic_result_t *terr = restopics[i];
/* Verify that topic order matches our request. */
if (strcmp(rd_kafka_topic_result_name(terr), topics[i]))
TEST_FAIL_LATER("Topic result order mismatch at #%d: "
"expected %s, got %s",
i, topics[i],
rd_kafka_topic_result_name(terr));
TEST_SAY("CreateTopics result: #%d: %s: %s: %s\n",
i,
rd_kafka_topic_result_name(terr),
rd_kafka_err2name(rd_kafka_topic_result_error(terr)),
rd_kafka_topic_result_error_string(terr));
if (rd_kafka_topic_result_error(terr) != exp_topicerr[i])
TEST_FAIL_LATER(
"Expected %s, not %d: %s",
rd_kafka_err2name(exp_topicerr[i]),
rd_kafka_topic_result_error(terr),
rd_kafka_err2name(rd_kafka_topic_result_error(
terr)));
}
/**
* Verify that the expecteded topics are created and the non-expected
* are not. Allow it some time to propagate.
*/
if (validate_only) {
/* No topics should have been created, give it some time
* before checking. */
rd_sleep(2);
metadata_tmout = 5 * 1000;
} else {
if (op_timeout > 0)
metadata_tmout = op_timeout + 1000;
else
metadata_tmout = 10 * 1000;
}
test_wait_metadata_update(rk,
exp_mdtopics,
exp_mdtopic_cnt,
exp_not_mdtopics,
exp_not_mdtopic_cnt,
metadata_tmout);
rd_kafka_event_destroy(rkev);
for (i = 0 ; i < MY_NEW_TOPICS_CNT ; i++) {
rd_kafka_NewTopic_destroy(new_topics[i]);
rd_free(topics[i]);
}
if (options)
rd_kafka_AdminOptions_destroy(options);
if (!useq)
rd_kafka_queue_destroy(q);
#undef MY_NEW_TOPICS_CNT
}
/**
* @brief Test deletion of topics
*
*
*/
static void do_test_DeleteTopics (const char *what,
rd_kafka_t *rk, rd_kafka_queue_t *useq,
int op_timeout) {
rd_kafka_queue_t *q = useq ? useq : rd_kafka_queue_new(rk);
const int skip_topic_cnt = 2;
#define MY_DEL_TOPICS_CNT 9
char *topics[MY_DEL_TOPICS_CNT];
rd_kafka_DeleteTopic_t *del_topics[MY_DEL_TOPICS_CNT];
rd_kafka_AdminOptions_t *options = NULL;
rd_kafka_resp_err_t exp_topicerr[MY_DEL_TOPICS_CNT] = {0};
rd_kafka_resp_err_t exp_err = RD_KAFKA_RESP_ERR_NO_ERROR;
/* Expected topics in metadata */
rd_kafka_metadata_topic_t exp_mdtopics[MY_DEL_TOPICS_CNT] = {{0}};
int exp_mdtopic_cnt = 0;
/* Not expected topics in metadata */
rd_kafka_metadata_topic_t exp_not_mdtopics[MY_DEL_TOPICS_CNT] = {{0}};
int exp_not_mdtopic_cnt = 0;
int i;
char errstr[512];
const char *errstr2;
rd_kafka_resp_err_t err;
test_timing_t timing;
rd_kafka_event_t *rkev;
const rd_kafka_DeleteTopics_result_t *res;
const rd_kafka_topic_result_t **restopics;
size_t restopic_cnt;
int metadata_tmout;
TEST_SAY(_C_MAG "[ %s DeleteTopics with %s, op_timeout %d ]\n",
rd_kafka_name(rk), what, op_timeout);
/**
* Construct DeleteTopic array
*/
for (i = 0 ; i < MY_DEL_TOPICS_CNT ; i++) {
char *topic = rd_strdup(test_mk_topic_name(__FUNCTION__, 1));
int notexist_topic = i >= MY_DEL_TOPICS_CNT - skip_topic_cnt;
topics[i] = topic;
del_topics[i] = rd_kafka_DeleteTopic_new(topic);
if (notexist_topic)
exp_topicerr[i] =
RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART;
else {
exp_topicerr[i] =
RD_KAFKA_RESP_ERR_NO_ERROR;
exp_mdtopics[exp_mdtopic_cnt++].topic = topic;
}
exp_not_mdtopics[exp_not_mdtopic_cnt++].topic = topic;
}
if (op_timeout != -1) {
options = rd_kafka_AdminOptions_new(
rk, RD_KAFKA_ADMIN_OP_ANY);
err = rd_kafka_AdminOptions_set_operation_timeout(
options, op_timeout, errstr, sizeof(errstr));
TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
}
/* Create the topics first, minus the skip count. */
test_CreateTopics_simple(rk, NULL, topics,
MY_DEL_TOPICS_CNT-skip_topic_cnt,
2/*num_partitions*/,
NULL);
/* Verify that topics are reported by metadata */
test_wait_metadata_update(rk,
exp_mdtopics, exp_mdtopic_cnt,
NULL, 0,
15*1000);
TIMING_START(&timing, "DeleteTopics");
TEST_SAY("Call DeleteTopics\n");
rd_kafka_DeleteTopics(rk, del_topics, MY_DEL_TOPICS_CNT,
options, q);
TIMING_ASSERT_LATER(&timing, 0, 50);
/* Poll result queue for DeleteTopics result.
* Print but otherwise ignore other event types
* (typically generic Error events). */
TIMING_START(&timing, "DeleteTopics.queue_poll");
while (1) {
rkev = rd_kafka_queue_poll(q, tmout_multip(20*1000));
TEST_SAY("DeleteTopics: got %s in %.3fms\n",
rd_kafka_event_name(rkev),
TIMING_DURATION(&timing) / 1000.0f);
if (rd_kafka_event_error(rkev))
TEST_SAY("%s: %s\n",
rd_kafka_event_name(rkev),
rd_kafka_event_error_string(rkev));
if (rd_kafka_event_type(rkev) ==
RD_KAFKA_EVENT_DELETETOPICS_RESULT)
break;
rd_kafka_event_destroy(rkev);
}
/* Convert event to proper result */
res = rd_kafka_event_DeleteTopics_result(rkev);
TEST_ASSERT(res, "expected DeleteTopics_result, not %s",
rd_kafka_event_name(rkev));
/* Expecting error */
err = rd_kafka_event_error(rkev);
errstr2 = rd_kafka_event_error_string(rkev);
TEST_ASSERT(err == exp_err,
"expected DeleteTopics to return %s, not %s (%s)",
rd_kafka_err2str(exp_err),
rd_kafka_err2str(err),
err ? errstr2 : "n/a");
TEST_SAY("DeleteTopics: returned %s (%s)\n",
rd_kafka_err2str(err), err ? errstr2 : "n/a");
/* Extract topics */
restopics = rd_kafka_DeleteTopics_result_topics(res, &restopic_cnt);
/* Scan topics for proper fields and expected failures. */
for (i = 0 ; i < (int)restopic_cnt ; i++) {
const rd_kafka_topic_result_t *terr = restopics[i];
/* Verify that topic order matches our request. */
if (strcmp(rd_kafka_topic_result_name(terr), topics[i]))
TEST_FAIL_LATER("Topic result order mismatch at #%d: "
"expected %s, got %s",
i, topics[i],
rd_kafka_topic_result_name(terr));
TEST_SAY("DeleteTopics result: #%d: %s: %s: %s\n",
i,
rd_kafka_topic_result_name(terr),
rd_kafka_err2name(rd_kafka_topic_result_error(terr)),
rd_kafka_topic_result_error_string(terr));
if (rd_kafka_topic_result_error(terr) != exp_topicerr[i])
TEST_FAIL_LATER(
"Expected %s, not %d: %s",
rd_kafka_err2name(exp_topicerr[i]),
rd_kafka_topic_result_error(terr),
rd_kafka_err2name(rd_kafka_topic_result_error(
terr)));
}
/**
* Verify that the expected topics are deleted and the non-expected
* are not. Allow it some time to propagate.
*/
if (op_timeout > 0)
metadata_tmout = op_timeout + 1000;
else
metadata_tmout = 10 * 1000;
test_wait_metadata_update(rk,
NULL, 0,
exp_not_mdtopics,
exp_not_mdtopic_cnt,
metadata_tmout);
rd_kafka_event_destroy(rkev);
for (i = 0 ; i < MY_DEL_TOPICS_CNT ; i++) {
rd_kafka_DeleteTopic_destroy(del_topics[i]);
rd_free(topics[i]);
}
if (options)
rd_kafka_AdminOptions_destroy(options);
if (!useq)
rd_kafka_queue_destroy(q);
#undef MY_DEL_TOPICS_CNT
}
/**
* Produces 'msgcnt' messages split over 'partition_cnt' partitions.
*/
static void produce_messages (uint64_t testid, const char *topic,
int partition_cnt, int msgcnt) {
int r;
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char errstr[512];
char msg[128];
int failcnt = 0;
int i;
rd_kafka_message_t *rkmessages;
int32_t partition;
int msgid = 0;
test_conf_init(&conf, &topic_conf, 20);
rd_kafka_conf_set_dr_cb(conf, dr_cb);
/* Make sure all replicas are in-sync after producing
* so that consume test wont fail. */
rd_kafka_topic_conf_set(topic_conf, "request.required.acks", "-1",
errstr, sizeof(errstr));
/* Create kafka instance */
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
rkt = rd_kafka_topic_new(rk, topic, topic_conf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
rd_strerror(errno));
/* Create messages. */
prod_msg_remains = msgcnt;
rkmessages = calloc(sizeof(*rkmessages), msgcnt / partition_cnt);
for (partition = 0 ; partition < partition_cnt ; partition++) {
int batch_cnt = msgcnt / partition_cnt;
for (i = 0 ; i < batch_cnt ; i++) {
rd_snprintf(msg, sizeof(msg),
"testid=%"PRIu64", partition=%i, msg=%i",
testid, (int)partition, msgid);
rkmessages[i].payload = rd_strdup(msg);
rkmessages[i].len = strlen(msg);
msgid++;
}
TEST_SAY("Start produce to partition %i: msgs #%d..%d\n",
(int)partition, msgid-batch_cnt, msgid);
/* Produce batch for this partition */
r = rd_kafka_produce_batch(rkt, partition, RD_KAFKA_MSG_F_FREE,
rkmessages, batch_cnt);
if (r == -1)
TEST_FAIL("Failed to produce "
"batch for partition %i: %s",
(int)partition,
rd_kafka_err2str(rd_kafka_errno2err(errno)));
/* Scan through messages to check for errors. */
for (i = 0 ; i < batch_cnt ; i++) {
if (rkmessages[i].err) {
failcnt++;
if (failcnt < 100)
TEST_SAY("Message #%i failed: %s\n",
i,
rd_kafka_err2str(rkmessages[i].
err));
}
}
/* All messages should've been produced. */
if (r < batch_cnt) {
TEST_SAY("Not all messages were accepted "
"by produce_batch(): %i < %i\n", r, batch_cnt);
if (batch_cnt - r != failcnt)
TEST_SAY("Discrepency between failed "
"messages (%i) "
"and return value %i (%i - %i)\n",
failcnt, batch_cnt - r, batch_cnt, r);
TEST_FAIL("%i/%i messages failed\n",
batch_cnt - r, batch_cnt);
}
TEST_SAY("Produced %i messages to partition %i, "
"waiting for deliveries\n", r, partition);
}
free(rkmessages);
/* Wait for messages to be delivered */
while (rd_kafka_outq_len(rk) > 0)
rd_kafka_poll(rk, 100);
if (fails)
TEST_FAIL("%i failures, see previous errors", fails);
if (prod_msg_remains != 0)
TEST_FAIL("Still waiting for %i messages to be produced",
prod_msg_remains);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy rdkafka instance */
TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
rd_kafka_destroy(rk);
}
rd_kafka_resp_err_t
rd_kafka_metadata (rd_kafka_t *rk, int all_topics,
rd_kafka_topic_t *only_rkt,
const struct rd_kafka_metadata **metadatap,
int timeout_ms) {
rd_kafka_q_t *rkq;
rd_kafka_broker_t *rkb;
rd_kafka_op_t *rko;
rd_ts_t ts_end = rd_timeout_init(timeout_ms);
rd_list_t topics;
/* Query any broker that is up, and if none are up pick the first one,
* if we're lucky it will be up before the timeout */
rkb = rd_kafka_broker_any_usable(rk, timeout_ms, 1,
"application metadata request");
if (!rkb)
return RD_KAFKA_RESP_ERR__TRANSPORT;
rkq = rd_kafka_q_new(rk);
rd_list_init(&topics, 0, rd_free);
if (!all_topics) {
if (only_rkt)
rd_list_add(&topics,
rd_strdup(rd_kafka_topic_a2i(only_rkt)->
rkt_topic->str));
else
rd_kafka_local_topics_to_list(rkb->rkb_rk, &topics);
}
/* Async: request metadata */
rko = rd_kafka_op_new(RD_KAFKA_OP_METADATA);
rd_kafka_op_set_replyq(rko, rkq, 0);
rko->rko_u.metadata.force = 1; /* Force metadata request regardless
* of outstanding metadata requests. */
rd_kafka_MetadataRequest(rkb, &topics, "application requested", rko);
rd_list_destroy(&topics);
rd_kafka_broker_destroy(rkb);
/* Wait for reply (or timeout) */
rko = rd_kafka_q_pop(rkq, rd_timeout_remains(ts_end), 0);
rd_kafka_q_destroy_owner(rkq);
/* Timeout */
if (!rko)
return RD_KAFKA_RESP_ERR__TIMED_OUT;
/* Error */
if (rko->rko_err) {
rd_kafka_resp_err_t err = rko->rko_err;
rd_kafka_op_destroy(rko);
return err;
}
/* Reply: pass metadata pointer to application who now owns it*/
rd_kafka_assert(rk, rko->rko_u.metadata.md);
*metadatap = rko->rko_u.metadata.md;
rko->rko_u.metadata.md = NULL;
rd_kafka_op_destroy(rko);
return RD_KAFKA_RESP_ERR_NO_ERROR;
}
/* @remark This test will fail if auto topic creation is enabled on the broker
* since the client will issue a topic-creating metadata request to find
* a new leader when the topic is removed.
*
* To run with trivup, do:
* ./interactive_broker_version.py .. -conf '{"auto_create_topics":"false"}' ..
* TESTS=0045 ./run-test.sh -k ./merged
*/
static void do_test_topic_remove (void) {
char *topic_f = rd_strdup(test_mk_topic_name("topic_f", 1));
char *topic_g = rd_strdup(test_mk_topic_name("topic_g", 1));
int parts_f = 5;
int parts_g = 9;
rd_kafka_t *rk;
rd_kafka_conf_t *conf;
rd_kafka_queue_t *queue;
rd_kafka_topic_partition_list_t *topics;
rd_kafka_resp_err_t err;
/**
* Topic removal test:
* - Create topic f & g
* - Subscribe to f & g
* - Verify f & g assignment
* - Remove topic f
* - Verify g assignment
* - Remove topic g
* - Verify empty assignment
*/
TEST_SAY("Topic removal testing\n");
test_conf_init(&conf, NULL, 60);
/* Decrease metadata interval to speed up topic change discovery. */
test_conf_set(conf, "metadata.max.age.ms", "5000");
rd_kafka_conf_set_events(conf, RD_KAFKA_EVENT_REBALANCE);
rk = test_create_consumer(test_str_id_generate_tmp(),
NULL, conf, NULL, NULL);
queue = rd_kafka_queue_get_consumer(rk);
TEST_SAY("Topic removal: creating topic %s (subscribed)\n", topic_f);
test_create_topic(topic_f, parts_f, 1);
TEST_SAY("Topic removal: creating topic %s (subscribed)\n", topic_g);
test_create_topic(topic_g, parts_g, 1);
rd_sleep(1); // FIXME: do check&wait loop instead
TEST_SAY("Topic removal: Subscribing to %s & %s\n", topic_f, topic_g);
topics = rd_kafka_topic_partition_list_new(2);
rd_kafka_topic_partition_list_add(topics, topic_f, RD_KAFKA_PARTITION_UA);
rd_kafka_topic_partition_list_add(topics, topic_g, RD_KAFKA_PARTITION_UA);
err = rd_kafka_subscribe(rk, topics);
TEST_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR,
"%s", rd_kafka_err2str(err));
rd_kafka_topic_partition_list_destroy(topics);
await_assignment("Topic removal: both topics exist", rk, queue, 2,
topic_f, parts_f,
topic_g, parts_g);
TEST_SAY("Topic removal: removing %s\n", topic_f);
test_kafka_topics("--delete --topic %s", topic_f);
await_revoke("Topic removal: rebalance after topic removal", rk, queue);
await_assignment("Topic removal: one topic exists", rk, queue, 1,
topic_g, parts_g);
TEST_SAY("Topic removal: removing %s\n", topic_g);
test_kafka_topics("--delete --topic %s", topic_g);
await_revoke("Topic removal: rebalance after 2nd topic removal",
rk, queue);
/* Should not see another rebalance since all topics now removed */
await_no_rebalance("Topic removal: empty", rk, queue, 10000);
test_consumer_close(rk);
rd_kafka_queue_destroy(queue);
rd_kafka_destroy(rk);
rd_free(topic_f);
rd_free(topic_g);
}
/* Produce a batch of messages to a single partition. */
static void test_single_partition (void) {
int partition = 0;
int r;
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char msg[128];
int msgcnt = 100000;
int failcnt = 0;
int i;
rd_kafka_message_t *rkmessages;
int msgcounter = 0;
msgid_next = 0;
test_conf_init(&conf, &topic_conf, 20);
/* Set delivery report callback */
rd_kafka_conf_set_dr_cb(conf, dr_single_partition_cb);
rd_kafka_conf_set_opaque(conf, &msgcounter);
/* Create kafka instance */
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
TEST_SAY("test_single_partition: Created kafka instance %s\n",
rd_kafka_name(rk));
rkt = rd_kafka_topic_new(rk, test_mk_topic_name("0011", 0),
topic_conf);
if (!rkt)
TEST_FAIL("Failed to create topic: %s\n",
rd_strerror(errno));
/* Create messages */
rkmessages = calloc(sizeof(*rkmessages), msgcnt);
for (i = 0 ; i < msgcnt ; i++) {
int *msgidp = malloc(sizeof(*msgidp));
*msgidp = i;
rd_snprintf(msg, sizeof(msg), "%s:%s test message #%i",
__FILE__, __FUNCTION__, i);
rkmessages[i].payload = rd_strdup(msg);
rkmessages[i].len = strlen(msg);
rkmessages[i]._private = msgidp;
}
r = rd_kafka_produce_batch(rkt, partition, RD_KAFKA_MSG_F_FREE,
rkmessages, msgcnt);
/* Scan through messages to check for errors. */
for (i = 0 ; i < msgcnt ; i++) {
if (rkmessages[i].err) {
failcnt++;
if (failcnt < 100)
TEST_SAY("Message #%i failed: %s\n",
i,
rd_kafka_err2str(rkmessages[i].err));
}
}
/* All messages should've been produced. */
if (r < msgcnt) {
TEST_SAY("Not all messages were accepted "
"by produce_batch(): %i < %i\n", r, msgcnt);
if (msgcnt - r != failcnt)
TEST_SAY("Discrepency between failed messages (%i) "
"and return value %i (%i - %i)\n",
failcnt, msgcnt - r, msgcnt, r);
TEST_FAIL("%i/%i messages failed\n", msgcnt - r, msgcnt);
}
free(rkmessages);
TEST_SAY("Single partition: "
"Produced %i messages, waiting for deliveries\n", r);
msgcounter = msgcnt;
/* Wait for messages to be delivered */
test_wait_delivery(rk, &msgcounter);
if (fails)
TEST_FAIL("%i failures, see previous errors", fails);
if (msgid_next != msgcnt)
TEST_FAIL("Still waiting for messages: next %i != end %i\n",
msgid_next, msgcnt);
/* Destroy topic */
rd_kafka_topic_destroy(rkt);
/* Destroy rdkafka instance */
TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
rd_kafka_destroy(rk);
return;
}
