/**
* Take action when the offset for a toppar becomes unusable.
*
* Locality: toppar handler thread
* Locks: toppar_lock() MUST be held
*/
void rd_kafka_offset_reset (rd_kafka_toppar_t *rktp, int64_t err_offset,
rd_kafka_resp_err_t err, const char *reason) {
int64_t offset = RD_KAFKA_OFFSET_INVALID;
rd_kafka_op_t *rko;
/* Enqueue op for toppar handler thread if we're on the wrong thread. */
if (!thrd_is_current(rktp->rktp_rkt->rkt_rk->rk_thread)) {
rd_kafka_op_t *rko = rd_kafka_op_new(RD_KAFKA_OP_CALLBACK);
rko->rko_op_cb = rd_kafka_offset_reset_op_cb;
rko->rko_rktp = rd_kafka_toppar_keep(rktp);
rko->rko_err = err;
rko->rko_offset = err_offset;
rko->rko_flags |= RD_KAFKA_OP_F_FREE;
rko->rko_payload = rd_strdup(reason);
rko->rko_len = strlen(reason);
rd_kafka_q_enq(&rktp->rktp_ops, rko);
return;
}
if (err_offset == RD_KAFKA_OFFSET_INVALID || err)
offset = rktp->rktp_rkt->rkt_conf.auto_offset_reset;
else
offset = err_offset;
if (offset == RD_KAFKA_OFFSET_INVALID) {
/* Error, auto.offset.reset tells us to error out. */
rko = rd_kafka_op_new(RD_KAFKA_OP_CONSUMER_ERR);
rko->rko_err = err;
rko->rko_rkmessage.offset = err_offset;
rko->rko_rkmessage.partition = rktp->rktp_partition;
rko->rko_payload = rd_strdup(reason);
rko->rko_len = strlen(rko->rko_payload);
rko->rko_flags |= RD_KAFKA_OP_F_FREE;
rko->rko_rktp = rd_kafka_toppar_keep(rktp);
rd_kafka_q_enq(&rktp->rktp_fetchq, rko);
rd_kafka_toppar_set_fetch_state(
rktp, RD_KAFKA_TOPPAR_FETCH_NONE);
} else {
/* Query logical offset */
rktp->rktp_query_offset = offset;
rd_kafka_toppar_set_fetch_state(
rktp, RD_KAFKA_TOPPAR_FETCH_OFFSET_QUERY);
}
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
"%s [%"PRId32"]: offset reset (at offset %s) "
"to %s: %s: %s",
rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition,
rd_kafka_offset2str(err_offset),
rd_kafka_offset2str(offset),
reason, rd_kafka_err2str(err));
if (rktp->rktp_fetch_state == RD_KAFKA_TOPPAR_FETCH_OFFSET_QUERY)
rd_kafka_toppar_offset_request(rktp, rktp->rktp_query_offset, 0);
}
/**
* @brief Handle a Metadata response message.
*
* @param topics are the requested topics (may be NULL)
*
* The metadata will be marshalled into 'struct rd_kafka_metadata*' structs.
*
* The marshalled metadata is returned in \p *mdp, (NULL on error).
* @returns an error code on parse failure, else NO_ERRRO.
*
* @locality rdkafka main thread
*/
rd_kafka_resp_err_t
rd_kafka_parse_Metadata (rd_kafka_broker_t *rkb,
rd_kafka_buf_t *request,
rd_kafka_buf_t *rkbuf,
struct rd_kafka_metadata **mdp) {
rd_kafka_t *rk = rkb->rkb_rk;
int i, j, k;
rd_tmpabuf_t tbuf;
struct rd_kafka_metadata *md;
size_t rkb_namelen;
const int log_decode_errors = LOG_ERR;
rd_list_t *missing_topics = NULL;
const rd_list_t *requested_topics = request->rkbuf_u.Metadata.topics;
int all_topics = request->rkbuf_u.Metadata.all_topics;
const char *reason = request->rkbuf_u.Metadata.reason ?
request->rkbuf_u.Metadata.reason : "(no reason)";
int ApiVersion = request->rkbuf_reqhdr.ApiVersion;
rd_kafkap_str_t cluster_id = RD_ZERO_INIT;
int32_t controller_id = -1;
rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
int broadcast_changes = 0;
rd_kafka_assert(NULL, thrd_is_current(rk->rk_thread));
/* Remove topics from missing_topics as they are seen in Metadata. */
if (requested_topics)
missing_topics = rd_list_copy(requested_topics,
rd_list_string_copy, NULL);
rd_kafka_broker_lock(rkb);
rkb_namelen = strlen(rkb->rkb_name)+1;
/* We assume that the marshalled representation is
* no more than 4 times larger than the wire representation. */
rd_tmpabuf_new(&tbuf,
sizeof(*md) + rkb_namelen + (rkbuf->rkbuf_totlen * 4),
0/*dont assert on fail*/);
if (!(md = rd_tmpabuf_alloc(&tbuf, sizeof(*md)))) {
err = RD_KAFKA_RESP_ERR__CRIT_SYS_RESOURCE;
goto err;
}
md->orig_broker_id = rkb->rkb_nodeid;
md->orig_broker_name = rd_tmpabuf_write(&tbuf,
rkb->rkb_name, rkb_namelen);
rd_kafka_broker_unlock(rkb);
/* Read Brokers */
rd_kafka_buf_read_i32a(rkbuf, md->broker_cnt);
if (md->broker_cnt > RD_KAFKAP_BROKERS_MAX)
rd_kafka_buf_parse_fail(rkbuf, "Broker_cnt %i > BROKERS_MAX %i",
md->broker_cnt, RD_KAFKAP_BROKERS_MAX);
if (!(md->brokers = rd_tmpabuf_alloc(&tbuf, md->broker_cnt *
sizeof(*md->brokers))))
rd_kafka_buf_parse_fail(rkbuf,
"%d brokers: tmpabuf memory shortage",
md->broker_cnt);
for (i = 0 ; i < md->broker_cnt ; i++) {
rd_kafka_buf_read_i32a(rkbuf, md->brokers[i].id);
rd_kafka_buf_read_str_tmpabuf(rkbuf, &tbuf, md->brokers[i].host);
rd_kafka_buf_read_i32a(rkbuf, md->brokers[i].port);
if (ApiVersion >= 1) {
rd_kafkap_str_t rack;
rd_kafka_buf_read_str(rkbuf, &rack);
}
}
if (ApiVersion >= 2)
rd_kafka_buf_read_str(rkbuf, &cluster_id);
if (ApiVersion >= 1) {
rd_kafka_buf_read_i32(rkbuf, &controller_id);
rd_rkb_dbg(rkb, METADATA,
"METADATA", "ClusterId: %.*s, ControllerId: %"PRId32,
RD_KAFKAP_STR_PR(&cluster_id), controller_id);
}
/* Read TopicMetadata */
rd_kafka_buf_read_i32a(rkbuf, md->topic_cnt);
rd_rkb_dbg(rkb, METADATA, "METADATA", "%i brokers, %i topics",
md->broker_cnt, md->topic_cnt);
if (md->topic_cnt > RD_KAFKAP_TOPICS_MAX)
rd_kafka_buf_parse_fail(rkbuf, "TopicMetadata_cnt %"PRId32
" > TOPICS_MAX %i",
md->topic_cnt, RD_KAFKAP_TOPICS_MAX);
if (!(md->topics = rd_tmpabuf_alloc(&tbuf,
md->topic_cnt *
sizeof(*md->topics))))
rd_kafka_buf_parse_fail(rkbuf,
"%d topics: tmpabuf memory shortage",
md->topic_cnt);
for (i = 0 ; i < md->topic_cnt ; i++) {
rd_kafka_buf_read_i16a(rkbuf, md->topics[i].err);
rd_kafka_buf_read_str_tmpabuf(rkbuf, &tbuf, md->topics[i].topic);
if (ApiVersion >= 1) {
int8_t is_internal;
rd_kafka_buf_read_i8(rkbuf, &is_internal);
}
/* PartitionMetadata */
rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partition_cnt);
if (md->topics[i].partition_cnt > RD_KAFKAP_PARTITIONS_MAX)
rd_kafka_buf_parse_fail(rkbuf,
"TopicMetadata[%i]."
"PartitionMetadata_cnt %i "
"> PARTITIONS_MAX %i",
i, md->topics[i].partition_cnt,
RD_KAFKAP_PARTITIONS_MAX);
if (!(md->topics[i].partitions =
rd_tmpabuf_alloc(&tbuf,
md->topics[i].partition_cnt *
sizeof(*md->topics[i].partitions))))
rd_kafka_buf_parse_fail(rkbuf,
"%s: %d partitions: "
"tmpabuf memory shortage",
md->topics[i].topic,
md->topics[i].partition_cnt);
for (j = 0 ; j < md->topics[i].partition_cnt ; j++) {
rd_kafka_buf_read_i16a(rkbuf, md->topics[i].partitions[j].err);
rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partitions[j].id);
rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partitions[j].leader);
/* Replicas */
rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partitions[j].replica_cnt);
if (md->topics[i].partitions[j].replica_cnt >
RD_KAFKAP_BROKERS_MAX)
rd_kafka_buf_parse_fail(rkbuf,
"TopicMetadata[%i]."
"PartitionMetadata[%i]."
"Replica_cnt "
"%i > BROKERS_MAX %i",
i, j,
md->topics[i].
partitions[j].
replica_cnt,
RD_KAFKAP_BROKERS_MAX);
if (!(md->topics[i].partitions[j].replicas =
rd_tmpabuf_alloc(&tbuf,
md->topics[i].
partitions[j].replica_cnt *
sizeof(*md->topics[i].
partitions[j].replicas))))
rd_kafka_buf_parse_fail(
rkbuf,
"%s [%"PRId32"]: %d replicas: "
"tmpabuf memory shortage",
md->topics[i].topic,
md->topics[i].partitions[j].id,
md->topics[i].partitions[j].replica_cnt);
for (k = 0 ;
k < md->topics[i].partitions[j].replica_cnt; k++)
rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partitions[j].
replicas[k]);
/* Isrs */
rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partitions[j].isr_cnt);
if (md->topics[i].partitions[j].isr_cnt >
RD_KAFKAP_BROKERS_MAX)
rd_kafka_buf_parse_fail(rkbuf,
"TopicMetadata[%i]."
"PartitionMetadata[%i]."
"Isr_cnt "
"%i > BROKERS_MAX %i",
i, j,
md->topics[i].
partitions[j].isr_cnt,
RD_KAFKAP_BROKERS_MAX);
if (!(md->topics[i].partitions[j].isrs =
rd_tmpabuf_alloc(&tbuf,
md->topics[i].
partitions[j].isr_cnt *
sizeof(*md->topics[i].
partitions[j].isrs))))
rd_kafka_buf_parse_fail(
rkbuf,
"%s [%"PRId32"]: %d isrs: "
"tmpabuf memory shortage",
md->topics[i].topic,
md->topics[i].partitions[j].id,
md->topics[i].partitions[j].isr_cnt);
for (k = 0 ;
k < md->topics[i].partitions[j].isr_cnt; k++)
rd_kafka_buf_read_i32a(rkbuf, md->topics[i].
partitions[j].isrs[k]);
}
/* Sort partitions by partition id */
qsort(md->topics[i].partitions,
md->topics[i].partition_cnt,
sizeof(*md->topics[i].partitions),
rd_kafka_metadata_partition_id_cmp);
}
/* Entire Metadata response now parsed without errors:
* update our internal state according to the response. */
/* Avoid metadata updates when we're terminating. */
if (rd_kafka_terminating(rkb->rkb_rk)) {
err = RD_KAFKA_RESP_ERR__DESTROY;
goto done;
}
if (md->broker_cnt == 0 && md->topic_cnt == 0) {
rd_rkb_dbg(rkb, METADATA, "METADATA",
"No brokers or topics in metadata: should retry");
err = RD_KAFKA_RESP_ERR__PARTIAL;
goto err;
}
/* Update our list of brokers. */
for (i = 0 ; i < md->broker_cnt ; i++) {
rd_rkb_dbg(rkb, METADATA, "METADATA",
" Broker #%i/%i: %s:%i NodeId %"PRId32,
i, md->broker_cnt,
md->brokers[i].host,
md->brokers[i].port,
md->brokers[i].id);
rd_kafka_broker_update(rkb->rkb_rk, rkb->rkb_proto,
&md->brokers[i]);
}
/* Update partition count and leader for each topic we know about */
for (i = 0 ; i < md->topic_cnt ; i++) {
rd_kafka_metadata_topic_t *mdt = &md->topics[i];
rd_rkb_dbg(rkb, METADATA, "METADATA",
" Topic #%i/%i: %s with %i partitions%s%s",
i, md->topic_cnt, mdt->topic,
mdt->partition_cnt,
mdt->err ? ": " : "",
mdt->err ? rd_kafka_err2str(mdt->err) : "");
/* Ignore topics in blacklist */
if (rkb->rkb_rk->rk_conf.topic_blacklist &&
rd_kafka_pattern_match(rkb->rkb_rk->rk_conf.topic_blacklist,
mdt->topic)) {
rd_rkb_dbg(rkb, TOPIC, "BLACKLIST",
"Ignoring blacklisted topic \"%s\" "
"in metadata", mdt->topic);
continue;
}
/* Ignore metadata completely for temporary errors. (issue #513)
* LEADER_NOT_AVAILABLE: Broker is rebalancing
*/
if (mdt->err == RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE &&
mdt->partition_cnt == 0) {
rd_rkb_dbg(rkb, TOPIC, "METADATA",
"Temporary error in metadata reply for "
"topic %s (PartCnt %i): %s: ignoring",
mdt->topic, mdt->partition_cnt,
rd_kafka_err2str(mdt->err));
if (missing_topics)
rd_list_free_cb(
missing_topics,
rd_list_remove_cmp(missing_topics,
mdt->topic,
(void *)strcmp));
continue;
}
/* Update local topic & partition state based on metadata */
rd_kafka_topic_metadata_update2(rkb, mdt);
if (requested_topics) {
rd_list_free_cb(missing_topics,
rd_list_remove_cmp(missing_topics,
mdt->topic,
(void*)strcmp));
if (!all_topics) {
rd_kafka_wrlock(rk);
rd_kafka_metadata_cache_topic_update(rk, mdt);
rd_kafka_wrunlock(rk);
}
}
}
/* Requested topics not seen in metadata? Propogate to topic code. */
if (missing_topics) {
char *topic;
rd_rkb_dbg(rkb, TOPIC, "METADATA",
"%d/%d requested topic(s) seen in metadata",
rd_list_cnt(requested_topics) -
rd_list_cnt(missing_topics),
rd_list_cnt(requested_topics));
for (i = 0 ; i < rd_list_cnt(missing_topics) ; i++)
rd_rkb_dbg(rkb, TOPIC, "METADATA", "wanted %s",
(char *)(missing_topics->rl_elems[i]));
RD_LIST_FOREACH(topic, missing_topics, i) {
shptr_rd_kafka_itopic_t *s_rkt;
s_rkt = rd_kafka_topic_find(rkb->rkb_rk, topic, 1/*lock*/);
if (s_rkt) {
rd_kafka_topic_metadata_none(
rd_kafka_topic_s2i(s_rkt));
rd_kafka_topic_destroy0(s_rkt);
}
}
}