/**
* Prepare a toppar for using an offset file.
*
* NOTE: toppar_lock(rktp) must be held.
*/
static void rd_kafka_offset_file_init (rd_kafka_toppar_t *rktp) {
struct stat st;
char spath[4096];
const char *path = rktp->rktp_rkt->rkt_conf.offset_store_path;
int64_t offset = -1;
if (stat(path, &st) == 0 && S_ISDIR(st.st_mode)) {
snprintf(spath, sizeof(spath),
"%s%s%s-%"PRId32".offset",
path, path[strlen(path)-1] == '/' ? "" : "/",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition);
path = spath;
}
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
"%s [%"PRId32"] using offset file %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
path);
rktp->rktp_offset_path = strdup(path);
rd_kafka_timer_start(rktp->rktp_rkt->rkt_rk,
&rktp->rktp_offset_commit_tmr,
rktp->rktp_rkt->rkt_conf.auto_commit_interval_ms *
1000,
rd_kafka_offset_file_commit_tmr_cb, rktp);
if (rktp->rktp_rkt->rkt_conf.offset_store_sync_interval_ms > 0)
rd_kafka_timer_start(rktp->rktp_rkt->rkt_rk,
&rktp->rktp_offset_sync_tmr,
rktp->rktp_rkt->rkt_conf.
offset_store_sync_interval_ms * 1000,
rd_kafka_offset_file_sync_tmr_cb, rktp);
if (rd_kafka_offset_file_open(rktp) != -1) {
/* Read offset from offset file. */
offset = rd_kafka_offset_file_read(rktp);
}
if (offset != -1) {
/* Start fetching from offset */
rktp->rktp_commited_offset = offset;
rktp->rktp_next_offset = offset;
rktp->rktp_fetch_state = RD_KAFKA_TOPPAR_FETCH_ACTIVE;
} else {
/* Offset was not usable: perform offset reset logic */
rktp->rktp_commited_offset = 0;
rd_kafka_offset_reset(rktp, RD_KAFKA_OFFSET_ERROR,
RD_KAFKA_RESP_ERR__FS,
"non-readable offset file");
}
}
/**
* Broker SASL init.
*
* Locality: broker thread
*/
void rd_kafka_broker_sasl_init (rd_kafka_broker_t *rkb) {
rd_kafka_t *rk = rkb->rkb_rk;
if (!rk->rk_conf.sasl.kinit_cmd ||
!strstr(rk->rk_conf.sasl.mechanisms, "GSSAPI"))
return; /* kinit not configured, no need to start timer */
rd_kafka_timer_start(&rk->rk_timers, &rkb->rkb_sasl_kinit_refresh_tmr,
rk->rk_conf.sasl.relogin_min_time * 1000,
rd_kafka_sasl_kinit_refresh_tmr_cb, rkb);
}
/**
* Prepare a toppar for using broker offset commit (broker 0.8.1 or later).
*
* NOTE: toppar_lock(rktp) must be held.
*/
static void rd_kafka_offset_broker_init (rd_kafka_toppar_t *rktp) {
rd_kafka_timer_start(rktp->rktp_rkt->rkt_rk,
&rktp->rktp_offset_commit_tmr,
rktp->rktp_rkt->rkt_conf.auto_commit_interval_ms *
1000,
rd_kafka_offset_broker_commit_tmr_cb, rktp);
/* Read offset from broker */
rktp->rktp_commited_offset = 0;
rd_kafka_offset_reset(rktp, RD_KAFKA_OFFSET_ERROR,
RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE,
"waiting for broker offset");
}
/**
* Initialize toppar's offset store.
*
* Locality: toppar handler thread
*/
void rd_kafka_offset_store_init (rd_kafka_toppar_t *rktp) {
static const char *store_names[] = { "none", "file", "broker" };
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
"%s [%"PRId32"]: using offset store method: %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
store_names[rktp->rktp_rkt->rkt_conf.offset_store_method]);
/* The committed offset is unknown at this point. */
rktp->rktp_committed_offset = RD_KAFKA_OFFSET_INVALID;
/* Set up the commit interval (for simple consumer). */
if (rd_kafka_is_simple_consumer(rktp->rktp_rkt->rkt_rk) &&
rktp->rktp_rkt->rkt_conf.auto_commit_interval_ms > 0)
rd_kafka_timer_start(&rktp->rktp_rkt->rkt_rk->rk_timers,
&rktp->rktp_offset_commit_tmr,
rktp->rktp_rkt->rkt_conf.
auto_commit_interval_ms * 1000ll,
rd_kafka_offset_auto_commit_tmr_cb,
rktp);
switch (rktp->rktp_rkt->rkt_conf.offset_store_method)
{
case RD_KAFKA_OFFSET_METHOD_FILE:
rd_kafka_offset_file_init(rktp);
break;
case RD_KAFKA_OFFSET_METHOD_BROKER:
rd_kafka_offset_broker_init(rktp);
break;
case RD_KAFKA_OFFSET_METHOD_NONE:
break;
default:
/* NOTREACHED */
return;
}
rktp->rktp_flags |= RD_KAFKA_TOPPAR_F_OFFSET_STORE;
}
/**
* Prepare a toppar for using an offset file.
*
* Locality: rdkafka main thread
* Locks: toppar_lock(rktp) must be held
*/
static void rd_kafka_offset_file_init (rd_kafka_toppar_t *rktp) {
char spath[4096];
const char *path = rktp->rktp_rkt->rkt_conf.offset_store_path;
int64_t offset = RD_KAFKA_OFFSET_INVALID;
if (rd_kafka_path_is_dir(path)) {
char tmpfile[1024];
char escfile[4096];
/* Include group.id in filename if configured. */
if (!RD_KAFKAP_STR_IS_NULL(rktp->rktp_rkt->rkt_rk->
rk_conf.group_id))
rd_snprintf(tmpfile, sizeof(tmpfile),
"%s-%"PRId32"-%.*s.offset",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_rk->
rk_conf.group_id));
else
rd_snprintf(tmpfile, sizeof(tmpfile),
"%s-%"PRId32".offset",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition);
/* Escape filename to make it safe. */
mk_esc_filename(tmpfile, escfile, sizeof(escfile));
rd_snprintf(spath, sizeof(spath), "%s%s%s",
path, path[strlen(path)-1] == '/' ? "" : "/", escfile);
path = spath;
}
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
"%s [%"PRId32"]: using offset file %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
path);
rktp->rktp_offset_path = rd_strdup(path);
/* Set up the offset file sync interval. */
if (rktp->rktp_rkt->rkt_conf.offset_store_sync_interval_ms > 0)
rd_kafka_timer_start(&rktp->rktp_rkt->rkt_rk->rk_timers,
&rktp->rktp_offset_sync_tmr,
rktp->rktp_rkt->rkt_conf.
offset_store_sync_interval_ms * 1000ll,
rd_kafka_offset_sync_tmr_cb, rktp);
if (rd_kafka_offset_file_open(rktp) != -1) {
/* Read offset from offset file. */
offset = rd_kafka_offset_file_read(rktp);
}
if (offset != RD_KAFKA_OFFSET_INVALID) {
/* Start fetching from offset */
rktp->rktp_stored_offset = offset;
rktp->rktp_committed_offset = offset;
rd_kafka_toppar_next_offset_handle(rktp, offset);
} else {
/* Offset was not usable: perform offset reset logic */
rktp->rktp_committed_offset = RD_KAFKA_OFFSET_INVALID;
rd_kafka_offset_reset(rktp, RD_KAFKA_OFFSET_INVALID,
RD_KAFKA_RESP_ERR__FS,
"non-readable offset file");
}
}
/**
* Prepare a toppar for using an offset file.
*
* NOTE: toppar_lock(rktp) must be held.
*/
static void rd_kafka_offset_file_init (rd_kafka_toppar_t *rktp) {
struct stat st;
char spath[4096];
const char *path = rktp->rktp_rkt->rkt_conf.offset_store_path;
int64_t offset = -1;
if (stat(path, &st) == 0 && S_ISDIR(st.st_mode)) {
char tmpfile[1024];
char escfile[4096];
/* Include group.id in filename if configured. */
if (!RD_KAFKAP_STR_IS_NULL(rktp->rktp_rkt->rkt_conf.group_id))
snprintf(tmpfile, sizeof(tmpfile),
"%s-%"PRId32"-%.*s.offset",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
RD_KAFKAP_STR_PR(rktp->rktp_rkt->
rkt_conf.group_id));
else
snprintf(tmpfile, sizeof(tmpfile),
"%s-%"PRId32".offset",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition);
/* Escape filename to make it safe. */
mk_esc_filename(tmpfile, escfile, sizeof(escfile));
snprintf(spath, sizeof(spath), "%s%s%s",
path, path[strlen(path)-1] == '/' ? "" : "/", escfile);
path = spath;
}
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
"%s [%"PRId32"] using offset file %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
path);
rktp->rktp_offset_path = strdup(path);
rd_kafka_timer_start(rktp->rktp_rkt->rkt_rk,
&rktp->rktp_offset_commit_tmr,
rktp->rktp_rkt->rkt_conf.auto_commit_interval_ms *
1000,
rd_kafka_offset_file_commit_tmr_cb, rktp);
if (rktp->rktp_rkt->rkt_conf.offset_store_sync_interval_ms > 0)
rd_kafka_timer_start(rktp->rktp_rkt->rkt_rk,
&rktp->rktp_offset_sync_tmr,
rktp->rktp_rkt->rkt_conf.
offset_store_sync_interval_ms * 1000,
rd_kafka_offset_file_sync_tmr_cb, rktp);
if (rd_kafka_offset_file_open(rktp) != -1) {
/* Read offset from offset file. */
offset = rd_kafka_offset_file_read(rktp);
}
if (offset != -1) {
/* Start fetching from offset */
rktp->rktp_commited_offset = offset;
rktp->rktp_next_offset = offset;
rktp->rktp_fetch_state = RD_KAFKA_TOPPAR_FETCH_ACTIVE;
} else {
/* Offset was not usable: perform offset reset logic */
rktp->rktp_commited_offset = 0;
rd_kafka_offset_reset(rktp, RD_KAFKA_OFFSET_ERROR,
RD_KAFKA_RESP_ERR__FS,
"non-readable offset file");
}
}
