/**
* Prepare a toppar for using an offset file.
*
* NOTE: toppar_lock(rktp) must be held.
*/
static void rd_kafka_offset_file_init (rd_kafka_toppar_t *rktp) {
struct stat st;
char spath[4096];
const char *path = rktp->rktp_rkt->rkt_conf.offset_store_path;
int64_t offset = -1;
if (stat(path, &st) == 0 && S_ISDIR(st.st_mode)) {
snprintf(spath, sizeof(spath),
"%s%s%s-%"PRId32".offset",
path, path[strlen(path)-1] == '/' ? "" : "/",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition);
path = spath;
}
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
"%s [%"PRId32"] using offset file %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
path);
rktp->rktp_offset_path = strdup(path);
rd_kafka_timer_start(rktp->rktp_rkt->rkt_rk,
&rktp->rktp_offset_commit_tmr,
rktp->rktp_rkt->rkt_conf.auto_commit_interval_ms *
1000,
rd_kafka_offset_file_commit_tmr_cb, rktp);
if (rktp->rktp_rkt->rkt_conf.offset_store_sync_interval_ms > 0)
rd_kafka_timer_start(rktp->rktp_rkt->rkt_rk,
&rktp->rktp_offset_sync_tmr,
rktp->rktp_rkt->rkt_conf.
offset_store_sync_interval_ms * 1000,
rd_kafka_offset_file_sync_tmr_cb, rktp);
if (rd_kafka_offset_file_open(rktp) != -1) {
/* Read offset from offset file. */
offset = rd_kafka_offset_file_read(rktp);
}
if (offset != -1) {
/* Start fetching from offset */
rktp->rktp_commited_offset = offset;
rktp->rktp_next_offset = offset;
rktp->rktp_fetch_state = RD_KAFKA_TOPPAR_FETCH_ACTIVE;
} else {
/* Offset was not usable: perform offset reset logic */
rktp->rktp_commited_offset = 0;
rd_kafka_offset_reset(rktp, RD_KAFKA_OFFSET_ERROR,
RD_KAFKA_RESP_ERR__FS,
"non-readable offset file");
}
}
/**
* Prepare a toppar for using an offset file.
*
* Locality: rdkafka main thread
* Locks: toppar_lock(rktp) must be held
*/
static void rd_kafka_offset_file_init (rd_kafka_toppar_t *rktp) {
char spath[4096];
const char *path = rktp->rktp_rkt->rkt_conf.offset_store_path;
int64_t offset = RD_KAFKA_OFFSET_INVALID;
if (rd_kafka_path_is_dir(path)) {
char tmpfile[1024];
char escfile[4096];
/* Include group.id in filename if configured. */
if (!RD_KAFKAP_STR_IS_NULL(rktp->rktp_rkt->rkt_rk->
rk_conf.group_id))
rd_snprintf(tmpfile, sizeof(tmpfile),
"%s-%"PRId32"-%.*s.offset",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_rk->
rk_conf.group_id));
else
rd_snprintf(tmpfile, sizeof(tmpfile),
"%s-%"PRId32".offset",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition);
/* Escape filename to make it safe. */
mk_esc_filename(tmpfile, escfile, sizeof(escfile));
rd_snprintf(spath, sizeof(spath), "%s%s%s",
path, path[strlen(path)-1] == '/' ? "" : "/", escfile);
path = spath;
}
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
"%s [%"PRId32"]: using offset file %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
path);
rktp->rktp_offset_path = rd_strdup(path);
/* Set up the offset file sync interval. */
if (rktp->rktp_rkt->rkt_conf.offset_store_sync_interval_ms > 0)
rd_kafka_timer_start(&rktp->rktp_rkt->rkt_rk->rk_timers,
&rktp->rktp_offset_sync_tmr,
rktp->rktp_rkt->rkt_conf.
offset_store_sync_interval_ms * 1000ll,
rd_kafka_offset_sync_tmr_cb, rktp);
if (rd_kafka_offset_file_open(rktp) != -1) {
/* Read offset from offset file. */
offset = rd_kafka_offset_file_read(rktp);
}
if (offset != RD_KAFKA_OFFSET_INVALID) {
/* Start fetching from offset */
rktp->rktp_stored_offset = offset;
rktp->rktp_committed_offset = offset;
rd_kafka_toppar_next_offset_handle(rktp, offset);
} else {
/* Offset was not usable: perform offset reset logic */
rktp->rktp_committed_offset = RD_KAFKA_OFFSET_INVALID;
rd_kafka_offset_reset(rktp, RD_KAFKA_OFFSET_INVALID,
RD_KAFKA_RESP_ERR__FS,
"non-readable offset file");
}
}
/**
* Write offset to offset file.
*
* Locality: toppar's broker thread
*/
static rd_kafka_resp_err_t
rd_kafka_offset_file_commit (rd_kafka_toppar_t *rktp) {
rd_kafka_itopic_t *rkt = rktp->rktp_rkt;
int attempt;
rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
int64_t offset = rktp->rktp_stored_offset;
for (attempt = 0 ; attempt < 2 ; attempt++) {
char buf[22];
int len;
if (!rktp->rktp_offset_fp)
if (rd_kafka_offset_file_open(rktp) == -1)
continue;
if (fseek(rktp->rktp_offset_fp, 0, SEEK_SET) == -1) {
rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
RD_KAFKA_RESP_ERR__FS,
"%s [%"PRId32"]: "
"Seek failed on offset file %s: %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
rktp->rktp_offset_path,
rd_strerror(errno));
err = RD_KAFKA_RESP_ERR__FS;
rd_kafka_offset_file_close(rktp);
continue;
}
len = rd_snprintf(buf, sizeof(buf), "%"PRId64"\n", offset);
if (fwrite(buf, 1, len, rktp->rktp_offset_fp) < 1) {
rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
RD_KAFKA_RESP_ERR__FS,
"%s [%"PRId32"]: "
"Failed to write offset %"PRId64" to "
"offset file %s: %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
offset,
rktp->rktp_offset_path,
rd_strerror(errno));
err = RD_KAFKA_RESP_ERR__FS;
rd_kafka_offset_file_close(rktp);
continue;
}
/* Need to flush before truncate to preserve write ordering */
(void)fflush(rktp->rktp_offset_fp);
/* Truncate file */
#ifdef _MSC_VER
if (_chsize_s(_fileno(rktp->rktp_offset_fp), len) == -1)
; /* Ignore truncate failures */
#else
if (ftruncate(fileno(rktp->rktp_offset_fp), len) == -1)
; /* Ignore truncate failures */
#endif
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
"%s [%"PRId32"]: wrote offset %"PRId64" to "
"file %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition, offset,
rktp->rktp_offset_path);
rktp->rktp_committed_offset = offset;
/* If sync interval is set to immediate we sync right away. */
if (rkt->rkt_conf.offset_store_sync_interval_ms == 0)
rd_kafka_offset_file_sync(rktp);
return RD_KAFKA_RESP_ERR_NO_ERROR;
}
return err;
}
/**
* Prepare a toppar for using an offset file.
*
* NOTE: toppar_lock(rktp) must be held.
*/
static void rd_kafka_offset_file_init (rd_kafka_toppar_t *rktp) {
struct stat st;
char spath[4096];
const char *path = rktp->rktp_rkt->rkt_conf.offset_store_path;
int64_t offset = -1;
if (stat(path, &st) == 0 && S_ISDIR(st.st_mode)) {
char tmpfile[1024];
char escfile[4096];
/* Include group.id in filename if configured. */
if (!RD_KAFKAP_STR_IS_NULL(rktp->rktp_rkt->rkt_conf.group_id))
snprintf(tmpfile, sizeof(tmpfile),
"%s-%"PRId32"-%.*s.offset",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
RD_KAFKAP_STR_PR(rktp->rktp_rkt->
rkt_conf.group_id));
else
snprintf(tmpfile, sizeof(tmpfile),
"%s-%"PRId32".offset",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition);
/* Escape filename to make it safe. */
mk_esc_filename(tmpfile, escfile, sizeof(escfile));
snprintf(spath, sizeof(spath), "%s%s%s",
path, path[strlen(path)-1] == '/' ? "" : "/", escfile);
path = spath;
}
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
"%s [%"PRId32"] using offset file %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
path);
rktp->rktp_offset_path = strdup(path);
rd_kafka_timer_start(rktp->rktp_rkt->rkt_rk,
&rktp->rktp_offset_commit_tmr,
rktp->rktp_rkt->rkt_conf.auto_commit_interval_ms *
1000,
rd_kafka_offset_file_commit_tmr_cb, rktp);
if (rktp->rktp_rkt->rkt_conf.offset_store_sync_interval_ms > 0)
rd_kafka_timer_start(rktp->rktp_rkt->rkt_rk,
&rktp->rktp_offset_sync_tmr,
rktp->rktp_rkt->rkt_conf.
offset_store_sync_interval_ms * 1000,
rd_kafka_offset_file_sync_tmr_cb, rktp);
if (rd_kafka_offset_file_open(rktp) != -1) {
/* Read offset from offset file. */
offset = rd_kafka_offset_file_read(rktp);
}
if (offset != -1) {
/* Start fetching from offset */
rktp->rktp_commited_offset = offset;
rktp->rktp_next_offset = offset;
rktp->rktp_fetch_state = RD_KAFKA_TOPPAR_FETCH_ACTIVE;
} else {
/* Offset was not usable: perform offset reset logic */
rktp->rktp_commited_offset = 0;
rd_kafka_offset_reset(rktp, RD_KAFKA_OFFSET_ERROR,
RD_KAFKA_RESP_ERR__FS,
"non-readable offset file");
}
}
/**
* NOTE: rktp lock is not required.
* Locality: rdkafka main thread
*/
static int rd_kafka_offset_file_commit (rd_kafka_toppar_t *rktp,
int64_t offset) {
rd_kafka_topic_t *rkt = rktp->rktp_rkt;
int attempt;
for (attempt = 0 ; attempt < 2 ; attempt++) {
char buf[22];
int len;
if (rktp->rktp_offset_fd == -1)
if (rd_kafka_offset_file_open(rktp) == -1)
continue;
if (lseek(rktp->rktp_offset_fd, 0, SEEK_SET) == -1) {
rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
RD_KAFKA_RESP_ERR__FS,
"%s [%"PRId32"]: "
"Seek failed on offset file %s: %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
rktp->rktp_offset_path,
strerror(errno));
rd_kafka_offset_file_close(rktp);
continue;
}
len = snprintf(buf, sizeof(buf), "%"PRId64"\n", offset);
if (write(rktp->rktp_offset_fd, buf, len) == -1) {
rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
RD_KAFKA_RESP_ERR__FS,
"%s [%"PRId32"]: "
"Failed to write offset %"PRId64" to "
"offset file %s (fd %i): %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
offset,
rktp->rktp_offset_path,
rktp->rktp_offset_fd,
strerror(errno));
rd_kafka_offset_file_close(rktp);
continue;
}
if (ftruncate(rktp->rktp_offset_fd, len) == -1)
; /* Ignore truncate failures */
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
"%s [%"PRId32"]: wrote offset %"PRId64" to "
"file %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition, offset,
rktp->rktp_offset_path);
rktp->rktp_commited_offset = offset;
/* If sync interval is set to immediate we sync right away. */
if (rkt->rkt_conf.offset_store_sync_interval_ms == 0)
fsync(rktp->rktp_offset_fd);
return 0;
}
return -1;
}
