static int64_t rd_kafka_offset_file_read (rd_kafka_toppar_t *rktp) {
char buf[22];
char *end;
int64_t offset;
size_t r;
if (fseek(rktp->rktp_offset_fp, 0, SEEK_SET) == -1) {
rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
RD_KAFKA_RESP_ERR__FS,
"%s [%"PRId32"]: "
"Seek (for read) failed on offset file %s: %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
rktp->rktp_offset_path,
rd_strerror(errno));
rd_kafka_offset_file_close(rktp);
return RD_KAFKA_OFFSET_INVALID;
}
r = fread(buf, 1, sizeof(buf) - 1, rktp->rktp_offset_fp);
if (r == 0) {
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
"%s [%"PRId32"]: offset file (%s) is empty",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
rktp->rktp_offset_path);
return RD_KAFKA_OFFSET_INVALID;
}
buf[r] = '\0';
offset = strtoull(buf, &end, 10);
if (buf == end) {
rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
RD_KAFKA_RESP_ERR__FS,
"%s [%"PRId32"]: "
"Unable to parse offset in %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
rktp->rktp_offset_path);
return RD_KAFKA_OFFSET_INVALID;
}
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
"%s [%"PRId32"]: Read offset %"PRId64" from offset "
"file (%s)",
rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition,
offset, rktp->rktp_offset_path);
return offset;
}
static int rd_kafka_offset_file_open (rd_kafka_toppar_t *rktp) {
rd_kafka_t *rk = rktp->rktp_rkt->rkt_rk;
int fd;
#ifndef _MSC_VER
mode_t mode = 0644;
#else
mode_t mode = _S_IREAD|_S_IWRITE;
#endif
if ((fd = rk->rk_conf.open_cb(rktp->rktp_offset_path,
O_CREAT|O_RDWR, mode,
rk->rk_conf.opaque)) == -1) {
rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
RD_KAFKA_RESP_ERR__FS,
"%s [%"PRId32"]: "
"Failed to open offset file %s: %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
rktp->rktp_offset_path, rd_strerror(errno));
return -1;
}
rktp->rktp_offset_fp =
#ifndef _MSC_VER
fdopen(fd, "r+");
#else
_fdopen(fd, "r+");
#endif
return 0;
}
/**
* NOTE: toppar_lock(rktp) must be held
*/
static int rd_kafka_offset_file_open (rd_kafka_toppar_t *rktp) {
rd_kafka_t *rk = rktp->rktp_rkt->rkt_rk;
int fd;
if ((fd = rk->rk_conf.open_cb(rktp->rktp_offset_path,
O_CREAT|O_RDWR, 0644,
rk->rk_conf.opaque)) == -1) {
rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
RD_KAFKA_RESP_ERR__FS,
"%s [%"PRId32"]: "
"Failed to open offset file %s: %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
rktp->rktp_offset_path, strerror(errno));
return -1;
}
rktp->rktp_offset_fd = fd;
return 0;
}
/**
* Write offset to offset file.
*
* Locality: toppar's broker thread
*/
static rd_kafka_resp_err_t
rd_kafka_offset_file_commit (rd_kafka_toppar_t *rktp) {
rd_kafka_itopic_t *rkt = rktp->rktp_rkt;
int attempt;
rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
int64_t offset = rktp->rktp_stored_offset;
for (attempt = 0 ; attempt < 2 ; attempt++) {
char buf[22];
int len;
if (!rktp->rktp_offset_fp)
if (rd_kafka_offset_file_open(rktp) == -1)
continue;
if (fseek(rktp->rktp_offset_fp, 0, SEEK_SET) == -1) {
rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
RD_KAFKA_RESP_ERR__FS,
"%s [%"PRId32"]: "
"Seek failed on offset file %s: %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
rktp->rktp_offset_path,
rd_strerror(errno));
err = RD_KAFKA_RESP_ERR__FS;
rd_kafka_offset_file_close(rktp);
continue;
}
len = rd_snprintf(buf, sizeof(buf), "%"PRId64"\n", offset);
if (fwrite(buf, 1, len, rktp->rktp_offset_fp) < 1) {
rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
RD_KAFKA_RESP_ERR__FS,
"%s [%"PRId32"]: "
"Failed to write offset %"PRId64" to "
"offset file %s: %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
offset,
rktp->rktp_offset_path,
rd_strerror(errno));
err = RD_KAFKA_RESP_ERR__FS;
rd_kafka_offset_file_close(rktp);
continue;
}
/* Need to flush before truncate to preserve write ordering */
(void)fflush(rktp->rktp_offset_fp);
/* Truncate file */
#ifdef _MSC_VER
if (_chsize_s(_fileno(rktp->rktp_offset_fp), len) == -1)
; /* Ignore truncate failures */
#else
if (ftruncate(fileno(rktp->rktp_offset_fp), len) == -1)
; /* Ignore truncate failures */
#endif
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
"%s [%"PRId32"]: wrote offset %"PRId64" to "
"file %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition, offset,
rktp->rktp_offset_path);
rktp->rktp_committed_offset = offset;
/* If sync interval is set to immediate we sync right away. */
if (rkt->rkt_conf.offset_store_sync_interval_ms == 0)
rd_kafka_offset_file_sync(rktp);
return RD_KAFKA_RESP_ERR_NO_ERROR;
}
return err;
}
/**
* NOTE: rktp lock is not required.
* Locality: rdkafka main thread
*/
static int rd_kafka_offset_file_commit (rd_kafka_toppar_t *rktp,
int64_t offset) {
rd_kafka_topic_t *rkt = rktp->rktp_rkt;
int attempt;
for (attempt = 0 ; attempt < 2 ; attempt++) {
char buf[22];
int len;
if (rktp->rktp_offset_fd == -1)
if (rd_kafka_offset_file_open(rktp) == -1)
continue;
if (lseek(rktp->rktp_offset_fd, 0, SEEK_SET) == -1) {
rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
RD_KAFKA_RESP_ERR__FS,
"%s [%"PRId32"]: "
"Seek failed on offset file %s: %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
rktp->rktp_offset_path,
strerror(errno));
rd_kafka_offset_file_close(rktp);
continue;
}
len = snprintf(buf, sizeof(buf), "%"PRId64"\n", offset);
if (write(rktp->rktp_offset_fd, buf, len) == -1) {
rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
RD_KAFKA_RESP_ERR__FS,
"%s [%"PRId32"]: "
"Failed to write offset %"PRId64" to "
"offset file %s (fd %i): %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
offset,
rktp->rktp_offset_path,
rktp->rktp_offset_fd,
strerror(errno));
rd_kafka_offset_file_close(rktp);
continue;
}
if (ftruncate(rktp->rktp_offset_fd, len) == -1)
; /* Ignore truncate failures */
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
"%s [%"PRId32"]: wrote offset %"PRId64" to "
"file %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition, offset,
rktp->rktp_offset_path);
rktp->rktp_commited_offset = offset;
/* If sync interval is set to immediate we sync right away. */
if (rkt->rkt_conf.offset_store_sync_interval_ms == 0)
fsync(rktp->rktp_offset_fd);
return 0;
}
return -1;
}