예제 #1
0
static int64_t rd_kafka_offset_file_read (rd_kafka_toppar_t *rktp) {
	char buf[22];
	char *end;
	int64_t offset;
	size_t r;

	if (fseek(rktp->rktp_offset_fp, 0, SEEK_SET) == -1) {
		rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
				RD_KAFKA_RESP_ERR__FS,
				"%s [%"PRId32"]: "
				"Seek (for read) failed on offset file %s: %s",
				rktp->rktp_rkt->rkt_topic->str,
				rktp->rktp_partition,
				rktp->rktp_offset_path,
				rd_strerror(errno));
		rd_kafka_offset_file_close(rktp);
		return RD_KAFKA_OFFSET_INVALID;
	}

	r = fread(buf, 1, sizeof(buf) - 1, rktp->rktp_offset_fp);
	if (r == 0) {
		rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
			     "%s [%"PRId32"]: offset file (%s) is empty",
			     rktp->rktp_rkt->rkt_topic->str,
			     rktp->rktp_partition,
			     rktp->rktp_offset_path);
		return RD_KAFKA_OFFSET_INVALID;
	}

	buf[r] = '\0';

	offset = strtoull(buf, &end, 10);
	if (buf == end) {
		rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
				RD_KAFKA_RESP_ERR__FS,
				"%s [%"PRId32"]: "
				"Unable to parse offset in %s",
				rktp->rktp_rkt->rkt_topic->str,
				rktp->rktp_partition,
				rktp->rktp_offset_path);
		return RD_KAFKA_OFFSET_INVALID;
	}


	rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
		     "%s [%"PRId32"]: Read offset %"PRId64" from offset "
		     "file (%s)",
		     rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition,
		     offset, rktp->rktp_offset_path);

	return offset;
}
예제 #2
0
static int rd_kafka_offset_file_open (rd_kafka_toppar_t *rktp) {
        rd_kafka_t *rk = rktp->rktp_rkt->rkt_rk;
        int fd;

#ifndef _MSC_VER
	mode_t mode = 0644;
#else
	mode_t mode = _S_IREAD|_S_IWRITE;
#endif
	if ((fd = rk->rk_conf.open_cb(rktp->rktp_offset_path,
                                      O_CREAT|O_RDWR, mode,
                                      rk->rk_conf.opaque)) == -1) {
		rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
				RD_KAFKA_RESP_ERR__FS,
				"%s [%"PRId32"]: "
				"Failed to open offset file %s: %s",
				rktp->rktp_rkt->rkt_topic->str,
				rktp->rktp_partition,
				rktp->rktp_offset_path, rd_strerror(errno));
		return -1;
	}

	rktp->rktp_offset_fp =
#ifndef _MSC_VER
		fdopen(fd, "r+");
#else
		_fdopen(fd, "r+");
#endif

	return 0;
}
예제 #3
0
/**
 * NOTE: toppar_lock(rktp) must be held
 */
static int rd_kafka_offset_file_open (rd_kafka_toppar_t *rktp) {
        rd_kafka_t *rk = rktp->rktp_rkt->rkt_rk;
	int fd;

	if ((fd = rk->rk_conf.open_cb(rktp->rktp_offset_path,
                                      O_CREAT|O_RDWR, 0644,
                                      rk->rk_conf.opaque)) == -1) {
		rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
				RD_KAFKA_RESP_ERR__FS,
				"%s [%"PRId32"]: "
				"Failed to open offset file %s: %s",
				rktp->rktp_rkt->rkt_topic->str,
				rktp->rktp_partition,
				rktp->rktp_offset_path, strerror(errno));
		return -1;
	}

	rktp->rktp_offset_fd = fd;

	return 0;
}
예제 #4
0
/**
 * Write offset to offset file.
 *
 * Locality: toppar's broker thread
 */
static rd_kafka_resp_err_t
rd_kafka_offset_file_commit (rd_kafka_toppar_t *rktp) {
	rd_kafka_itopic_t *rkt = rktp->rktp_rkt;
	int attempt;
        rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
        int64_t offset = rktp->rktp_stored_offset;

	for (attempt = 0 ; attempt < 2 ; attempt++) {
		char buf[22];
		int len;

		if (!rktp->rktp_offset_fp)
			if (rd_kafka_offset_file_open(rktp) == -1)
				continue;

		if (fseek(rktp->rktp_offset_fp, 0, SEEK_SET) == -1) {
			rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
					RD_KAFKA_RESP_ERR__FS,
					"%s [%"PRId32"]: "
					"Seek failed on offset file %s: %s",
					rktp->rktp_rkt->rkt_topic->str,
					rktp->rktp_partition,
					rktp->rktp_offset_path,
					rd_strerror(errno));
                        err = RD_KAFKA_RESP_ERR__FS;
			rd_kafka_offset_file_close(rktp);
			continue;
		}

		len = rd_snprintf(buf, sizeof(buf), "%"PRId64"\n", offset);

		if (fwrite(buf, 1, len, rktp->rktp_offset_fp) < 1) {
			rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
					RD_KAFKA_RESP_ERR__FS,
					"%s [%"PRId32"]: "
					"Failed to write offset %"PRId64" to "
					"offset file %s: %s",
					rktp->rktp_rkt->rkt_topic->str,
					rktp->rktp_partition,
					offset,
					rktp->rktp_offset_path,
					rd_strerror(errno));
                        err = RD_KAFKA_RESP_ERR__FS;
			rd_kafka_offset_file_close(rktp);
			continue;
		}

                /* Need to flush before truncate to preserve write ordering */
                (void)fflush(rktp->rktp_offset_fp);

		/* Truncate file */
#ifdef _MSC_VER
		if (_chsize_s(_fileno(rktp->rktp_offset_fp), len) == -1)
			; /* Ignore truncate failures */
#else
		if (ftruncate(fileno(rktp->rktp_offset_fp), len) == -1)
			; /* Ignore truncate failures */
#endif
		rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
			     "%s [%"PRId32"]: wrote offset %"PRId64" to "
			     "file %s",
			     rktp->rktp_rkt->rkt_topic->str,
			     rktp->rktp_partition, offset,
			     rktp->rktp_offset_path);

		rktp->rktp_committed_offset = offset;

		/* If sync interval is set to immediate we sync right away. */
		if (rkt->rkt_conf.offset_store_sync_interval_ms == 0)
			rd_kafka_offset_file_sync(rktp);


		return RD_KAFKA_RESP_ERR_NO_ERROR;
	}


	return err;
}
예제 #5
0
/**
 * NOTE: rktp lock is not required.
 * Locality: rdkafka main thread
 */
static int rd_kafka_offset_file_commit (rd_kafka_toppar_t *rktp,
					int64_t offset) {
	rd_kafka_topic_t *rkt = rktp->rktp_rkt;
	int attempt;

	for (attempt = 0 ; attempt < 2 ; attempt++) {
		char buf[22];
		int len;

		if (rktp->rktp_offset_fd == -1)
			if (rd_kafka_offset_file_open(rktp) == -1)
				continue;

		if (lseek(rktp->rktp_offset_fd, 0, SEEK_SET) == -1) {
			rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
					RD_KAFKA_RESP_ERR__FS,
					"%s [%"PRId32"]: "
					"Seek failed on offset file %s: %s",
					rktp->rktp_rkt->rkt_topic->str,
					rktp->rktp_partition,
					rktp->rktp_offset_path,
					strerror(errno));
			rd_kafka_offset_file_close(rktp);
			continue;
		}

		len = snprintf(buf, sizeof(buf), "%"PRId64"\n", offset);

		if (write(rktp->rktp_offset_fd, buf, len) == -1) {
			rd_kafka_op_err(rktp->rktp_rkt->rkt_rk,
					RD_KAFKA_RESP_ERR__FS,
					"%s [%"PRId32"]: "
					"Failed to write offset %"PRId64" to "
					"offset file %s (fd %i): %s",
					rktp->rktp_rkt->rkt_topic->str,
					rktp->rktp_partition,
					offset,
					rktp->rktp_offset_path,
					rktp->rktp_offset_fd,
					strerror(errno));
			rd_kafka_offset_file_close(rktp);
			continue;
		}

		if (ftruncate(rktp->rktp_offset_fd, len) == -1)
			; /* Ignore truncate failures */

		rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
			     "%s [%"PRId32"]: wrote offset %"PRId64" to "
			     "file %s",
			     rktp->rktp_rkt->rkt_topic->str,
			     rktp->rktp_partition, offset,
			     rktp->rktp_offset_path);

		rktp->rktp_commited_offset = offset;

		/* If sync interval is set to immediate we sync right away. */
		if (rkt->rkt_conf.offset_store_sync_interval_ms == 0)
			fsync(rktp->rktp_offset_fd);

		return 0;
	}


	return -1;
}