static int write_thread_list_state(struct thread_io_list *s,
const char *prefix)
{
struct verify_state_hdr hdr;
uint64_t crc;
ssize_t ret;
int fd;
fd = open_state_file((const char *) s->name, prefix, s->index, 1);
if (fd == -1)
return 1;
crc = fio_crc32c((void *)s, thread_io_list_sz(s));
hdr.version = cpu_to_le64((uint64_t) VSTATE_HDR_VERSION);
hdr.size = cpu_to_le64((uint64_t) thread_io_list_sz(s));
hdr.crc = cpu_to_le64(crc);
ret = write(fd, &hdr, sizeof(hdr));
if (ret != sizeof(hdr))
goto write_fail;
ret = write(fd, s, thread_io_list_sz(s));
if (ret != thread_io_list_sz(s)) {
write_fail:
if (ret < 0)
perror("fio: write state file");
log_err("fio: failed to write state file\n");
ret = 1;
} else
ret = 0;
close(fd);
return ret;
}
static void show_verify_state(void *buf, size_t size)
{
struct verify_state_hdr *hdr = buf;
struct thread_io_list *s;
uint32_t crc;
hdr->version = le64_to_cpu(hdr->version);
hdr->size = le64_to_cpu(hdr->size);
hdr->crc = le64_to_cpu(hdr->crc);
printf("Version:\t0x%x\n", (unsigned int) hdr->version);
printf("Size:\t\t%u\n", (unsigned int) hdr->size);
printf("CRC:\t\t0x%x\n", (unsigned int) hdr->crc);
size -= sizeof(*hdr);
if (hdr->size != size) {
log_err("Size mismatch\n");
return;
}
s = buf + sizeof(*hdr);
crc = fio_crc32c((unsigned char *) s, hdr->size);
if (crc != hdr->crc) {
log_err("crc mismatch %x != %x\n", crc, (unsigned int) hdr->crc);
return;
}
if (hdr->version == 0x03)
show(s, size);
else
log_err("Unsupported version %d\n", (int) hdr->version);
}
int verify_load_state(struct thread_data *td, const char *prefix)
{
struct verify_state_hdr hdr;
void *s = NULL;
uint64_t crc;
ssize_t ret;
int fd;
if (!td->o.verify_state)
return 0;
fd = open_state_file(td->o.name, prefix, td->thread_number - 1, 0);
if (fd == -1)
return 1;
ret = read(fd, &hdr, sizeof(hdr));
if (ret != sizeof(hdr)) {
if (ret < 0)
td_verror(td, errno, "read verify state hdr");
log_err("fio: failed reading verify state header\n");
goto err;
}
hdr.version = le64_to_cpu(hdr.version);
hdr.size = le64_to_cpu(hdr.size);
hdr.crc = le64_to_cpu(hdr.crc);
if (hdr.version != VSTATE_HDR_VERSION &&
hdr.version != VSTATE_HDR_VERSION_V1) {
log_err("fio: bad version in verify state header\n");
goto err;
}
s = malloc(hdr.size);
ret = read(fd, s, hdr.size);
if (ret != hdr.size) {
if (ret < 0)
td_verror(td, errno, "read verify state");
log_err("fio: failed reading verity state\n");
goto err;
}
crc = fio_crc32c(s, hdr.size);
if (crc != hdr.crc) {
log_err("fio: verify state is corrupt\n");
goto err;
}
close(fd);
verify_convert_assign_state(td, s, hdr.version);
return 0;
err:
if (s)
free(s);
close(fd);
return 1;
}
int verify_state_hdr(struct verify_state_hdr *hdr, struct thread_io_list *s)
{
uint64_t crc;
hdr->version = le64_to_cpu(hdr->version);
hdr->size = le64_to_cpu(hdr->size);
hdr->crc = le64_to_cpu(hdr->crc);
if (hdr->version != VSTATE_HDR_VERSION)
return 1;
crc = fio_crc32c((void *)s, hdr->size);
if (crc != hdr->crc)
return 1;
return 0;
}
static uint64_t t_crc32c(void)
{
struct timeval s;
uint64_t ret;
void *buf;
int i;
buf = malloc(CHUNK);
randomize_buf(buf, CHUNK, 0x8989);
fio_gettime(&s, NULL);
for (i = 0; i < NR_CHUNKS; i++)
fio_crc32c(buf, CHUNK);
ret = utime_since_now(&s);
free(buf);
return ret;
}
static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc)
{
void *p = io_u_verify_off(hdr, vc);
struct vhdr_crc32 *vh = hdr_priv(hdr);
uint32_t c;
dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len);
c = fio_crc32c(p, hdr->len - hdr_size(vc->td, hdr));
if (c == vh->crc32)
return 0;
vc->name = "crc32c";
vc->good_crc = &vh->crc32;
vc->bad_crc = &c;
vc->crc_len = 4;
log_verify_failure(hdr, vc);
return EILSEQ;
}
static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha512 *vh = hdr_priv(hdr);
struct fio_sha512_ctx sha512_ctx = {
.buf = vh->sha512,
};
fio_sha512_init(&sha512_ctx);
fio_sha512_update(&sha512_ctx, p, len);
}
static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha256 *vh = hdr_priv(hdr);
struct fio_sha256_ctx sha256_ctx = {
.buf = vh->sha256,
};
fio_sha256_init(&sha256_ctx);
fio_sha256_update(&sha256_ctx, p, len);
fio_sha256_final(&sha256_ctx);
}
static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha1 *vh = hdr_priv(hdr);
struct fio_sha1_ctx sha1_ctx = {
.H = vh->sha1,
};
fio_sha1_init(&sha1_ctx);
fio_sha1_update(&sha1_ctx, p, len);
fio_sha1_final(&sha1_ctx);
}
static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc7 *vh = hdr_priv(hdr);
vh->crc7 = fio_crc7(p, len);
}
static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc16 *vh = hdr_priv(hdr);
vh->crc16 = fio_crc16(p, len);
}
static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc32 *vh = hdr_priv(hdr);
vh->crc32 = fio_crc32(p, len);
}
static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc32 *vh = hdr_priv(hdr);
vh->crc32 = fio_crc32c(p, len);
}
static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc64 *vh = hdr_priv(hdr);
vh->crc64 = fio_crc64(p, len);
}
static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_md5 *vh = hdr_priv(hdr);
struct fio_md5_ctx md5_ctx = {
.hash = (uint32_t *) vh->md5_digest,
};
fio_md5_init(&md5_ctx);
fio_md5_update(&md5_ctx, p, len);
fio_md5_final(&md5_ctx);
}
static void __fill_hdr(struct verify_header *hdr, int verify_type,
uint32_t len, uint64_t rand_seed)
{
void *p = hdr;
hdr->magic = FIO_HDR_MAGIC;
hdr->verify_type = verify_type;
hdr->len = len;
hdr->rand_seed = rand_seed;
hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32));
}
static void fill_hdr(struct verify_header *hdr, int verify_type, uint32_t len,
uint64_t rand_seed)
{
if (verify_type != VERIFY_PATTERN_NO_HDR)
__fill_hdr(hdr, verify_type, len, rand_seed);
}
static void populate_hdr(struct thread_data *td, struct io_u *io_u,
struct verify_header *hdr, unsigned int header_num,
unsigned int header_len)
{
unsigned int data_len;
void *data, *p;
p = (void *) hdr;
fill_hdr(hdr, td->o.verify, header_len, io_u->rand_seed);
data_len = header_len - hdr_size(td, hdr);
data = p + hdr_size(td, hdr);
switch (td->o.verify) {
case VERIFY_MD5:
dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n",
io_u, hdr->len);
fill_md5(hdr, data, data_len);
break;
case VERIFY_CRC64:
dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc64(hdr, data, data_len);
break;
case VERIFY_CRC32C:
case VERIFY_CRC32C_INTEL:
dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
io_u, hdr->len);
fill_crc32c(hdr, data, data_len);
break;
case VERIFY_CRC32:
dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc32(hdr, data, data_len);
break;
case VERIFY_CRC16:
dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc16(hdr, data, data_len);
break;
case VERIFY_CRC7:
dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc7(hdr, data, data_len);
break;
case VERIFY_SHA256:
dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha256(hdr, data, data_len);
break;
case VERIFY_SHA512:
dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha512(hdr, data, data_len);
break;
case VERIFY_XXHASH:
dprint(FD_VERIFY, "fill xxhash io_u %p, len %u\n",
io_u, hdr->len);
fill_xxhash(hdr, data, data_len);
break;
case VERIFY_META:
dprint(FD_VERIFY, "fill meta io_u %p, len %u\n",
io_u, hdr->len);
fill_meta(hdr, td, io_u, header_num);
break;
case VERIFY_SHA1:
dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha1(hdr, data, data_len);
break;
case VERIFY_PATTERN:
case VERIFY_PATTERN_NO_HDR:
/* nothing to do here */
break;
default:
log_err("fio: bad verify type: %d\n", td->o.verify);
assert(0);
}
if (td->o.verify_offset && hdr_size(td, hdr))
memswp(p, p + td->o.verify_offset, hdr_size(td, hdr));
}
/*
* fill body of io_u->buf with random data and add a header with the
* checksum of choice
*/
void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
{
if (td->o.verify == VERIFY_NULL)
return;
io_u->numberio = td->io_issues[io_u->ddir];
fill_pattern_headers(td, io_u, 0, 0);
}
int get_next_verify(struct thread_data *td, struct io_u *io_u)
{
struct io_piece *ipo = NULL;
/*
* this io_u is from a requeue, we already filled the offsets
*/
if (io_u->file)
return 0;
if (!RB_EMPTY_ROOT(&td->io_hist_tree)) {
struct rb_node *n = rb_first(&td->io_hist_tree);
ipo = rb_entry(n, struct io_piece, rb_node);
/*
* Ensure that the associated IO has completed
*/
read_barrier();
if (ipo->flags & IP_F_IN_FLIGHT)
goto nothing;
rb_erase(n, &td->io_hist_tree);
assert(ipo->flags & IP_F_ONRB);
ipo->flags &= ~IP_F_ONRB;
} else if (!flist_empty(&td->io_hist_list)) {
ipo = flist_first_entry(&td->io_hist_list, struct io_piece, list);
/*
* Ensure that the associated IO has completed
*/
read_barrier();
if (ipo->flags & IP_F_IN_FLIGHT)
goto nothing;
flist_del(&ipo->list);
assert(ipo->flags & IP_F_ONLIST);
ipo->flags &= ~IP_F_ONLIST;
}
if (ipo) {
td->io_hist_len--;
io_u->offset = ipo->offset;
io_u->buflen = ipo->len;
io_u->numberio = ipo->numberio;
io_u->file = ipo->file;
io_u_set(io_u, IO_U_F_VER_LIST);
if (ipo->flags & IP_F_TRIMMED)
io_u_set(io_u, IO_U_F_TRIMMED);
if (!fio_file_open(io_u->file)) {
int r = td_io_open_file(td, io_u->file);
if (r) {
dprint(FD_VERIFY, "failed file %s open\n",
io_u->file->file_name);
return 1;
}
}
get_file(ipo->file);
assert(fio_file_open(io_u->file));
io_u->ddir = DDIR_READ;
io_u->xfer_buf = io_u->buf;
io_u->xfer_buflen = io_u->buflen;
remove_trim_entry(td, ipo);
free(ipo);
dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
if (!td->o.verify_pattern_bytes) {
io_u->rand_seed = __rand(&td->verify_state);
if (sizeof(int) != sizeof(long *))
io_u->rand_seed *= __rand(&td->verify_state);
}
return 0;
}
nothing:
dprint(FD_VERIFY, "get_next_verify: empty\n");
return 1;
}
static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha512 *vh = hdr_priv(hdr);
struct fio_sha512_ctx sha512_ctx = {
.buf = vh->sha512,
};
fio_sha512_init(&sha512_ctx);
fio_sha512_update(&sha512_ctx, p, len);
}
static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha256 *vh = hdr_priv(hdr);
struct fio_sha256_ctx sha256_ctx = {
.buf = vh->sha256,
};
fio_sha256_init(&sha256_ctx);
fio_sha256_update(&sha256_ctx, p, len);
}
static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha1 *vh = hdr_priv(hdr);
struct fio_sha1_ctx sha1_ctx = {
.H = vh->sha1,
};
fio_sha1_init(&sha1_ctx);
fio_sha1_update(&sha1_ctx, p, len);
}
static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc7 *vh = hdr_priv(hdr);
vh->crc7 = fio_crc7(p, len);
}
static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc16 *vh = hdr_priv(hdr);
vh->crc16 = fio_crc16(p, len);
}
static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc32 *vh = hdr_priv(hdr);
vh->crc32 = fio_crc32(p, len);
}
static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc32 *vh = hdr_priv(hdr);
vh->crc32 = fio_crc32c(p, len);
}
static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc64 *vh = hdr_priv(hdr);
vh->crc64 = fio_crc64(p, len);
}
static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_md5 *vh = hdr_priv(hdr);
struct fio_md5_ctx md5_ctx = {
.hash = (uint32_t *) vh->md5_digest,
};
fio_md5_init(&md5_ctx);
fio_md5_update(&md5_ctx, p, len);
}
static void populate_hdr(struct thread_data *td, struct io_u *io_u,
struct verify_header *hdr, unsigned int header_num,
unsigned int header_len)
{
unsigned int data_len;
void *data, *p;
p = (void *) hdr;
hdr->magic = FIO_HDR_MAGIC;
hdr->verify_type = td->o.verify;
hdr->len = header_len;
hdr->rand_seed = io_u->rand_seed;
hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32));
data_len = header_len - hdr_size(hdr);
data = p + hdr_size(hdr);
switch (td->o.verify) {
case VERIFY_MD5:
dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n",
io_u, hdr->len);
fill_md5(hdr, data, data_len);
break;
case VERIFY_CRC64:
dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc64(hdr, data, data_len);
break;
case VERIFY_CRC32C:
case VERIFY_CRC32C_INTEL:
dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
io_u, hdr->len);
fill_crc32c(hdr, data, data_len);
break;
case VERIFY_CRC32:
dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc32(hdr, data, data_len);
break;
case VERIFY_CRC16:
dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc16(hdr, data, data_len);
break;
case VERIFY_CRC7:
dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc7(hdr, data, data_len);
break;
case VERIFY_SHA256:
dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha256(hdr, data, data_len);
break;
case VERIFY_SHA512:
dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha512(hdr, data, data_len);
break;
case VERIFY_META:
dprint(FD_VERIFY, "fill meta io_u %p, len %u\n",
io_u, hdr->len);
fill_meta(hdr, td, io_u, header_num);
break;
case VERIFY_SHA1:
dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha1(hdr, data, data_len);
break;
case VERIFY_PATTERN:
/* nothing to do here */
break;
default:
log_err("fio: bad verify type: %d\n", td->o.verify);
assert(0);
}
if (td->o.verify_offset)
memswp(p, p + td->o.verify_offset, hdr_size(hdr));
}
/*
* fill body of io_u->buf with random data and add a header with the
* checksum of choice
*/
void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
{
if (td->o.verify == VERIFY_NULL)
return;
fill_pattern_headers(td, io_u, 0, 0);
}
int get_next_verify(struct thread_data *td, struct io_u *io_u)
{
struct io_piece *ipo = NULL;
/*
* this io_u is from a requeue, we already filled the offsets
*/
if (io_u->file)
return 0;
if (!RB_EMPTY_ROOT(&td->io_hist_tree)) {
struct rb_node *n = rb_first(&td->io_hist_tree);
ipo = rb_entry(n, struct io_piece, rb_node);
rb_erase(n, &td->io_hist_tree);
assert(ipo->flags & IP_F_ONRB);
ipo->flags &= ~IP_F_ONRB;
} else if (!flist_empty(&td->io_hist_list)) {
ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
flist_del(&ipo->list);
assert(ipo->flags & IP_F_ONLIST);
ipo->flags &= ~IP_F_ONLIST;
}
if (ipo) {
td->io_hist_len--;
io_u->offset = ipo->offset;
io_u->buflen = ipo->len;
io_u->file = ipo->file;
io_u->flags |= IO_U_F_VER_LIST;
if (ipo->flags & IP_F_TRIMMED)
io_u->flags |= IO_U_F_TRIMMED;
if (!fio_file_open(io_u->file)) {
int r = td_io_open_file(td, io_u->file);
if (r) {
dprint(FD_VERIFY, "failed file %s open\n",
io_u->file->file_name);
return 1;
}
}
get_file(ipo->file);
assert(fio_file_open(io_u->file));
io_u->ddir = DDIR_READ;
io_u->xfer_buf = io_u->buf;
io_u->xfer_buflen = io_u->buflen;
remove_trim_entry(td, ipo);
free(ipo);
dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
return 0;
}
dprint(FD_VERIFY, "get_next_verify: empty\n");
return 1;
}
void fio_verify_init(struct thread_data *td)
{
if (td->o.verify == VERIFY_CRC32C_INTEL ||
td->o.verify == VERIFY_CRC32C) {
crc32c_intel_probe();
}
}
static void *verify_async_thread(void *data)
{
struct thread_data *td = data;
struct io_u *io_u;
int ret = 0;
if (td->o.verify_cpumask_set &&
fio_setaffinity(td->pid, td->o.verify_cpumask)) {
log_err("fio: failed setting verify thread affinity\n");
goto done;
}
do {
FLIST_HEAD(list);
read_barrier();
if (td->verify_thread_exit)
break;
pthread_mutex_lock(&td->io_u_lock);
while (flist_empty(&td->verify_list) &&
!td->verify_thread_exit) {
ret = pthread_cond_wait(&td->verify_cond,
&td->io_u_lock);
if (ret) {
pthread_mutex_unlock(&td->io_u_lock);
break;
}
}
flist_splice_init(&td->verify_list, &list);
pthread_mutex_unlock(&td->io_u_lock);
if (flist_empty(&list))
continue;
while (!flist_empty(&list)) {
io_u = flist_entry(list.next, struct io_u, verify_list);
flist_del(&io_u->verify_list);
ret = verify_io_u(td, io_u);
put_io_u(td, io_u);
if (!ret)
continue;
if (td_non_fatal_error(td, ERROR_TYPE_VERIFY_BIT, ret)) {
update_error_count(td, ret);
td_clear_error(td);
ret = 0;
}
}
} while (!ret);
if (ret) {
td_verror(td, ret, "async_verify");
if (td->o.verify_fatal)
td->terminate = 1;
}
done:
pthread_mutex_lock(&td->io_u_lock);
td->nr_verify_threads--;
pthread_mutex_unlock(&td->io_u_lock);
pthread_cond_signal(&td->free_cond);
return NULL;
}
int verify_async_init(struct thread_data *td)
{
int i, ret;
pthread_attr_t attr;
pthread_attr_init(&attr);
pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN);
td->verify_thread_exit = 0;
td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async);
for (i = 0; i < td->o.verify_async; i++) {
ret = pthread_create(&td->verify_threads[i], &attr,
verify_async_thread, td);
if (ret) {
log_err("fio: async verify creation failed: %s\n",
strerror(ret));
break;
}
ret = pthread_detach(td->verify_threads[i]);
if (ret) {
log_err("fio: async verify thread detach failed: %s\n",
strerror(ret));
break;
}
td->nr_verify_threads++;
}
pthread_attr_destroy(&attr);
if (i != td->o.verify_async) {
log_err("fio: only %d verify threads started, exiting\n", i);
td->verify_thread_exit = 1;
write_barrier();
pthread_cond_broadcast(&td->verify_cond);
return 1;
}
return 0;
}