int get_next_trim(struct thread_data *td, struct io_u *io_u)
{
struct io_piece *ipo;
/*
* this io_u is from a requeue, we already filled the offsets
*/
if (io_u->file)
return 0;
if (flist_empty(&td->trim_list))
return 1;
assert(td->trim_entries);
ipo = flist_entry(td->trim_list.next, struct io_piece, trim_list);
remove_trim_entry(td, ipo);
io_u->offset = ipo->offset;
io_u->buflen = ipo->len;
io_u->file = ipo->file;
/*
* If not verifying that trimmed ranges return zeroed data,
* remove this from the to-read verify lists
*/
if (!td->o.trim_zero) {
if (ipo->flags & IP_F_ONLIST)
flist_del(&ipo->list);
else {
assert(ipo->flags & IP_F_ONRB);
rb_erase(&ipo->rb_node, &td->io_hist_tree);
}
td->io_hist_len--;
free(ipo);
} else
ipo->flags |= IP_F_TRIMMED;
if (!fio_file_open(io_u->file)) {
int r = td_io_open_file(td, io_u->file);
if (r) {
dprint(FD_VERIFY, "failed file %s open\n",
io_u->file->file_name);
return 1;
}
}
get_file(io_u->file);
assert(fio_file_open(io_u->file));
io_u->ddir = DDIR_TRIM;
io_u->xfer_buf = NULL;
io_u->xfer_buflen = io_u->buflen;
dprint(FD_VERIFY, "get_next_trim: ret io_u %p\n", io_u);
return 0;
}
void prune_io_piece_log(struct thread_data *td)
{
struct io_piece *ipo;
struct rb_node *n;
while ((n = rb_first(&td->io_hist_tree)) != NULL) {
ipo = rb_entry(n, struct io_piece, rb_node);
rb_erase(n, &td->io_hist_tree);
remove_trim_entry(td, ipo);
td->io_hist_len--;
free(ipo);
}
while (!flist_empty(&td->io_hist_list)) {
ipo = flist_entry(&td->io_hist_list, struct io_piece, list);
flist_del(&ipo->list);
remove_trim_entry(td, ipo);
td->io_hist_len--;
free(ipo);
}
}
int read_iolog_get(struct thread_data *td, struct io_u *io_u)
{
struct io_piece *ipo;
unsigned long elapsed;
while (!flist_empty(&td->io_log_list)) {
int ret;
ipo = flist_entry(td->io_log_list.next, struct io_piece, list);
flist_del(&ipo->list);
remove_trim_entry(td, ipo);
ret = ipo_special(td, ipo);
if (ret < 0) {
free(ipo);
break;
} else if (ret > 0) {
free(ipo);
continue;
}
io_u->ddir = ipo->ddir;
if (ipo->ddir != DDIR_WAIT) {
io_u->offset = ipo->offset;
io_u->buflen = ipo->len;
io_u->file = td->files[ipo->fileno];
get_file(io_u->file);
dprint(FD_IO, "iolog: get %llu/%lu/%s\n", io_u->offset,
io_u->buflen, io_u->file->file_name);
if (ipo->delay)
iolog_delay(td, ipo->delay);
} else {
elapsed = mtime_since_genesis();
if (ipo->delay > elapsed)
usec_sleep(td, (ipo->delay - elapsed) * 1000);
}
free(ipo);
if (io_u->ddir != DDIR_WAIT)
return 0;
}
td->done = 1;
return 1;
}
/*
* log a successful write, so we can unwind the log for verify
*/
void log_io_piece(struct thread_data *td, struct io_u *io_u)
{
struct rb_node **p, *parent;
struct io_piece *ipo, *__ipo;
ipo = malloc(sizeof(struct io_piece));
init_ipo(ipo);
ipo->file = io_u->file;
ipo->offset = io_u->offset;
ipo->len = io_u->buflen;
ipo->numberio = io_u->numberio;
ipo->flags = IP_F_IN_FLIGHT;
io_u->ipo = ipo;
if (io_u_should_trim(td, io_u)) {
flist_add_tail(&ipo->trim_list, &td->trim_list);
td->trim_entries++;
}
/*
* We don't need to sort the entries, if:
*
* Sequential writes, or
* Random writes that lay out the file as it goes along
*
* For both these cases, just reading back data in the order we
* wrote it out is the fastest.
*
* One exception is if we don't have a random map AND we are doing
* verifies, in that case we need to check for duplicate blocks and
* drop the old one, which we rely on the rb insert/lookup for
* handling.
*/
if (((!td->o.verifysort) || !td_random(td) || !td->o.overwrite) &&
(file_randommap(td, ipo->file) || td->o.verify == VERIFY_NONE)) {
INIT_FLIST_HEAD(&ipo->list);
flist_add_tail(&ipo->list, &td->io_hist_list);
ipo->flags |= IP_F_ONLIST;
td->io_hist_len++;
return;
}
RB_CLEAR_NODE(&ipo->rb_node);
/*
* Sort the entry into the verification list
*/
restart:
p = &td->io_hist_tree.rb_node;
parent = NULL;
while (*p) {
parent = *p;
__ipo = rb_entry(parent, struct io_piece, rb_node);
if (ipo->file < __ipo->file)
p = &(*p)->rb_left;
else if (ipo->file > __ipo->file)
p = &(*p)->rb_right;
else if (ipo->offset < __ipo->offset)
p = &(*p)->rb_left;
else if (ipo->offset > __ipo->offset)
p = &(*p)->rb_right;
else {
dprint(FD_IO, "iolog: overlap %llu/%lu, %llu/%lu",
__ipo->offset, __ipo->len,
ipo->offset, ipo->len);
td->io_hist_len--;
rb_erase(parent, &td->io_hist_tree);
remove_trim_entry(td, __ipo);
free(__ipo);
goto restart;
}
}
rb_link_node(&ipo->rb_node, parent, p);
rb_insert_color(&ipo->rb_node, &td->io_hist_tree);
ipo->flags |= IP_F_ONRB;
td->io_hist_len++;
}
static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha512 *vh = hdr_priv(hdr);
struct fio_sha512_ctx sha512_ctx = {
.buf = vh->sha512,
};
fio_sha512_init(&sha512_ctx);
fio_sha512_update(&sha512_ctx, p, len);
}
static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha256 *vh = hdr_priv(hdr);
struct fio_sha256_ctx sha256_ctx = {
.buf = vh->sha256,
};
fio_sha256_init(&sha256_ctx);
fio_sha256_update(&sha256_ctx, p, len);
fio_sha256_final(&sha256_ctx);
}
static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha1 *vh = hdr_priv(hdr);
struct fio_sha1_ctx sha1_ctx = {
.H = vh->sha1,
};
fio_sha1_init(&sha1_ctx);
fio_sha1_update(&sha1_ctx, p, len);
fio_sha1_final(&sha1_ctx);
}
static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc7 *vh = hdr_priv(hdr);
vh->crc7 = fio_crc7(p, len);
}
static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc16 *vh = hdr_priv(hdr);
vh->crc16 = fio_crc16(p, len);
}
static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc32 *vh = hdr_priv(hdr);
vh->crc32 = fio_crc32(p, len);
}
static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc32 *vh = hdr_priv(hdr);
vh->crc32 = fio_crc32c(p, len);
}
static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc64 *vh = hdr_priv(hdr);
vh->crc64 = fio_crc64(p, len);
}
static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_md5 *vh = hdr_priv(hdr);
struct fio_md5_ctx md5_ctx = {
.hash = (uint32_t *) vh->md5_digest,
};
fio_md5_init(&md5_ctx);
fio_md5_update(&md5_ctx, p, len);
fio_md5_final(&md5_ctx);
}
static void __fill_hdr(struct verify_header *hdr, int verify_type,
uint32_t len, uint64_t rand_seed)
{
void *p = hdr;
hdr->magic = FIO_HDR_MAGIC;
hdr->verify_type = verify_type;
hdr->len = len;
hdr->rand_seed = rand_seed;
hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32));
}
static void fill_hdr(struct verify_header *hdr, int verify_type, uint32_t len,
uint64_t rand_seed)
{
if (verify_type != VERIFY_PATTERN_NO_HDR)
__fill_hdr(hdr, verify_type, len, rand_seed);
}
static void populate_hdr(struct thread_data *td, struct io_u *io_u,
struct verify_header *hdr, unsigned int header_num,
unsigned int header_len)
{
unsigned int data_len;
void *data, *p;
p = (void *) hdr;
fill_hdr(hdr, td->o.verify, header_len, io_u->rand_seed);
data_len = header_len - hdr_size(td, hdr);
data = p + hdr_size(td, hdr);
switch (td->o.verify) {
case VERIFY_MD5:
dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n",
io_u, hdr->len);
fill_md5(hdr, data, data_len);
break;
case VERIFY_CRC64:
dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc64(hdr, data, data_len);
break;
case VERIFY_CRC32C:
case VERIFY_CRC32C_INTEL:
dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
io_u, hdr->len);
fill_crc32c(hdr, data, data_len);
break;
case VERIFY_CRC32:
dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc32(hdr, data, data_len);
break;
case VERIFY_CRC16:
dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc16(hdr, data, data_len);
break;
case VERIFY_CRC7:
dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc7(hdr, data, data_len);
break;
case VERIFY_SHA256:
dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha256(hdr, data, data_len);
break;
case VERIFY_SHA512:
dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha512(hdr, data, data_len);
break;
case VERIFY_XXHASH:
dprint(FD_VERIFY, "fill xxhash io_u %p, len %u\n",
io_u, hdr->len);
fill_xxhash(hdr, data, data_len);
break;
case VERIFY_META:
dprint(FD_VERIFY, "fill meta io_u %p, len %u\n",
io_u, hdr->len);
fill_meta(hdr, td, io_u, header_num);
break;
case VERIFY_SHA1:
dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha1(hdr, data, data_len);
break;
case VERIFY_PATTERN:
case VERIFY_PATTERN_NO_HDR:
/* nothing to do here */
break;
default:
log_err("fio: bad verify type: %d\n", td->o.verify);
assert(0);
}
if (td->o.verify_offset && hdr_size(td, hdr))
memswp(p, p + td->o.verify_offset, hdr_size(td, hdr));
}
/*
* fill body of io_u->buf with random data and add a header with the
* checksum of choice
*/
void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
{
if (td->o.verify == VERIFY_NULL)
return;
io_u->numberio = td->io_issues[io_u->ddir];
fill_pattern_headers(td, io_u, 0, 0);
}
int get_next_verify(struct thread_data *td, struct io_u *io_u)
{
struct io_piece *ipo = NULL;
/*
* this io_u is from a requeue, we already filled the offsets
*/
if (io_u->file)
return 0;
if (!RB_EMPTY_ROOT(&td->io_hist_tree)) {
struct rb_node *n = rb_first(&td->io_hist_tree);
ipo = rb_entry(n, struct io_piece, rb_node);
/*
* Ensure that the associated IO has completed
*/
read_barrier();
if (ipo->flags & IP_F_IN_FLIGHT)
goto nothing;
rb_erase(n, &td->io_hist_tree);
assert(ipo->flags & IP_F_ONRB);
ipo->flags &= ~IP_F_ONRB;
} else if (!flist_empty(&td->io_hist_list)) {
ipo = flist_first_entry(&td->io_hist_list, struct io_piece, list);
/*
* Ensure that the associated IO has completed
*/
read_barrier();
if (ipo->flags & IP_F_IN_FLIGHT)
goto nothing;
flist_del(&ipo->list);
assert(ipo->flags & IP_F_ONLIST);
ipo->flags &= ~IP_F_ONLIST;
}
if (ipo) {
td->io_hist_len--;
io_u->offset = ipo->offset;
io_u->buflen = ipo->len;
io_u->numberio = ipo->numberio;
io_u->file = ipo->file;
io_u_set(io_u, IO_U_F_VER_LIST);
if (ipo->flags & IP_F_TRIMMED)
io_u_set(io_u, IO_U_F_TRIMMED);
if (!fio_file_open(io_u->file)) {
int r = td_io_open_file(td, io_u->file);
if (r) {
dprint(FD_VERIFY, "failed file %s open\n",
io_u->file->file_name);
return 1;
}
}
get_file(ipo->file);
assert(fio_file_open(io_u->file));
io_u->ddir = DDIR_READ;
io_u->xfer_buf = io_u->buf;
io_u->xfer_buflen = io_u->buflen;
remove_trim_entry(td, ipo);
free(ipo);
dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
if (!td->o.verify_pattern_bytes) {
io_u->rand_seed = __rand(&td->verify_state);
if (sizeof(int) != sizeof(long *))
io_u->rand_seed *= __rand(&td->verify_state);
}
return 0;
}
nothing:
dprint(FD_VERIFY, "get_next_verify: empty\n");
return 1;
}
static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha512 *vh = hdr_priv(hdr);
struct fio_sha512_ctx sha512_ctx = {
.buf = vh->sha512,
};
fio_sha512_init(&sha512_ctx);
fio_sha512_update(&sha512_ctx, p, len);
}
static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha256 *vh = hdr_priv(hdr);
struct fio_sha256_ctx sha256_ctx = {
.buf = vh->sha256,
};
fio_sha256_init(&sha256_ctx);
fio_sha256_update(&sha256_ctx, p, len);
}
static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha1 *vh = hdr_priv(hdr);
struct fio_sha1_ctx sha1_ctx = {
.H = vh->sha1,
};
fio_sha1_init(&sha1_ctx);
fio_sha1_update(&sha1_ctx, p, len);
}
static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc7 *vh = hdr_priv(hdr);
vh->crc7 = fio_crc7(p, len);
}
static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc16 *vh = hdr_priv(hdr);
vh->crc16 = fio_crc16(p, len);
}
static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc32 *vh = hdr_priv(hdr);
vh->crc32 = fio_crc32(p, len);
}
static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc32 *vh = hdr_priv(hdr);
vh->crc32 = fio_crc32c(p, len);
}
static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_crc64 *vh = hdr_priv(hdr);
vh->crc64 = fio_crc64(p, len);
}
static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_md5 *vh = hdr_priv(hdr);
struct fio_md5_ctx md5_ctx = {
.hash = (uint32_t *) vh->md5_digest,
};
fio_md5_init(&md5_ctx);
fio_md5_update(&md5_ctx, p, len);
}
static void populate_hdr(struct thread_data *td, struct io_u *io_u,
struct verify_header *hdr, unsigned int header_num,
unsigned int header_len)
{
unsigned int data_len;
void *data, *p;
p = (void *) hdr;
hdr->magic = FIO_HDR_MAGIC;
hdr->verify_type = td->o.verify;
hdr->len = header_len;
hdr->rand_seed = io_u->rand_seed;
hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32));
data_len = header_len - hdr_size(hdr);
data = p + hdr_size(hdr);
switch (td->o.verify) {
case VERIFY_MD5:
dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n",
io_u, hdr->len);
fill_md5(hdr, data, data_len);
break;
case VERIFY_CRC64:
dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc64(hdr, data, data_len);
break;
case VERIFY_CRC32C:
case VERIFY_CRC32C_INTEL:
dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
io_u, hdr->len);
fill_crc32c(hdr, data, data_len);
break;
case VERIFY_CRC32:
dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc32(hdr, data, data_len);
break;
case VERIFY_CRC16:
dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc16(hdr, data, data_len);
break;
case VERIFY_CRC7:
dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n",
io_u, hdr->len);
fill_crc7(hdr, data, data_len);
break;
case VERIFY_SHA256:
dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha256(hdr, data, data_len);
break;
case VERIFY_SHA512:
dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha512(hdr, data, data_len);
break;
case VERIFY_META:
dprint(FD_VERIFY, "fill meta io_u %p, len %u\n",
io_u, hdr->len);
fill_meta(hdr, td, io_u, header_num);
break;
case VERIFY_SHA1:
dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n",
io_u, hdr->len);
fill_sha1(hdr, data, data_len);
break;
case VERIFY_PATTERN:
/* nothing to do here */
break;
default:
log_err("fio: bad verify type: %d\n", td->o.verify);
assert(0);
}
if (td->o.verify_offset)
memswp(p, p + td->o.verify_offset, hdr_size(hdr));
}
/*
* fill body of io_u->buf with random data and add a header with the
* checksum of choice
*/
void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
{
if (td->o.verify == VERIFY_NULL)
return;
fill_pattern_headers(td, io_u, 0, 0);
}
int get_next_verify(struct thread_data *td, struct io_u *io_u)
{
struct io_piece *ipo = NULL;
/*
* this io_u is from a requeue, we already filled the offsets
*/
if (io_u->file)
return 0;
if (!RB_EMPTY_ROOT(&td->io_hist_tree)) {
struct rb_node *n = rb_first(&td->io_hist_tree);
ipo = rb_entry(n, struct io_piece, rb_node);
rb_erase(n, &td->io_hist_tree);
assert(ipo->flags & IP_F_ONRB);
ipo->flags &= ~IP_F_ONRB;
} else if (!flist_empty(&td->io_hist_list)) {
ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
flist_del(&ipo->list);
assert(ipo->flags & IP_F_ONLIST);
ipo->flags &= ~IP_F_ONLIST;
}
if (ipo) {
td->io_hist_len--;
io_u->offset = ipo->offset;
io_u->buflen = ipo->len;
io_u->file = ipo->file;
io_u->flags |= IO_U_F_VER_LIST;
if (ipo->flags & IP_F_TRIMMED)
io_u->flags |= IO_U_F_TRIMMED;
if (!fio_file_open(io_u->file)) {
int r = td_io_open_file(td, io_u->file);
if (r) {
dprint(FD_VERIFY, "failed file %s open\n",
io_u->file->file_name);
return 1;
}
}
get_file(ipo->file);
assert(fio_file_open(io_u->file));
io_u->ddir = DDIR_READ;
io_u->xfer_buf = io_u->buf;
io_u->xfer_buflen = io_u->buflen;
remove_trim_entry(td, ipo);
free(ipo);
dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
return 0;
}
dprint(FD_VERIFY, "get_next_verify: empty\n");
return 1;
}
void fio_verify_init(struct thread_data *td)
{
if (td->o.verify == VERIFY_CRC32C_INTEL ||
td->o.verify == VERIFY_CRC32C) {
crc32c_intel_probe();
}
}
static void *verify_async_thread(void *data)
{
struct thread_data *td = data;
struct io_u *io_u;
int ret = 0;
if (td->o.verify_cpumask_set &&
fio_setaffinity(td->pid, td->o.verify_cpumask)) {
log_err("fio: failed setting verify thread affinity\n");
goto done;
}
do {
FLIST_HEAD(list);
read_barrier();
if (td->verify_thread_exit)
break;
pthread_mutex_lock(&td->io_u_lock);
while (flist_empty(&td->verify_list) &&
!td->verify_thread_exit) {
ret = pthread_cond_wait(&td->verify_cond,
&td->io_u_lock);
if (ret) {
pthread_mutex_unlock(&td->io_u_lock);
break;
}
}
flist_splice_init(&td->verify_list, &list);
pthread_mutex_unlock(&td->io_u_lock);
if (flist_empty(&list))
continue;
while (!flist_empty(&list)) {
io_u = flist_entry(list.next, struct io_u, verify_list);
flist_del(&io_u->verify_list);
ret = verify_io_u(td, io_u);
put_io_u(td, io_u);
if (!ret)
continue;
if (td_non_fatal_error(td, ERROR_TYPE_VERIFY_BIT, ret)) {
update_error_count(td, ret);
td_clear_error(td);
ret = 0;
}
}
} while (!ret);
if (ret) {
td_verror(td, ret, "async_verify");
if (td->o.verify_fatal)
td->terminate = 1;
}
done:
pthread_mutex_lock(&td->io_u_lock);
td->nr_verify_threads--;
pthread_mutex_unlock(&td->io_u_lock);
pthread_cond_signal(&td->free_cond);
return NULL;
}
int verify_async_init(struct thread_data *td)
{
int i, ret;
pthread_attr_t attr;
pthread_attr_init(&attr);
pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN);
td->verify_thread_exit = 0;
td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async);
for (i = 0; i < td->o.verify_async; i++) {
ret = pthread_create(&td->verify_threads[i], &attr,
verify_async_thread, td);
if (ret) {
log_err("fio: async verify creation failed: %s\n",
strerror(ret));
break;
}
ret = pthread_detach(td->verify_threads[i]);
if (ret) {
log_err("fio: async verify thread detach failed: %s\n",
strerror(ret));
break;
}
td->nr_verify_threads++;
}
pthread_attr_destroy(&attr);
if (i != td->o.verify_async) {
log_err("fio: only %d verify threads started, exiting\n", i);
td->verify_thread_exit = 1;
write_barrier();
pthread_cond_broadcast(&td->verify_cond);
return 1;
}
return 0;
}