int get_next_trim(struct thread_data *td, struct io_u *io_u) { struct io_piece *ipo; /* * this io_u is from a requeue, we already filled the offsets */ if (io_u->file) return 0; if (flist_empty(&td->trim_list)) return 1; assert(td->trim_entries); ipo = flist_entry(td->trim_list.next, struct io_piece, trim_list); remove_trim_entry(td, ipo); io_u->offset = ipo->offset; io_u->buflen = ipo->len; io_u->file = ipo->file; /* * If not verifying that trimmed ranges return zeroed data, * remove this from the to-read verify lists */ if (!td->o.trim_zero) { if (ipo->flags & IP_F_ONLIST) flist_del(&ipo->list); else { assert(ipo->flags & IP_F_ONRB); rb_erase(&ipo->rb_node, &td->io_hist_tree); } td->io_hist_len--; free(ipo); } else ipo->flags |= IP_F_TRIMMED; if (!fio_file_open(io_u->file)) { int r = td_io_open_file(td, io_u->file); if (r) { dprint(FD_VERIFY, "failed file %s open\n", io_u->file->file_name); return 1; } } get_file(io_u->file); assert(fio_file_open(io_u->file)); io_u->ddir = DDIR_TRIM; io_u->xfer_buf = NULL; io_u->xfer_buflen = io_u->buflen; dprint(FD_VERIFY, "get_next_trim: ret io_u %p\n", io_u); return 0; }
static void fio_server_fork_item_done(struct fio_fork_item *ffi) { dprint(FD_NET, "pid %u exited, sig=%u, exitval=%d\n", (int) ffi->pid, ffi->signal, ffi->exitval); /* * Fold STOP and QUIT... */ fio_net_send_stop(server_fd, ffi->exitval, ffi->signal); fio_net_send_quit(server_fd); flist_del(&ffi->list); free(ffi); }
void unlog_io_piece(struct thread_data *td, struct io_u *io_u) { struct io_piece *ipo = io_u->ipo; if (!ipo) return; if (ipo->flags & IP_F_ONRB) rb_erase(&ipo->rb_node, &td->io_hist_tree); else if (ipo->flags & IP_F_ONLIST) flist_del(&ipo->list); free(ipo); io_u->ipo = NULL; td->io_hist_len--; }
int read_iolog_get(struct thread_data *td, struct io_u *io_u) { struct io_piece *ipo; unsigned long elapsed; while (!flist_empty(&td->io_log_list)) { int ret; ipo = flist_entry(td->io_log_list.next, struct io_piece, list); flist_del(&ipo->list); remove_trim_entry(td, ipo); ret = ipo_special(td, ipo); if (ret < 0) { free(ipo); break; } else if (ret > 0) { free(ipo); continue; } io_u->ddir = ipo->ddir; if (ipo->ddir != DDIR_WAIT) { io_u->offset = ipo->offset; io_u->buflen = ipo->len; io_u->file = td->files[ipo->fileno]; get_file(io_u->file); dprint(FD_IO, "iolog: get %llu/%lu/%s\n", io_u->offset, io_u->buflen, io_u->file->file_name); if (ipo->delay) iolog_delay(td, ipo->delay); } else { elapsed = mtime_since_genesis(); if (ipo->delay > elapsed) usec_sleep(td, (ipo->delay - elapsed) * 1000); } free(ipo); if (io_u->ddir != DDIR_WAIT) return 0; } td->done = 1; return 1; }
static int get_next_rand_offset(struct thread_data *td, struct fio_file *f, enum fio_ddir ddir, uint64_t *b) { struct rand_off *r; int i, ret = 1; if (!should_sort_io(td)) return get_off_from_method(td, f, ddir, b); if (!flist_empty(&td->next_rand_list)) { fetch: r = flist_first_entry(&td->next_rand_list, struct rand_off, list); flist_del(&r->list); *b = r->off; free(r); return 0; }
void prune_io_piece_log(struct thread_data *td) { struct io_piece *ipo; struct rb_node *n; while ((n = rb_first(&td->io_hist_tree)) != NULL) { ipo = rb_entry(n, struct io_piece, rb_node); rb_erase(n, &td->io_hist_tree); free(ipo); } while (!flist_empty(&td->io_hist_list)) { ipo = flist_entry(td->io_hist_list.next, struct io_piece, list); flist_del(&ipo->list); free(ipo); } }
void requeue_io_u(struct thread_data *td, struct io_u **io_u) { struct io_u *__io_u = *io_u; dprint(FD_IO, "requeue %p\n", __io_u); td_io_u_lock(td); __io_u->flags |= IO_U_F_FREE; if ((__io_u->flags & IO_U_F_FLIGHT) && ddir_rw(__io_u->ddir)) td->io_issues[__io_u->ddir]--; __io_u->flags &= ~IO_U_F_FLIGHT; if (__io_u->flags & IO_U_F_IN_CUR_DEPTH) td->cur_depth--; flist_del(&__io_u->list); flist_add_tail(&__io_u->list, &td->io_u_requeues); td_io_u_unlock(td); *io_u = NULL; }
static int get_next_rand_offset(struct thread_data *td, struct fio_file *f, enum fio_ddir ddir, uint64_t *b) { struct rand_off *r; int i, ret = 1; if (!should_sort_io(td)) return get_off_from_method(td, f, ddir, b); if (!flist_empty(&td->next_rand_list)) { struct rand_off *r; fetch: r = flist_entry(td->next_rand_list.next, struct rand_off, list); flist_del(&r->list); *b = r->off; free(r); return 0; } for (i = 0; i < td->o.verifysort_nr; i++) { r = malloc(sizeof(*r)); ret = get_off_from_method(td, f, ddir, &r->off); if (ret) { free(r); break; } flist_add(&r->list, &td->next_rand_list); } if (ret && !i) return ret; assert(!flist_empty(&td->next_rand_list)); flist_sort(NULL, &td->next_rand_list, flist_cmp); goto fetch; }
int read_iolog_get(struct thread_data *td, struct io_u *io_u) { struct io_piece *ipo; while (!flist_empty(&td->io_log_list)) { int ret; ipo = flist_entry(td->io_log_list.next, struct io_piece, list); flist_del(&ipo->list); ret = ipo_special(td, ipo); if (ret < 0) { free(ipo); break; } else if (ret > 0) { free(ipo); continue; } io_u->offset = ipo->offset; io_u->buflen = ipo->len; io_u->ddir = ipo->ddir; io_u->file = td->files[ipo->fileno]; get_file(io_u->file); dprint(FD_IO, "iolog: get %llu/%lu/%s\n", io_u->offset, io_u->buflen, io_u->file->file_name); if (ipo->delay) iolog_delay(td, ipo->delay); free(ipo); return 0; } td->done = 1; return 1; }
/* * Invoked from our compress helper thread, when logging would have exceeded * the specified memory limitation. Compresses the previously stored * entries. */ static int gz_work(struct tp_work *work) { struct iolog_flush_data *data; struct iolog_compress *c; struct flist_head list; unsigned int seq; z_stream stream; size_t total = 0; int ret; INIT_FLIST_HEAD(&list); data = container_of(work, struct iolog_flush_data, work); stream.zalloc = Z_NULL; stream.zfree = Z_NULL; stream.opaque = Z_NULL; ret = deflateInit(&stream, Z_DEFAULT_COMPRESSION); if (ret != Z_OK) { log_err("fio: failed to init gz stream\n"); return 0; } seq = ++data->log->chunk_seq; stream.next_in = (void *) data->samples; stream.avail_in = data->nr_samples * log_entry_sz(data->log); dprint(FD_COMPRESS, "deflate input size=%lu, seq=%u\n", (unsigned long) stream.avail_in, seq); do { c = get_new_chunk(seq); stream.avail_out = GZ_CHUNK; stream.next_out = c->buf; ret = deflate(&stream, Z_NO_FLUSH); if (ret < 0) { log_err("fio: deflate log (%d)\n", ret); free_chunk(c); goto err; } c->len = GZ_CHUNK - stream.avail_out; flist_add_tail(&c->list, &list); total += c->len; } while (stream.avail_in); stream.next_out = c->buf + c->len; stream.avail_out = GZ_CHUNK - c->len; ret = deflate(&stream, Z_FINISH); if (ret == Z_STREAM_END) c->len = GZ_CHUNK - stream.avail_out; else { do { c = get_new_chunk(seq); stream.avail_out = GZ_CHUNK; stream.next_out = c->buf; ret = deflate(&stream, Z_FINISH); c->len = GZ_CHUNK - stream.avail_out; total += c->len; flist_add_tail(&c->list, &list); } while (ret != Z_STREAM_END); } dprint(FD_COMPRESS, "deflated to size=%lu\n", (unsigned long) total); ret = deflateEnd(&stream); if (ret != Z_OK) log_err("fio: deflateEnd %d\n", ret); free(data->samples); if (!flist_empty(&list)) { pthread_mutex_lock(&data->log->chunk_lock); flist_splice_tail(&list, &data->log->chunk_list); pthread_mutex_unlock(&data->log->chunk_lock); } ret = 0; done: if (work->wait) { work->done = 1; pthread_cond_signal(&work->cv); } else free(data); return ret; err: while (!flist_empty(&list)) { c = flist_first_entry(list.next, struct iolog_compress, list); flist_del(&c->list); free_chunk(c); } ret = 1; goto done; }
/* * Inflate stored compressed chunks, or write them directly to the log * file if so instructed. */ static int inflate_gz_chunks(struct io_log *log, FILE *f) { struct inflate_chunk_iter iter = { .chunk_sz = log->log_gz, }; z_stream stream; while (!flist_empty(&log->chunk_list)) { struct iolog_compress *ic; ic = flist_first_entry(&log->chunk_list, struct iolog_compress, list); flist_del(&ic->list); if (log->log_gz_store) { size_t ret; dprint(FD_COMPRESS, "log write chunk size=%lu, " "seq=%u\n", (unsigned long) ic->len, ic->seq); ret = fwrite(ic->buf, ic->len, 1, f); if (ret != 1 || ferror(f)) { iter.err = errno; log_err("fio: error writing compressed log\n"); } } else inflate_chunk(ic, log->log_gz_store, f, &stream, &iter); free_chunk(ic); } if (iter.seq) { finish_chunk(&stream, f, &iter); free(iter.buf); } return iter.err; } /* * Open compressed log file and decompress the stored chunks and * write them to stdout. The chunks are stored sequentially in the * file, so we iterate over them and do them one-by-one. */ int iolog_file_inflate(const char *file) { struct inflate_chunk_iter iter = { .chunk_sz = 64 * 1024 * 1024, }; struct iolog_compress ic; z_stream stream; struct stat sb; ssize_t ret; size_t total; void *buf; FILE *f; f = fopen(file, "r"); if (!f) { perror("fopen"); return 1; } if (stat(file, &sb) < 0) { fclose(f); perror("stat"); return 1; } ic.buf = buf = malloc(sb.st_size); ic.len = sb.st_size; ic.seq = 1; ret = fread(ic.buf, ic.len, 1, f); if (ret < 0) { perror("fread"); fclose(f); return 1; } else if (ret != 1) { log_err("fio: short read on reading log\n"); fclose(f); return 1; } fclose(f); /* * Each chunk will return Z_STREAM_END. We don't know how many * chunks are in the file, so we just keep looping and incrementing * the sequence number until we have consumed the whole compressed * file. */ total = ic.len; do { size_t ret; ret = inflate_chunk(&ic, 1, stdout, &stream, &iter); total -= ret; if (!total) break; if (iter.err) break; ic.seq++; ic.len -= ret; ic.buf += ret; } while (1); if (iter.seq) { finish_chunk(&stream, stdout, &iter); free(iter.buf); } free(buf); return iter.err; } #else static int inflate_gz_chunks(struct io_log *log, FILE *f) { return 0; } int iolog_file_inflate(const char *file) { log_err("fio: log inflation not possible without zlib\n"); return 1; } #endif void flush_log(struct io_log *log) { void *buf; FILE *f; f = fopen(log->filename, "w"); if (!f) { perror("fopen log"); return; } buf = set_file_buffer(f); inflate_gz_chunks(log, f); flush_samples(f, log->log, log->nr_samples * log_entry_sz(log)); fclose(f); clear_file_buffer(buf); }
static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha512 *vh = hdr_priv(hdr); struct fio_sha512_ctx sha512_ctx = { .buf = vh->sha512, }; fio_sha512_init(&sha512_ctx); fio_sha512_update(&sha512_ctx, p, len); } static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha256 *vh = hdr_priv(hdr); struct fio_sha256_ctx sha256_ctx = { .buf = vh->sha256, }; fio_sha256_init(&sha256_ctx); fio_sha256_update(&sha256_ctx, p, len); fio_sha256_final(&sha256_ctx); } static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha1 *vh = hdr_priv(hdr); struct fio_sha1_ctx sha1_ctx = { .H = vh->sha1, }; fio_sha1_init(&sha1_ctx); fio_sha1_update(&sha1_ctx, p, len); fio_sha1_final(&sha1_ctx); } static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc7 *vh = hdr_priv(hdr); vh->crc7 = fio_crc7(p, len); } static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc16 *vh = hdr_priv(hdr); vh->crc16 = fio_crc16(p, len); } static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc32 *vh = hdr_priv(hdr); vh->crc32 = fio_crc32(p, len); } static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc32 *vh = hdr_priv(hdr); vh->crc32 = fio_crc32c(p, len); } static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc64 *vh = hdr_priv(hdr); vh->crc64 = fio_crc64(p, len); } static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_md5 *vh = hdr_priv(hdr); struct fio_md5_ctx md5_ctx = { .hash = (uint32_t *) vh->md5_digest, }; fio_md5_init(&md5_ctx); fio_md5_update(&md5_ctx, p, len); fio_md5_final(&md5_ctx); } static void __fill_hdr(struct verify_header *hdr, int verify_type, uint32_t len, uint64_t rand_seed) { void *p = hdr; hdr->magic = FIO_HDR_MAGIC; hdr->verify_type = verify_type; hdr->len = len; hdr->rand_seed = rand_seed; hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32)); } static void fill_hdr(struct verify_header *hdr, int verify_type, uint32_t len, uint64_t rand_seed) { if (verify_type != VERIFY_PATTERN_NO_HDR) __fill_hdr(hdr, verify_type, len, rand_seed); } static void populate_hdr(struct thread_data *td, struct io_u *io_u, struct verify_header *hdr, unsigned int header_num, unsigned int header_len) { unsigned int data_len; void *data, *p; p = (void *) hdr; fill_hdr(hdr, td->o.verify, header_len, io_u->rand_seed); data_len = header_len - hdr_size(td, hdr); data = p + hdr_size(td, hdr); switch (td->o.verify) { case VERIFY_MD5: dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", io_u, hdr->len); fill_md5(hdr, data, data_len); break; case VERIFY_CRC64: dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", io_u, hdr->len); fill_crc64(hdr, data, data_len); break; case VERIFY_CRC32C: case VERIFY_CRC32C_INTEL: dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", io_u, hdr->len); fill_crc32c(hdr, data, data_len); break; case VERIFY_CRC32: dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", io_u, hdr->len); fill_crc32(hdr, data, data_len); break; case VERIFY_CRC16: dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", io_u, hdr->len); fill_crc16(hdr, data, data_len); break; case VERIFY_CRC7: dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", io_u, hdr->len); fill_crc7(hdr, data, data_len); break; case VERIFY_SHA256: dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", io_u, hdr->len); fill_sha256(hdr, data, data_len); break; case VERIFY_SHA512: dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", io_u, hdr->len); fill_sha512(hdr, data, data_len); break; case VERIFY_XXHASH: dprint(FD_VERIFY, "fill xxhash io_u %p, len %u\n", io_u, hdr->len); fill_xxhash(hdr, data, data_len); break; case VERIFY_META: dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", io_u, hdr->len); fill_meta(hdr, td, io_u, header_num); break; case VERIFY_SHA1: dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n", io_u, hdr->len); fill_sha1(hdr, data, data_len); break; case VERIFY_PATTERN: case VERIFY_PATTERN_NO_HDR: /* nothing to do here */ break; default: log_err("fio: bad verify type: %d\n", td->o.verify); assert(0); } if (td->o.verify_offset && hdr_size(td, hdr)) memswp(p, p + td->o.verify_offset, hdr_size(td, hdr)); } /* * fill body of io_u->buf with random data and add a header with the * checksum of choice */ void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) { if (td->o.verify == VERIFY_NULL) return; io_u->numberio = td->io_issues[io_u->ddir]; fill_pattern_headers(td, io_u, 0, 0); } int get_next_verify(struct thread_data *td, struct io_u *io_u) { struct io_piece *ipo = NULL; /* * this io_u is from a requeue, we already filled the offsets */ if (io_u->file) return 0; if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { struct rb_node *n = rb_first(&td->io_hist_tree); ipo = rb_entry(n, struct io_piece, rb_node); /* * Ensure that the associated IO has completed */ read_barrier(); if (ipo->flags & IP_F_IN_FLIGHT) goto nothing; rb_erase(n, &td->io_hist_tree); assert(ipo->flags & IP_F_ONRB); ipo->flags &= ~IP_F_ONRB; } else if (!flist_empty(&td->io_hist_list)) { ipo = flist_first_entry(&td->io_hist_list, struct io_piece, list); /* * Ensure that the associated IO has completed */ read_barrier(); if (ipo->flags & IP_F_IN_FLIGHT) goto nothing; flist_del(&ipo->list); assert(ipo->flags & IP_F_ONLIST); ipo->flags &= ~IP_F_ONLIST; } if (ipo) { td->io_hist_len--; io_u->offset = ipo->offset; io_u->buflen = ipo->len; io_u->numberio = ipo->numberio; io_u->file = ipo->file; io_u_set(io_u, IO_U_F_VER_LIST); if (ipo->flags & IP_F_TRIMMED) io_u_set(io_u, IO_U_F_TRIMMED); if (!fio_file_open(io_u->file)) { int r = td_io_open_file(td, io_u->file); if (r) { dprint(FD_VERIFY, "failed file %s open\n", io_u->file->file_name); return 1; } } get_file(ipo->file); assert(fio_file_open(io_u->file)); io_u->ddir = DDIR_READ; io_u->xfer_buf = io_u->buf; io_u->xfer_buflen = io_u->buflen; remove_trim_entry(td, ipo); free(ipo); dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); if (!td->o.verify_pattern_bytes) { io_u->rand_seed = __rand(&td->verify_state); if (sizeof(int) != sizeof(long *)) io_u->rand_seed *= __rand(&td->verify_state); } return 0; } nothing: dprint(FD_VERIFY, "get_next_verify: empty\n"); return 1; }
static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha512 *vh = hdr_priv(hdr); struct fio_sha512_ctx sha512_ctx = { .buf = vh->sha512, }; fio_sha512_init(&sha512_ctx); fio_sha512_update(&sha512_ctx, p, len); } static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha256 *vh = hdr_priv(hdr); struct fio_sha256_ctx sha256_ctx = { .buf = vh->sha256, }; fio_sha256_init(&sha256_ctx); fio_sha256_update(&sha256_ctx, p, len); } static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_sha1 *vh = hdr_priv(hdr); struct fio_sha1_ctx sha1_ctx = { .H = vh->sha1, }; fio_sha1_init(&sha1_ctx); fio_sha1_update(&sha1_ctx, p, len); } static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc7 *vh = hdr_priv(hdr); vh->crc7 = fio_crc7(p, len); } static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc16 *vh = hdr_priv(hdr); vh->crc16 = fio_crc16(p, len); } static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc32 *vh = hdr_priv(hdr); vh->crc32 = fio_crc32(p, len); } static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc32 *vh = hdr_priv(hdr); vh->crc32 = fio_crc32c(p, len); } static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc64 *vh = hdr_priv(hdr); vh->crc64 = fio_crc64(p, len); } static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_md5 *vh = hdr_priv(hdr); struct fio_md5_ctx md5_ctx = { .hash = (uint32_t *) vh->md5_digest, }; fio_md5_init(&md5_ctx); fio_md5_update(&md5_ctx, p, len); } static void populate_hdr(struct thread_data *td, struct io_u *io_u, struct verify_header *hdr, unsigned int header_num, unsigned int header_len) { unsigned int data_len; void *data, *p; p = (void *) hdr; hdr->magic = FIO_HDR_MAGIC; hdr->verify_type = td->o.verify; hdr->len = header_len; hdr->rand_seed = io_u->rand_seed; hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32)); data_len = header_len - hdr_size(hdr); data = p + hdr_size(hdr); switch (td->o.verify) { case VERIFY_MD5: dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", io_u, hdr->len); fill_md5(hdr, data, data_len); break; case VERIFY_CRC64: dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", io_u, hdr->len); fill_crc64(hdr, data, data_len); break; case VERIFY_CRC32C: case VERIFY_CRC32C_INTEL: dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", io_u, hdr->len); fill_crc32c(hdr, data, data_len); break; case VERIFY_CRC32: dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", io_u, hdr->len); fill_crc32(hdr, data, data_len); break; case VERIFY_CRC16: dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", io_u, hdr->len); fill_crc16(hdr, data, data_len); break; case VERIFY_CRC7: dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", io_u, hdr->len); fill_crc7(hdr, data, data_len); break; case VERIFY_SHA256: dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", io_u, hdr->len); fill_sha256(hdr, data, data_len); break; case VERIFY_SHA512: dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", io_u, hdr->len); fill_sha512(hdr, data, data_len); break; case VERIFY_META: dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", io_u, hdr->len); fill_meta(hdr, td, io_u, header_num); break; case VERIFY_SHA1: dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n", io_u, hdr->len); fill_sha1(hdr, data, data_len); break; case VERIFY_PATTERN: /* nothing to do here */ break; default: log_err("fio: bad verify type: %d\n", td->o.verify); assert(0); } if (td->o.verify_offset) memswp(p, p + td->o.verify_offset, hdr_size(hdr)); } /* * fill body of io_u->buf with random data and add a header with the * checksum of choice */ void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) { if (td->o.verify == VERIFY_NULL) return; fill_pattern_headers(td, io_u, 0, 0); } int get_next_verify(struct thread_data *td, struct io_u *io_u) { struct io_piece *ipo = NULL; /* * this io_u is from a requeue, we already filled the offsets */ if (io_u->file) return 0; if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { struct rb_node *n = rb_first(&td->io_hist_tree); ipo = rb_entry(n, struct io_piece, rb_node); rb_erase(n, &td->io_hist_tree); assert(ipo->flags & IP_F_ONRB); ipo->flags &= ~IP_F_ONRB; } else if (!flist_empty(&td->io_hist_list)) { ipo = flist_entry(td->io_hist_list.next, struct io_piece, list); flist_del(&ipo->list); assert(ipo->flags & IP_F_ONLIST); ipo->flags &= ~IP_F_ONLIST; } if (ipo) { td->io_hist_len--; io_u->offset = ipo->offset; io_u->buflen = ipo->len; io_u->file = ipo->file; io_u->flags |= IO_U_F_VER_LIST; if (ipo->flags & IP_F_TRIMMED) io_u->flags |= IO_U_F_TRIMMED; if (!fio_file_open(io_u->file)) { int r = td_io_open_file(td, io_u->file); if (r) { dprint(FD_VERIFY, "failed file %s open\n", io_u->file->file_name); return 1; } } get_file(ipo->file); assert(fio_file_open(io_u->file)); io_u->ddir = DDIR_READ; io_u->xfer_buf = io_u->buf; io_u->xfer_buflen = io_u->buflen; remove_trim_entry(td, ipo); free(ipo); dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); return 0; } dprint(FD_VERIFY, "get_next_verify: empty\n"); return 1; } void fio_verify_init(struct thread_data *td) { if (td->o.verify == VERIFY_CRC32C_INTEL || td->o.verify == VERIFY_CRC32C) { crc32c_intel_probe(); } } static void *verify_async_thread(void *data) { struct thread_data *td = data; struct io_u *io_u; int ret = 0; if (td->o.verify_cpumask_set && fio_setaffinity(td->pid, td->o.verify_cpumask)) { log_err("fio: failed setting verify thread affinity\n"); goto done; } do { FLIST_HEAD(list); read_barrier(); if (td->verify_thread_exit) break; pthread_mutex_lock(&td->io_u_lock); while (flist_empty(&td->verify_list) && !td->verify_thread_exit) { ret = pthread_cond_wait(&td->verify_cond, &td->io_u_lock); if (ret) { pthread_mutex_unlock(&td->io_u_lock); break; } } flist_splice_init(&td->verify_list, &list); pthread_mutex_unlock(&td->io_u_lock); if (flist_empty(&list)) continue; while (!flist_empty(&list)) { io_u = flist_entry(list.next, struct io_u, verify_list); flist_del(&io_u->verify_list); ret = verify_io_u(td, io_u); put_io_u(td, io_u); if (!ret) continue; if (td_non_fatal_error(td, ERROR_TYPE_VERIFY_BIT, ret)) { update_error_count(td, ret); td_clear_error(td); ret = 0; } } } while (!ret); if (ret) { td_verror(td, ret, "async_verify"); if (td->o.verify_fatal) td->terminate = 1; } done: pthread_mutex_lock(&td->io_u_lock); td->nr_verify_threads--; pthread_mutex_unlock(&td->io_u_lock); pthread_cond_signal(&td->free_cond); return NULL; } int verify_async_init(struct thread_data *td) { int i, ret; pthread_attr_t attr; pthread_attr_init(&attr); pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN); td->verify_thread_exit = 0; td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async); for (i = 0; i < td->o.verify_async; i++) { ret = pthread_create(&td->verify_threads[i], &attr, verify_async_thread, td); if (ret) { log_err("fio: async verify creation failed: %s\n", strerror(ret)); break; } ret = pthread_detach(td->verify_threads[i]); if (ret) { log_err("fio: async verify thread detach failed: %s\n", strerror(ret)); break; } td->nr_verify_threads++; } pthread_attr_destroy(&attr); if (i != td->o.verify_async) { log_err("fio: only %d verify threads started, exiting\n", i); td->verify_thread_exit = 1; write_barrier(); pthread_cond_broadcast(&td->verify_cond); return 1; } return 0; }